Files
mesh-controller/cmd/mesh-controller/config_moved_integration_test.go
T

468 lines
20 KiB
Go

//go:build integration
package main
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"reflect"
"sort"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/overlay"
)
func aConfiguredMesh(t *testing.T) (*stores, configuredMesh, map[string]map[string]any) {
t.Helper()
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
for i, name := range []string{"workstation", "desktop"} {
record, err := inv.AddNode(ctx, name)
if err != nil {
t.Fatal(err)
}
if err := inv.SetPlace(ctx, name, name+".example:51820", "here", false, fmt.Sprintf("10.77.0.%d", i+3)); err != nil {
t.Fatal(err)
}
if err := inv.RecordProfile(ctx, record.ID, map[string]any{"capabilities": []any{
map[string]any{"name": "container-runtime", "present": true},
map[string]any{"name": "systemd", "present": true}}}); err != nil {
t.Fatal(err)
}
if err := inv.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
t.Fatal(err)
}
if err := inv.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil {
t.Fatal(err)
}
if _, err := inv.Assign(ctx, name, overlay.Name); err != nil {
t.Fatal(err)
}
}
raw, err := os.ReadFile(filepath.Join("testdata", "config-moved", "mesh.json"))
if err != nil {
t.Fatal(err)
}
var w configuredMesh
if err := json.Unmarshal(raw, &w); err != nil {
t.Fatal(err)
}
list, err := catalogue.ParsePredefined(w.Predefined)
if err != nil {
t.Fatal(err)
}
catalogue.UsePredefined(list)
served := map[string]map[string]any{}
for _, text := range w.Modules {
var shape map[string]json.RawMessage
if err := json.Unmarshal(text, &shape); err != nil {
t.Fatal(err)
}
var name string
_ = json.Unmarshal(shape["module"], &name)
if s, ok := shape["serves-checked"]; ok {
var serves map[string]any
if err := json.Unmarshal(s, &serves); err != nil {
t.Fatal(err)
}
served[name] = serves
delete(shape, "serves-checked")
}
text, err := json.Marshal(shape)
if err != nil {
t.Fatal(err)
}
m, err := catalogue.ParseManifest(text)
if err != nil {
t.Fatalf("%s: %v", name, err)
}
register(t, open, m)
}
for _, l := range w.Layers {
values, err := json.Marshal(l.Values)
if err != nil {
t.Fatal(err)
}
if l.Node == "" {
inv.ExecForTest(t, `insert into settings (node, module, values) values (null, $1, $2)`, l.Module, string(values))
continue
}
inv.ExecForTest(t, `insert into settings (node, module, values) select id, $2, $3 from node where name = $1`,
l.Node, l.Module, string(values))
}
for _, module := range movedKeys(w.On) {
for _, node := range w.On[module] {
if _, err := assign(ctx, open, node, module); err != nil {
t.Fatalf("assign %s %s: %v", node, module, err)
}
}
}
return open, w, served
}
func TestEveryMachineIsToldTheSameAfterTheMoveOfConfig(t *testing.T) {
open, w, served := aConfiguredMesh(t)
ctx := t.Context()
readBackTries = 0
t.Cleanup(func() { readBackTries = 15 })
inv := open.inventory
config := &standInMeshConfig{open: open, values: map[string]any{}}
inv.ReadsConfigFrom(config.laid)
before := toldEveryMachine(t, open, served, w.On)
listedBefore := effectiveEverywhere(t, open, w)
factsBefore, err := gatherFacts(ctx, open, "2.11.17")
if err != nil {
t.Fatal(err)
}
asked, err := runConfigMove(ctx, inv, config, false, "the move of config, run by a test", "")
if err != nil || !asked.DryRun || asked.Switched || len(asked.Problems) > 0 || len(config.sets) > 0 {
t.Fatalf("the move, not confirmed, says %+v (%v) and set %v", asked, err, config.sets)
}
// nodered's live layer says the file's name for its time zone: set under the config key (issue 751).
if s := setAt(asked, "module.anchor.nodered.time-zone"); len(s) != 1 || s[0].Value != "Europe/Brussels" ||
!strings.HasSuffix(s[0].From[0], ": timeZone") {
t.Errorf("nodered's time zone moves as %+v", s)
}
for _, s := range asked.Sets {
if s.Key == "timeZone" {
t.Errorf("a key is set under the file's name: %+v", s)
}
}
// claude-code's whole-mesh managed settings become mesh config for agent-managed-settings (ADR 0348's
// consequences), and its whole-mesh layer sets no module value.
if s := setAt(asked, "mesh.agent-managed-settings"); len(s) != 1 || s[0].From[0] != "claude-code on the whole mesh: managed_settings" {
t.Errorf("claude-code's managed settings move as %+v", s)
}
for _, node := range fourMachines {
if s := setAt(asked, "module."+node+".claude-code.managed_settings"); len(s) > 0 {
t.Errorf("claude-code's managed settings are also set on %s: %+v", node, s)
}
}
// A whole-mesh layer of a value no predefined key holds goes to each machine it is assigned to, under that
// machine's layer: power's on the workstation is the workstation's, its own elsewhere.
if s := setAt(asked, "module.workstation.power.handle-power-key"); len(s) != 1 || s[0].Value != "suspend" {
t.Errorf("power's key on the workstation moves as %+v", s)
}
if s := setAt(asked, "module.anchor.power.handle-power-key"); len(s) != 1 || s[0].Value != "poweroff" {
t.Errorf("power's key on the anchor moves as %+v", s)
}
for _, s := range asked.Sets {
if s.Level == "node" {
t.Errorf("a machine's layer became node config: %+v", s)
}
}
var stays []string
for _, s := range asked.Stays {
stays = append(stays, strings.Fields(s)[0])
}
sort.Strings(stays)
if want := []string{"bazarr", "bazarr", "bookshelf", "jackett", "kernel", "kometa", "lidarr", "nzbget", "ombi",
"plex", "qbittorrent", "radarr", "sonarr", "tautulli"}; len(stays) < 10 || !containsString(stays, "kernel") {
t.Errorf("the layers that stay are %v, want those of %v", stays, want)
}
if after := toldEveryMachine(t, open, served, w.On); !reflect.DeepEqual(after, before) {
t.Fatal("a dry run changed what a machine is told")
}
moved, err := runConfigMove(ctx, inv, config, true, "the move of config, run by a test", "release 57 is delivered")
if err != nil || moved.DryRun || !moved.Switched {
t.Fatalf("the move says %+v (%v)", moved, err)
}
if len(config.sets) != len(asked.Sets) {
t.Fatalf("the move set %d values, and said it would set %d", len(config.sets), len(asked.Sets))
}
if held := inv.QueryForTest(t, `select module from settings s where values <> '{}'
and module in (select name from module where manifest ? 'config')`); len(held) != 0 {
t.Fatalf("a layer of a module in the config form still holds values: %v", held)
}
after := toldEveryMachine(t, open, served, w.On)
for _, node := range fourMachines {
if after[node] != before[node] {
t.Errorf("%s is told something else after the move:\n%s\nwas:\n%s", node, after[node], before[node])
}
}
if listedAfter := effectiveEverywhere(t, open, w); !reflect.DeepEqual(listedAfter, listedBefore) {
t.Errorf("settings lists other values after the move:\n%v\nwas:\n%v", listedAfter, listedBefore)
}
// And config lists what settings listed: each module's resolved value on each machine is the value settings
// said before the move.
resolved, err := config.Resolved(ctx)
if err != nil {
t.Fatal(err)
}
for _, r := range resolved {
said, listed := listedBefore[r.Node+"."+r.Module+"."+r.Key]
if r.Source == "unset" {
if listed && said != nil {
t.Errorf("%s of %s on %s is unset in config, and settings said %v", r.Key, r.Module, r.Node, said)
}
continue
}
var v any
_ = json.Unmarshal(r.Value, &v)
if !listed || !reflect.DeepEqual(v, said) {
t.Errorf("config resolves %s of %s on %s as %v, and settings said %v", r.Key, r.Module, r.Node, v, said)
}
}
// The facts snapshot carries mesh-config's values for the merge gate, as each machine's layer of each module, and
// each machine's declaration composes to the digest it did before the move.
factsAfter, err := gatherFacts(ctx, open, "2.11.17")
if err != nil {
t.Fatal(err)
}
// Machines are named in a snapshot by stand-ins of its own; each is found by what it composes.
digests := func(f snapshotFacts) ([]string, string) {
var out []string
zone := ""
for _, m := range f.Machines {
out = append(out, m.Declaration.Digest)
for _, l := range m.Settings {
if l.Module == "nodered" {
zone = compactJSON(l.Values["time-zone"])
}
}
}
sort.Strings(out)
return out, zone
}
was, _ := digests(factsBefore)
now, zone := digests(factsAfter)
if zone != `"Europe/Brussels"` {
t.Errorf("the facts snapshot does not carry nodered's time zone from mesh-config: %s", zone)
}
if len(now) != len(fourMachines) || now[0] == "" || !reflect.DeepEqual(now, was) {
t.Errorf("the declarations in the facts snapshot are %v after the move, %v before", now, was)
}
for _, s := range factsAfter.Settings {
if _, isConfig := map[string]bool{"betterbird": true, "claude-code": true, "power": true, "xdg": true}[s.Module]; isConfig &&
len(s.Values) > 0 {
t.Errorf("the facts snapshot still carries the whole mesh's layer of %s: %v", s.Module, s.Values)
}
}
config.sets = nil
again, err := runConfigMove(ctx, inv, config, true, "the move of config, run by a test", "again")
if err != nil || len(config.sets) != 0 || len(again.Moved) != 0 {
t.Fatalf("asked again, the move set %v and says %+v (%v)", config.sets, again, err)
}
if after := toldEveryMachine(t, open, served, w.On); !reflect.DeepEqual(after, before) {
t.Fatal("the move asked again changed what a machine is told")
}
}
func TestTheMoveOfConfigRefusesWhatWouldChangeAMachine(t *testing.T) {
open, _, _ := aConfiguredMesh(t)
ctx := t.Context()
readBackTries = 0
t.Cleanup(func() { readBackTries = 15 })
inv := open.inventory
config := &standInMeshConfig{open: open, values: map[string]any{}}
inv.ReadsConfigFrom(config.laid)
inv.ExecForTest(t, `update settings set values = values || '{"endpoints": {"web": {"reach": "public"}}}'
where module = 'nodered'`)
inv.ExecForTest(t, `update settings set values = values || '{"colour": "blue"}' where module = 'unifi'`)
inv.ExecForTest(t, `update settings set values = '{"font-size": 900}' where module = 'dunst'`)
answer, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered")
if err == nil || answer.Switched || len(config.sets) > 0 {
t.Fatalf("the move went ahead: %+v (%v), set %v", answer, err, config.sets)
}
for _, want := range []string{"nodered on anchor still holds placement (endpoints): run move-placement first",
`unifi on anchor says "colour", which unifi does not declare`, "font-size at dunst's own value on workstation"} {
if !strings.Contains(err.Error(), want) {
t.Errorf("the refusal does not say %q: %v", want, err)
}
}
inv.ExecForTest(t, `update settings set values = values - 'endpoints' where module = 'nodered'`)
inv.ExecForTest(t, `update settings set values = values - 'colour' where module = 'unifi'`)
inv.ExecForTest(t, `update settings set values = '{"font-size": 13, "width": 500}' where module = 'dunst'`)
// Set in mesh-config before the move, on a module whose layers say nothing of it: the switch would lay it.
config.values["module.anchor.grafana."+movedKeys(func() map[string]catalogue.SettingDeclaration {
shelf, _ := inv.Catalogue(ctx)
return shelf["grafana"].Settings
}())[0]] = "set before the move"
answer, err = runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered")
if err == nil || !strings.Contains(err.Error(), "grafana on anchor resolves") || answer.Switched {
t.Fatalf("a value no layer says was laid by the switch: %+v (%v)", answer, err)
}
if moved, _ := inv.ConfigMoved(ctx); moved.Moved {
t.Fatal("the switch is on after a refused move")
}
if held := inv.QueryForTest(t, `select count(*) from settings where values <> '{}'`); held[0][0] == "0" {
t.Fatal("a refused move emptied the layers")
}
}
func TestTheSettingsVerbForwardsToConfigOnceTheValuesMoved(t *testing.T) {
open, w, served := aConfiguredMesh(t)
ctx := t.Context()
inv := open.inventory
readBackTries = 0
t.Cleanup(func() { readBackTries = 15 })
config := &standInMeshConfig{open: open, values: map[string]any{}}
inv.ReadsConfigFrom(config.laid)
savedRead, savedAsk := readConfigValues, configAsk
readConfigValues, configAsk = config.laid, config.ask
t.Cleanup(func() { readConfigValues, configAsk = savedRead, savedAsk })
if _, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil {
t.Fatal(err)
}
shown := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"show", "nodered", "--node", "anchor"}) })
for _, want := range []string{`"time-zone": "Europe/Brussels"`, `time-zone = "Europe/Brussels" (module)`,
"mesh-config.config", `"mqtt": {`} {
if !strings.Contains(shown, want) {
t.Errorf("settings show does not say %q:\n%s", want, shown)
}
}
if strings.Contains(shown, "timeZone") {
t.Errorf("settings show still says the file's name:\n%s", shown)
}
config.asked = nil
said := stdoutOf(t, func() error {
return settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12, "width": 500}`, "--node", "workstation"})
})
if config.values["module.workstation.dunst.font-size"] != float64(12) || !strings.Contains(said, "~ font-size") ||
containsString(config.asked, "set module.workstation.dunst.width") {
t.Fatalf("settings set reached mesh-config as %v (%v):\n%s", config.values, config.asked, said)
}
told := toldEveryMachine(t, open, served, w.On)["workstation"]
if !strings.Contains(told, "12") {
t.Error("the workstation is not told the font size set through settings")
}
if err := settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12}`, "--node", "workstation"}); err == nil ||
!strings.Contains(err.Error(), "would no longer set width") {
t.Fatalf("a set dropping a value without --replace: %v", err)
}
if err := settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12}`}); err == nil ||
!strings.Contains(err.Error(), "no own value for the whole mesh") {
t.Fatalf("a module value for the whole mesh: %v", err)
}
t.Setenv(verbVar, "settings")
for _, c := range []struct{ module, node, values, key, held, why string }{
// A file root reads: refused as before the move (novox/hq issue 339).
{"hostname", "anchor", `{"hostname": "elsewhere"}`, "module.anchor.hostname.hostname", "Ace", "issue 339"},
{"messenger", "laptop", `{"desktop-machines": ["g14", "shanks"], "time-zone": "UTC"}`,
"module.laptop.messenger.time-zone", "Europe/Brussels", "issue 339"},
} {
err := settingsCommand(ctx, []string{"set", c.module, c.values, "--node", c.node})
if err == nil || !strings.Contains(err.Error(), c.why) || config.values[c.key] != c.held {
t.Fatalf("a trusted value of %s through a verb: %v, holds %v", c.module, err, config.values[c.key])
}
}
// And a trusted value no file makes the terminal's is refused as trusted, since mesh-config takes the controller's
// word for it (ADR 0348 §8).
if err := refuseTrustedThroughAVerb(ctx, inv, "messenger", "laptop", []string{"time-zone"}); err == nil ||
!strings.Contains(err.Error(), "ADR 0348 §8") {
t.Fatalf("a trusted value through a verb: %v", err)
}
if err := refuseTrustedThroughAVerb(ctx, inv, "dunst", "workstation", []string{"width"}); err != nil {
t.Fatalf("a value nobody trusts through a verb: %v", err)
}
stdoutOf(t, func() error {
return settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12, "width": 600}`, "--node", "workstation"})
})
if config.values["module.workstation.dunst.width"] != float64(600) {
t.Fatalf("a value nobody trusts was not set through a verb: %v", config.values["module.workstation.dunst.width"])
}
t.Setenv(verbVar, "")
stdoutOf(t, func() error { return settingsCommand(ctx, []string{"clear", "dunst", "--node", "workstation"}) })
for _, k := range []string{"font-size", "width"} {
if _, held := config.values["module.workstation.dunst."+k]; held {
t.Errorf("settings clear left dunst's %s on the workstation", k)
}
}
config.asked = nil
proposed := stdoutOf(t, func() error {
return settingsCommand(ctx, []string{"propose", "hostname", `{"hostname": "elsewhere"}`, "--node", "anchor"})
})
if !containsString(config.asked, "propose module.anchor.hostname.hostname") || !strings.Contains(proposed, "p-1") {
t.Fatalf("settings propose reached mesh-config as %v:\n%s", config.asked, proposed)
}
config.asked = nil
kept := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"show", "kernel", "--node", "laptop"}) })
if !strings.Contains(kept, "transparent-huge-pages") || len(config.asked) > 0 {
t.Fatalf("a module in the settings form is shown as %q, asking mesh-config %v", kept, config.asked)
}
}
func TestAnApprovalRaisedBeforeTheMoveOfConfigIsRefusedAfterIt(t *testing.T) {
open, _, _ := aConfiguredMesh(t)
ctx := t.Context()
readBackTries = 0
t.Cleanup(func() { readBackTries = 15 })
inv := open.inventory
config := &standInMeshConfig{open: open, values: map[string]any{}}
inv.ReadsConfigFrom(config.laid)
shelf, err := inv.Catalogue(ctx)
if err != nil {
t.Fatal(err)
}
key := movedKeys(func() map[string]catalogue.SettingDeclaration { return shelf["grafana"].Settings }())[0]
values := map[string]any{key: "raised before the move"}
p := settingsProposal{Module: "grafana", Node: "anchor", Values: values, From: "g14/claude-code", At: time.Now(),
Digest: layerDigest(values), BeforeDigest: layerDigest(nil)}
if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has {
t.Fatalf("grafana has a layer on anchor before the test: %v %v", has, err)
}
if moved, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil || !moved.Switched {
t.Fatalf("the move says %+v (%v)", moved, err)
}
if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads {
t.Fatalf("grafana does not read config after the move: %v %v", reads, err)
}
_, err = setLayerIn(open)(ctx, p, "s30", "approved by the operator")
if err == nil || !strings.Contains(err.Error(), "moved to mesh-config since the operator was shown the change") ||
!strings.Contains(err.Error(), "propose it again") {
t.Fatalf("the stale approval: %v", err)
}
if layer, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has && len(layer) > 0 {
t.Errorf("the refused approval wrote the layer: %v %v %v", layer, has, err)
}
if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads {
t.Errorf("the refused approval turned ReadsConfig off: %v %v", reads, err)
}
}
func TestAPlacementOnlyApprovalRaisedBeforeTheMoveIsSetAfterIt(t *testing.T) {
open, _, _ := aConfiguredMesh(t)
ctx := t.Context()
readBackTries = 0
t.Cleanup(func() { readBackTries = 15 })
inv := open.inventory
config := &standInMeshConfig{open: open, values: map[string]any{}}
inv.ReadsConfigFrom(config.laid)
values := map[string]any{catalogue.PlacesSetting: map[string]any{"data": "/srv/grafana-data"}}
p := settingsProposal{Module: "grafana", Node: "anchor", Values: values, From: "g14/claude-code", At: time.Now(),
Digest: layerDigest(values), BeforeDigest: layerDigest(nil)}
if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has {
t.Fatalf("grafana has a layer on anchor before the test: %v %v", has, err)
}
if moved, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil || !moved.Switched {
t.Fatalf("the move says %+v (%v)", moved, err)
}
if _, err := setLayerIn(open)(ctx, p, "s31", "approved by the operator"); err != nil {
t.Fatalf("the placement-only approval after the move: %v", err)
}
if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || !has {
t.Errorf("the approval did not set the layer: %v %v", has, err)
}
if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads {
t.Errorf("the approval turned ReadsConfig off: %v %v", reads, err)
}
}