//go:build integration package main import ( "encoding/json" "fmt" "os" "path/filepath" "reflect" "sort" "strings" "testing" "time" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/overlay" ) func aConfiguredMesh(t *testing.T) (*stores, configuredMesh, map[string]map[string]any) { t.Helper() open := aMesh(t) ctx := t.Context() inv := open.inventory for i, name := range []string{"workstation", "desktop"} { record, err := inv.AddNode(ctx, name) if err != nil { t.Fatal(err) } if err := inv.SetPlace(ctx, name, name+".example:51820", "here", false, fmt.Sprintf("10.77.0.%d", i+3)); err != nil { t.Fatal(err) } if err := inv.RecordProfile(ctx, record.ID, map[string]any{"capabilities": []any{ map[string]any{"name": "container-runtime", "present": true}, map[string]any{"name": "systemd", "present": true}}}); err != nil { t.Fatal(err) } if err := inv.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil { t.Fatal(err) } if err := inv.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil { t.Fatal(err) } if _, err := inv.Assign(ctx, name, overlay.Name); err != nil { t.Fatal(err) } } raw, err := os.ReadFile(filepath.Join("testdata", "config-moved", "mesh.json")) if err != nil { t.Fatal(err) } var w configuredMesh if err := json.Unmarshal(raw, &w); err != nil { t.Fatal(err) } list, err := catalogue.ParsePredefined(w.Predefined) if err != nil { t.Fatal(err) } catalogue.UsePredefined(list) served := map[string]map[string]any{} for _, text := range w.Modules { var shape map[string]json.RawMessage if err := json.Unmarshal(text, &shape); err != nil { t.Fatal(err) } var name string _ = json.Unmarshal(shape["module"], &name) if s, ok := shape["serves-checked"]; ok { var serves map[string]any if err := json.Unmarshal(s, &serves); err != nil { t.Fatal(err) } served[name] = serves delete(shape, "serves-checked") } text, err := json.Marshal(shape) if err != nil { t.Fatal(err) } m, err := catalogue.ParseManifest(text) if err != nil { t.Fatalf("%s: %v", name, err) } register(t, open, m) } for _, l := range w.Layers { values, err := json.Marshal(l.Values) if err != nil { t.Fatal(err) } if l.Node == "" { inv.ExecForTest(t, `insert into settings (node, module, values) values (null, $1, $2)`, l.Module, string(values)) continue } inv.ExecForTest(t, `insert into settings (node, module, values) select id, $2, $3 from node where name = $1`, l.Node, l.Module, string(values)) } for _, module := range movedKeys(w.On) { for _, node := range w.On[module] { if _, err := assign(ctx, open, node, module); err != nil { t.Fatalf("assign %s %s: %v", node, module, err) } } } return open, w, served } func TestEveryMachineIsToldTheSameAfterTheMoveOfConfig(t *testing.T) { open, w, served := aConfiguredMesh(t) ctx := t.Context() readBackTries = 0 t.Cleanup(func() { readBackTries = 15 }) inv := open.inventory config := &standInMeshConfig{open: open, values: map[string]any{}} inv.ReadsConfigFrom(config.laid) before := toldEveryMachine(t, open, served, w.On) listedBefore := effectiveEverywhere(t, open, w) factsBefore, err := gatherFacts(ctx, open, "2.11.17") if err != nil { t.Fatal(err) } asked, err := runConfigMove(ctx, inv, config, false, "the move of config, run by a test", "") if err != nil || !asked.DryRun || asked.Switched || len(asked.Problems) > 0 || len(config.sets) > 0 { t.Fatalf("the move, not confirmed, says %+v (%v) and set %v", asked, err, config.sets) } // nodered's live layer says the file's name for its time zone: set under the config key (issue 751). if s := setAt(asked, "module.anchor.nodered.time-zone"); len(s) != 1 || s[0].Value != "Europe/Brussels" || !strings.HasSuffix(s[0].From[0], ": timeZone") { t.Errorf("nodered's time zone moves as %+v", s) } for _, s := range asked.Sets { if s.Key == "timeZone" { t.Errorf("a key is set under the file's name: %+v", s) } } // claude-code's whole-mesh managed settings become mesh config for agent-managed-settings (ADR 0348's // consequences), and its whole-mesh layer sets no module value. if s := setAt(asked, "mesh.agent-managed-settings"); len(s) != 1 || s[0].From[0] != "claude-code on the whole mesh: managed_settings" { t.Errorf("claude-code's managed settings move as %+v", s) } for _, node := range fourMachines { if s := setAt(asked, "module."+node+".claude-code.managed_settings"); len(s) > 0 { t.Errorf("claude-code's managed settings are also set on %s: %+v", node, s) } } // A whole-mesh layer of a value no predefined key holds goes to each machine it is assigned to, under that // machine's layer: power's on the workstation is the workstation's, its own elsewhere. if s := setAt(asked, "module.workstation.power.handle-power-key"); len(s) != 1 || s[0].Value != "suspend" { t.Errorf("power's key on the workstation moves as %+v", s) } if s := setAt(asked, "module.anchor.power.handle-power-key"); len(s) != 1 || s[0].Value != "poweroff" { t.Errorf("power's key on the anchor moves as %+v", s) } for _, s := range asked.Sets { if s.Level == "node" { t.Errorf("a machine's layer became node config: %+v", s) } } var stays []string for _, s := range asked.Stays { stays = append(stays, strings.Fields(s)[0]) } sort.Strings(stays) if want := []string{"bazarr", "bazarr", "bookshelf", "jackett", "kernel", "kometa", "lidarr", "nzbget", "ombi", "plex", "qbittorrent", "radarr", "sonarr", "tautulli"}; len(stays) < 10 || !containsString(stays, "kernel") { t.Errorf("the layers that stay are %v, want those of %v", stays, want) } if after := toldEveryMachine(t, open, served, w.On); !reflect.DeepEqual(after, before) { t.Fatal("a dry run changed what a machine is told") } moved, err := runConfigMove(ctx, inv, config, true, "the move of config, run by a test", "release 57 is delivered") if err != nil || moved.DryRun || !moved.Switched { t.Fatalf("the move says %+v (%v)", moved, err) } if len(config.sets) != len(asked.Sets) { t.Fatalf("the move set %d values, and said it would set %d", len(config.sets), len(asked.Sets)) } if held := inv.QueryForTest(t, `select module from settings s where values <> '{}' and module in (select name from module where manifest ? 'config')`); len(held) != 0 { t.Fatalf("a layer of a module in the config form still holds values: %v", held) } after := toldEveryMachine(t, open, served, w.On) for _, node := range fourMachines { if after[node] != before[node] { t.Errorf("%s is told something else after the move:\n%s\nwas:\n%s", node, after[node], before[node]) } } if listedAfter := effectiveEverywhere(t, open, w); !reflect.DeepEqual(listedAfter, listedBefore) { t.Errorf("settings lists other values after the move:\n%v\nwas:\n%v", listedAfter, listedBefore) } // And config lists what settings listed: each module's resolved value on each machine is the value settings // said before the move. resolved, err := config.Resolved(ctx) if err != nil { t.Fatal(err) } for _, r := range resolved { said, listed := listedBefore[r.Node+"."+r.Module+"."+r.Key] if r.Source == "unset" { if listed && said != nil { t.Errorf("%s of %s on %s is unset in config, and settings said %v", r.Key, r.Module, r.Node, said) } continue } var v any _ = json.Unmarshal(r.Value, &v) if !listed || !reflect.DeepEqual(v, said) { t.Errorf("config resolves %s of %s on %s as %v, and settings said %v", r.Key, r.Module, r.Node, v, said) } } // The facts snapshot carries mesh-config's values for the merge gate, as each machine's layer of each module, and // each machine's declaration composes to the digest it did before the move. factsAfter, err := gatherFacts(ctx, open, "2.11.17") if err != nil { t.Fatal(err) } // Machines are named in a snapshot by stand-ins of its own; each is found by what it composes. digests := func(f snapshotFacts) ([]string, string) { var out []string zone := "" for _, m := range f.Machines { out = append(out, m.Declaration.Digest) for _, l := range m.Settings { if l.Module == "nodered" { zone = compactJSON(l.Values["time-zone"]) } } } sort.Strings(out) return out, zone } was, _ := digests(factsBefore) now, zone := digests(factsAfter) if zone != `"Europe/Brussels"` { t.Errorf("the facts snapshot does not carry nodered's time zone from mesh-config: %s", zone) } if len(now) != len(fourMachines) || now[0] == "" || !reflect.DeepEqual(now, was) { t.Errorf("the declarations in the facts snapshot are %v after the move, %v before", now, was) } for _, s := range factsAfter.Settings { if _, isConfig := map[string]bool{"betterbird": true, "claude-code": true, "power": true, "xdg": true}[s.Module]; isConfig && len(s.Values) > 0 { t.Errorf("the facts snapshot still carries the whole mesh's layer of %s: %v", s.Module, s.Values) } } config.sets = nil again, err := runConfigMove(ctx, inv, config, true, "the move of config, run by a test", "again") if err != nil || len(config.sets) != 0 || len(again.Moved) != 0 { t.Fatalf("asked again, the move set %v and says %+v (%v)", config.sets, again, err) } if after := toldEveryMachine(t, open, served, w.On); !reflect.DeepEqual(after, before) { t.Fatal("the move asked again changed what a machine is told") } } func TestTheMoveOfConfigRefusesWhatWouldChangeAMachine(t *testing.T) { open, _, _ := aConfiguredMesh(t) ctx := t.Context() readBackTries = 0 t.Cleanup(func() { readBackTries = 15 }) inv := open.inventory config := &standInMeshConfig{open: open, values: map[string]any{}} inv.ReadsConfigFrom(config.laid) inv.ExecForTest(t, `update settings set values = values || '{"endpoints": {"web": {"reach": "public"}}}' where module = 'nodered'`) inv.ExecForTest(t, `update settings set values = values || '{"colour": "blue"}' where module = 'unifi'`) inv.ExecForTest(t, `update settings set values = '{"font-size": 900}' where module = 'dunst'`) answer, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered") if err == nil || answer.Switched || len(config.sets) > 0 { t.Fatalf("the move went ahead: %+v (%v), set %v", answer, err, config.sets) } for _, want := range []string{"nodered on anchor still holds placement (endpoints): run move-placement first", `unifi on anchor says "colour", which unifi does not declare`, "font-size at dunst's own value on workstation"} { if !strings.Contains(err.Error(), want) { t.Errorf("the refusal does not say %q: %v", want, err) } } inv.ExecForTest(t, `update settings set values = values - 'endpoints' where module = 'nodered'`) inv.ExecForTest(t, `update settings set values = values - 'colour' where module = 'unifi'`) inv.ExecForTest(t, `update settings set values = '{"font-size": 13, "width": 500}' where module = 'dunst'`) // Set in mesh-config before the move, on a module whose layers say nothing of it: the switch would lay it. config.values["module.anchor.grafana."+movedKeys(func() map[string]catalogue.SettingDeclaration { shelf, _ := inv.Catalogue(ctx) return shelf["grafana"].Settings }())[0]] = "set before the move" answer, err = runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered") if err == nil || !strings.Contains(err.Error(), "grafana on anchor resolves") || answer.Switched { t.Fatalf("a value no layer says was laid by the switch: %+v (%v)", answer, err) } if moved, _ := inv.ConfigMoved(ctx); moved.Moved { t.Fatal("the switch is on after a refused move") } if held := inv.QueryForTest(t, `select count(*) from settings where values <> '{}'`); held[0][0] == "0" { t.Fatal("a refused move emptied the layers") } } func TestTheSettingsVerbForwardsToConfigOnceTheValuesMoved(t *testing.T) { open, w, served := aConfiguredMesh(t) ctx := t.Context() inv := open.inventory readBackTries = 0 t.Cleanup(func() { readBackTries = 15 }) config := &standInMeshConfig{open: open, values: map[string]any{}} inv.ReadsConfigFrom(config.laid) savedRead, savedAsk := readConfigValues, configAsk readConfigValues, configAsk = config.laid, config.ask t.Cleanup(func() { readConfigValues, configAsk = savedRead, savedAsk }) if _, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil { t.Fatal(err) } shown := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"show", "nodered", "--node", "anchor"}) }) for _, want := range []string{`"time-zone": "Europe/Brussels"`, `time-zone = "Europe/Brussels" (module)`, "mesh-config.config", `"mqtt": {`} { if !strings.Contains(shown, want) { t.Errorf("settings show does not say %q:\n%s", want, shown) } } if strings.Contains(shown, "timeZone") { t.Errorf("settings show still says the file's name:\n%s", shown) } config.asked = nil said := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12, "width": 500}`, "--node", "workstation"}) }) if config.values["module.workstation.dunst.font-size"] != float64(12) || !strings.Contains(said, "~ font-size") || containsString(config.asked, "set module.workstation.dunst.width") { t.Fatalf("settings set reached mesh-config as %v (%v):\n%s", config.values, config.asked, said) } told := toldEveryMachine(t, open, served, w.On)["workstation"] if !strings.Contains(told, "12") { t.Error("the workstation is not told the font size set through settings") } if err := settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12}`, "--node", "workstation"}); err == nil || !strings.Contains(err.Error(), "would no longer set width") { t.Fatalf("a set dropping a value without --replace: %v", err) } if err := settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12}`}); err == nil || !strings.Contains(err.Error(), "no own value for the whole mesh") { t.Fatalf("a module value for the whole mesh: %v", err) } t.Setenv(verbVar, "settings") for _, c := range []struct{ module, node, values, key, held, why string }{ // A file root reads: refused as before the move (novox/hq issue 339). {"hostname", "anchor", `{"hostname": "elsewhere"}`, "module.anchor.hostname.hostname", "Ace", "issue 339"}, {"messenger", "laptop", `{"desktop-machines": ["g14", "shanks"], "time-zone": "UTC"}`, "module.laptop.messenger.time-zone", "Europe/Brussels", "issue 339"}, } { err := settingsCommand(ctx, []string{"set", c.module, c.values, "--node", c.node}) if err == nil || !strings.Contains(err.Error(), c.why) || config.values[c.key] != c.held { t.Fatalf("a trusted value of %s through a verb: %v, holds %v", c.module, err, config.values[c.key]) } } // And a trusted value no file makes the terminal's is refused as trusted, since mesh-config takes the controller's // word for it (ADR 0348 §8). if err := refuseTrustedThroughAVerb(ctx, inv, "messenger", "laptop", []string{"time-zone"}); err == nil || !strings.Contains(err.Error(), "ADR 0348 §8") { t.Fatalf("a trusted value through a verb: %v", err) } if err := refuseTrustedThroughAVerb(ctx, inv, "dunst", "workstation", []string{"width"}); err != nil { t.Fatalf("a value nobody trusts through a verb: %v", err) } stdoutOf(t, func() error { return settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12, "width": 600}`, "--node", "workstation"}) }) if config.values["module.workstation.dunst.width"] != float64(600) { t.Fatalf("a value nobody trusts was not set through a verb: %v", config.values["module.workstation.dunst.width"]) } t.Setenv(verbVar, "") stdoutOf(t, func() error { return settingsCommand(ctx, []string{"clear", "dunst", "--node", "workstation"}) }) for _, k := range []string{"font-size", "width"} { if _, held := config.values["module.workstation.dunst."+k]; held { t.Errorf("settings clear left dunst's %s on the workstation", k) } } config.asked = nil proposed := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"propose", "hostname", `{"hostname": "elsewhere"}`, "--node", "anchor"}) }) if !containsString(config.asked, "propose module.anchor.hostname.hostname") || !strings.Contains(proposed, "p-1") { t.Fatalf("settings propose reached mesh-config as %v:\n%s", config.asked, proposed) } config.asked = nil kept := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"show", "kernel", "--node", "laptop"}) }) if !strings.Contains(kept, "transparent-huge-pages") || len(config.asked) > 0 { t.Fatalf("a module in the settings form is shown as %q, asking mesh-config %v", kept, config.asked) } } func TestAnApprovalRaisedBeforeTheMoveOfConfigIsRefusedAfterIt(t *testing.T) { open, _, _ := aConfiguredMesh(t) ctx := t.Context() readBackTries = 0 t.Cleanup(func() { readBackTries = 15 }) inv := open.inventory config := &standInMeshConfig{open: open, values: map[string]any{}} inv.ReadsConfigFrom(config.laid) shelf, err := inv.Catalogue(ctx) if err != nil { t.Fatal(err) } key := movedKeys(func() map[string]catalogue.SettingDeclaration { return shelf["grafana"].Settings }())[0] values := map[string]any{key: "raised before the move"} p := settingsProposal{Module: "grafana", Node: "anchor", Values: values, From: "g14/claude-code", At: time.Now(), Digest: layerDigest(values), BeforeDigest: layerDigest(nil)} if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has { t.Fatalf("grafana has a layer on anchor before the test: %v %v", has, err) } if moved, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil || !moved.Switched { t.Fatalf("the move says %+v (%v)", moved, err) } if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads { t.Fatalf("grafana does not read config after the move: %v %v", reads, err) } _, err = setLayerIn(open)(ctx, p, "s30", "approved by the operator") if err == nil || !strings.Contains(err.Error(), "moved to mesh-config since the operator was shown the change") || !strings.Contains(err.Error(), "propose it again") { t.Fatalf("the stale approval: %v", err) } if layer, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has && len(layer) > 0 { t.Errorf("the refused approval wrote the layer: %v %v %v", layer, has, err) } if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads { t.Errorf("the refused approval turned ReadsConfig off: %v %v", reads, err) } } func TestAPlacementOnlyApprovalRaisedBeforeTheMoveIsSetAfterIt(t *testing.T) { open, _, _ := aConfiguredMesh(t) ctx := t.Context() readBackTries = 0 t.Cleanup(func() { readBackTries = 15 }) inv := open.inventory config := &standInMeshConfig{open: open, values: map[string]any{}} inv.ReadsConfigFrom(config.laid) values := map[string]any{catalogue.PlacesSetting: map[string]any{"data": "/srv/grafana-data"}} p := settingsProposal{Module: "grafana", Node: "anchor", Values: values, From: "g14/claude-code", At: time.Now(), Digest: layerDigest(values), BeforeDigest: layerDigest(nil)} if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has { t.Fatalf("grafana has a layer on anchor before the test: %v %v", has, err) } if moved, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil || !moved.Switched { t.Fatalf("the move says %+v (%v)", moved, err) } if _, err := setLayerIn(open)(ctx, p, "s31", "approved by the operator"); err != nil { t.Fatalf("the placement-only approval after the move: %v", err) } if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || !has { t.Errorf("the approval did not set the layer: %v %v", has, err) } if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads { t.Errorf("the approval turned ReadsConfig off: %v %v", reads, err) } }