diff --git a/cmd/mesh-controller/placement_moved_test.go b/cmd/mesh-controller/placement_moved_test.go new file mode 100644 index 00000000..5831cc56 --- /dev/null +++ b/cmd/mesh-controller/placement_moved_test.go @@ -0,0 +1,113 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// **Placement leaves the settings layers and no machine is told anything new** (novox/hq ADR 0348 clause +// 11, issue 651). A mesh is seeded as the store holds it before migration 0097 — `places`, `accesses` and a +// kept network among the settings, at the mesh's level and a machine's, one layer of placement only — then +// migrated, and every machine's declaration is composed. What each machine is told of the seeded modules is +// byte for byte what the controller before this change composed from the same seed: the golden files were +// written by the integration branch's controller at bfc79c6 running this same test (MESH_WRITE_GOLDEN=1), +// where there was no migration 0097 to run. +func TestADeclarationIsTheSameAfterPlacementMoves(t *testing.T) { + open, migrate := aMeshBefore(t, 97) + ctx := t.Context() + inv := open.inventory + image := "registry.invalid:5000/library@sha256:" + strings.Repeat("a", 64) + register(t, open, catalogue.Manifest{Module: "library", Version: "1", + Accesses: []catalogue.Access{{ID: "series"}, {ID: "downloads", Mode: catalogue.AccessRead}}, + Resources: []map[string]any{ + {"id": "config", "type": "directory", "mode": "0750"}, + {"id": "server", "type": "container", "image": image, + "volumes": []any{"${dir:config}:/config", "${access:series}:/series", "${access:downloads}:/downloads:ro"}}, + // Today's form: a mergeable file naming no keys takes every key of the layer (ADR 0326 §3). + {"id": "runtime-config", "type": "file", "path": "${dir:config}/runtime.json", "mode": "0640", + "content": `{"log":"info"}`, "merge": "json"}, + }}) + register(t, open, catalogue.Manifest{Module: "found", Version: "1", + Resources: []map[string]any{{"id": "server", "type": "container", "image": image}}}) + if err := inv.SetAdopted(ctx, "anchor", true); err != nil { + t.Fatal(err) + } + for _, l := range []struct{ node, module, values string }{ + {"", "library", `{"accesses": {"downloads": "/storage/downloads"}, "log": "warn"}`}, + {"anchor", "library", `{"places": {"config": {"path": "/var/lib/library/config", "owner": "1001:2000"}}, + "accesses": {"series": "/storage/media/series"}, "workers": 4}`}, + {"laptop", "library", `{"accesses": {"series": "/home/media/series"}}`}, + {"anchor", "found", `{"networks": {"server": ["predecessor_default"]}}`}, + } { + if l.node == "" { + inv.ExecForTest(t, `insert into settings (node, module, values) values (null, $1, $2)`, l.module, l.values) + continue + } + inv.ExecForTest(t, `insert into settings (node, module, values) select id, $2, $3 from node where name = $1`, + l.node, l.module, l.values) + } + + migrate() + // Assigned once the store is migrated: an assignment reads the layers, as this controller holds them. + for _, a := range [][]string{{"anchor", "library", "found"}, {"laptop", "library"}} { + if _, err := assign(ctx, open, a[0], a[1:]...); err != nil { + t.Fatalf("assign %v: %v", a, err) + } + } + + gens, err := generators(ctx, open) + if err != nil { + t.Fatal(err) + } + for _, node := range []string{"anchor", "laptop"} { + plan, settings, err := planFor(ctx, open, node) + if err != nil { + t.Fatal(err) + } + with, _, err := renderingFor(ctx, open, node, plan, settings, gens, Allocating) + if err != nil { + t.Fatal(err) + } + composed, err := plan.Compose(with) + if err != nil { + t.Fatal(err) + } + if len(composed.LeftOut) > 0 { + t.Fatalf("%s leaves out %v", node, composed.LeftOut) + } + var told []map[string]any + for _, r := range composed.Resources { + if owner := composed.Owner[r["id"].(string)]; owner == "library" || owner == "found" { + told = append(told, r) + } + } + sort.Slice(told, func(i, j int) bool { return told[i]["id"].(string) < told[j]["id"].(string) }) + got, err := json.MarshalIndent(told, "", " ") + if err != nil { + t.Fatal(err) + } + golden := filepath.Join("testdata", "placement-moved", node+".json") + if os.Getenv("MESH_WRITE_GOLDEN") != "" { + if err := os.MkdirAll(filepath.Dir(golden), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(golden, append(got, '\n'), 0o644); err != nil { + t.Fatal(err) + } + continue + } + want, err := os.ReadFile(golden) + if err != nil { + t.Fatal(err) + } + if string(want) != string(got)+"\n" { + t.Errorf("%s is told something else after placement moved:\n%s\nwas:\n%s", node, got, want) + } + } +} diff --git a/internal/inventory/migrations/0097-placement-is-not-config.sql b/internal/inventory/migrations/0097-placement-is-not-config.sql new file mode 100644 index 00000000..553cf6d6 --- /dev/null +++ b/internal/inventory/migrations/0097-placement-is-not-config.sql @@ -0,0 +1,46 @@ +-- Placement is not config (novox/hq ADR 0348 clause 11, issue 651). +-- +-- Where a module's directories are on a machine (`places`), where the operator's data it reads is +-- (`accesses`) and which found networks a taken container keeps (`networks`) were kept as keys of a +-- settings layer. They are facts of an assignment that the controller decides, never config values, and +-- the config values are leaving for the mesh-config module (ADR 0348 clause 9) while placement stays with +-- the controller. So they get a table of their own beside the assignment, and every settings layer's +-- placement keys move into it, unchanged. +-- +-- Scoped exactly as the layer they came from: one machine's, or the whole mesh's (node null) for a +-- module's every assignment, which is how a mesh-wide `places` or `accesses` layer composes today. The +-- composer reads placement from here only; the `settings` verb still takes the three keys and routes +-- them here, so nothing a person types changes. +create table placement ( + node uuid references node(id) on delete cascade, + module text not null references module(name) on delete cascade, + -- Only `places`, `accesses` and `networks`, as the settings layer held them. + values jsonb not null, + set_at timestamptz not null default now(), + set_by text +); +create unique index placement_for_the_mesh on placement (module) where node is null; +create unique index placement_for_a_node on placement (node, module) where node is not null; + +-- Why a layer went into the history, beside how (`replaced_by`): said for a layer this migration moved. +alter table settings_history add column why text; + +-- Each layer that held a placement key is kept whole in the history first, as a `settings history` reads it. +insert into settings_history (node, module, values, set_at, replaced_by, set_by, why) +select node, module, values, set_at, 'moved', set_by, 'moved to placement, ADR 0348 clause 11' + from settings + where values ?| array['places', 'accesses', 'networks']; + +insert into placement (node, module, values, set_at, set_by) +select node, module, + (select jsonb_object_agg(e.key, e.value) from jsonb_each(values) as e + where e.key in ('places', 'accesses', 'networks')), + set_at, set_by + from settings + where values ?| array['places', 'accesses', 'networks']; + +-- The layer keeps its other keys, when it was set and who set it; a layer that held only placement stays, +-- empty, so a layer that existed still exists. +update settings + set values = values - 'places' - 'accesses' - 'networks' + where values ?| array['places', 'accesses', 'networks']; diff --git a/internal/inventory/placement_test.go b/internal/inventory/placement_test.go new file mode 100644 index 00000000..0483fe7b --- /dev/null +++ b/internal/inventory/placement_test.go @@ -0,0 +1,222 @@ +package inventory + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// placedLibrary is a module with each kind of placement: a directory to place, two accesses, a container that +// may keep a found network. +func placedLibrary() catalogue.Manifest { + return catalogue.Manifest{Module: "library", Version: "1", + Accesses: []catalogue.Access{{ID: "series"}, {ID: "downloads"}}, + Resources: []map[string]any{ + {"id": "config", "type": "directory", "mode": "0750"}, + {"id": "server", "type": "container", + "image": "registry.invalid:5000/library@sha256:" + strings.Repeat("a", 64), + "volumes": []any{"${dir:config}:/config", "${access:series}:/series", "${access:downloads}:/downloads"}}, + {"id": "runtime-config", "type": "file", "path": "${dir:config}/runtime.json", "mode": "0640", + "content": `{"log":"info"}`, "merge": "json"}, + }} +} + +// The layers of the issue (651): a mesh-wide layer with an access and a config value, a node's layer with a +// place, an access and a kept network beside a config value, a node's layer of placement only, and one with +// none. +var placementLayers = []struct { + node string + values string +}{ + {"", `{"accesses": {"downloads": "/storage/downloads"}, "log": "warn"}`}, + {"anchor", `{"places": {"config": {"path": "/var/lib/library/config", "owner": "1001:2000"}}, + "accesses": {"series": "/storage/media/series"}, "networks": {"server": ["predecessor_default"]}, "workers": 4}`}, + {"laptop", `{"accesses": {"series": "/home/media/series"}}`}, + {"spare", `{"log": "debug"}`}, +} + +// **Placement moves out of the settings layers, and every layer reads as it did** (novox/hq ADR 0348 clause +// 11, issue 651). Migration 0097 moves each layer's places, accesses and kept networks beside it: no settings +// row holds a placement key after, every layer the settings verb and composition read is what it was, each +// moved layer is kept whole in the history with why, and a layer without placement is not touched. +func TestPlacementMovesOutOfTheSettingsLayers(t *testing.T) { + inv, migrate := ForTestBefore(t, 97) + ctx := t.Context() + if err := inv.RegisterModule(ctx, placedLibrary(), Source{}); err != nil { + t.Fatal(err) + } + for _, n := range []string{"anchor", "laptop", "spare"} { + if _, err := inv.AddNode(ctx, n); err != nil { + t.Fatal(err) + } + } + if err := inv.SetAdopted(ctx, "anchor", true); err != nil { + t.Fatal(err) + } + // As the mesh holds them before the migration: placement among the settings. + for _, l := range placementLayers { + if l.node == "" { + inv.ExecForTest(t, `insert into settings (node, module, values, set_by) values (null, 'library', $1, 'the operator')`, l.values) + continue + } + inv.ExecForTest(t, `insert into settings (node, module, values) select id, 'library', $2 from node where name = $1`, + l.node, l.values) + } + setAt := inv.QueryForTest(t, `select coalesce(n.name, ''), s.set_at from settings s left join node n on n.id = s.node order by 1`) + + migrate() + + if held := inv.QueryForTest(t, `select module from settings where values ?| array['places', 'accesses', 'networks']`); len(held) != 0 { + t.Fatalf("a settings layer still holds a placement key: %v", held) + } + if after := inv.QueryForTest(t, `select coalesce(n.name, ''), s.set_at from settings s left join node n on n.id = s.node order by 1`); !reflect.DeepEqual(after, setAt) { + t.Fatalf("the layers or when they were set changed: %v, was %v", after, setAt) + } + for _, l := range placementLayers { + var want map[string]any + if err := json.Unmarshal([]byte(l.values), &want); err != nil { + t.Fatal(err) + } + got, has, err := inv.Layer(ctx, l.node, "library") + if err != nil || !has || !reflect.DeepEqual(roundTrip(t, got), want) { + t.Errorf("the layer on %q reads %v (%v, %v), was %v", l.node, got, has, err, want) + } + } + // Composition's view: the config values in Values, the placement in Placement, together the layer. + layers, err := inv.SettingsFor(ctx, "anchor", "library") + if err != nil || len(layers) != 2 { + t.Fatalf("anchor's layers: %v, %v", layers, err) + } + if layers[1].Values["workers"] == nil || layers[1].Values["places"] != nil || layers[1].Placement["places"] == nil || + layers[1].Placement["networks"] == nil || layers[0].Placement["accesses"] == nil || layers[0].Values["log"] != "warn" { + t.Fatalf("anchor's layers split wrong: %+v", layers) + } + for i, l := range layers { + var want map[string]any + _ = json.Unmarshal([]byte(placementLayers[i].values), &want) + if !reflect.DeepEqual(roundTrip(t, l.Whole()), want) { + t.Errorf("anchor's layer %s is %v whole, was %v", l.From, l.Whole(), want) + } + } + // Judged as composition judges them, as before: anchor is adopted, so its kept network composes. + if err := catalogue.JudgeSettings(placedLibrary(), layers, true); err != nil { + t.Fatalf("the moved layers no longer compose: %v", err) + } + + moved := inv.QueryForTest(t, `select coalesce(n.name, ''), h.replaced_by, h.why, h.set_by from settings_history h + left join node n on n.id = h.node order by 1`) + want := [][]string{ + {`""`, `"moved"`, `"moved to placement, ADR 0348 clause 11"`, `"the operator"`}, + {`"anchor"`, `"moved"`, `"moved to placement, ADR 0348 clause 11"`, `null`}, + {`"laptop"`, `"moved"`, `"moved to placement, ADR 0348 clause 11"`, `null`}, + } + if !reflect.DeepEqual(moved, want) { + t.Fatalf("the history of what moved: %v, want %v", moved, want) + } + past, err := inv.SettingsHistory(ctx, "anchor", "library") + if err != nil || len(past) != 1 || past[0].Why != "moved to placement, ADR 0348 clause 11" || past[0].Values["places"] == nil { + t.Fatalf("settings history on anchor: %+v, %v", past, err) + } +} + +// **The settings verb's path still takes a placement key, and routes it** (issue 651): a set keeps the +// placement beside the layer and the rest in it, reads back whole, is refused as before when the placement +// cannot compose (nothing written), and a set or a clear replaces the placement with the layer. +func TestASettingRoutesPlacementBesideTheLayer(t *testing.T) { + inv := ForTest(t) + ctx := t.Context() + if err := inv.RegisterModule(ctx, placedLibrary(), Source{}); err != nil { + t.Fatal(err) + } + if _, err := inv.AddNode(ctx, "anchor"); err != nil { + t.Fatal(err) + } + // Placement alone is a layer as well: the whole mesh's says where both accesses are. + meshWide := map[string]any{"accesses": map[string]any{ + "downloads": "/storage/downloads", "series": "/storage/series"}} + if err := inv.SetSettings(ctx, "", "library", meshWide); err != nil { + t.Fatal(err) + } + if got, has, _ := inv.Layer(ctx, "", "library"); !has || !reflect.DeepEqual(roundTrip(t, got), roundTrip(t, meshWide)) { + t.Fatalf("the mesh-wide layer of placement only reads %v (%v)", got, has) + } + set := map[string]any{"log": "warn", + "places": map[string]any{"config": map[string]any{"path": "/var/lib/library/config"}}, + "accesses": map[string]any{"series": "/storage/media/series"}} + if err := inv.SetSettings(ctx, "anchor", "library", set); err != nil { + t.Fatal(err) + } + stored := inv.QueryForTest(t, `select s.values, p.values from settings s join placement p using (module, node) + where s.node is not null`) + if len(stored) != 1 || stored[0][0] != `{"log":"warn"}` || + !strings.Contains(stored[0][1], `"places"`) || !strings.Contains(stored[0][1], `"accesses"`) { + t.Fatalf("the layer and its placement are kept as %v", stored) + } + if got, _, _ := inv.Layer(ctx, "anchor", "library"); !reflect.DeepEqual(roundTrip(t, got), roundTrip(t, set)) { + t.Fatalf("the layer reads %v, was set %v", got, set) + } + + // Refused as before, by the same judgement, and nothing changes. + for name, bad := range map[string]map[string]any{ + "an undeclared directory": {"places": map[string]any{"cache": "/var/lib/library/cache"}}, + "the machine's own": {"accesses": map[string]any{"series": "/etc"}}, + "a kept network on a converged machine": { + "networks": map[string]any{"server": []any{"predecessor_default"}}}, + } { + if err := inv.SetSettings(ctx, "anchor", "library", bad); err == nil { + t.Errorf("%s was kept", name) + } + } + if err := inv.SetSettings(ctx, "", "library", map[string]any{"accesses": meshWide["accesses"], + "networks": map[string]any{"server": []any{"x"}}}); err == nil || !strings.Contains(err.Error(), "per node") { + t.Errorf("a mesh-wide kept network: %v", err) + } + if got, _, _ := inv.Layer(ctx, "anchor", "library"); !reflect.DeepEqual(roundTrip(t, got), roundTrip(t, set)) { + t.Fatalf("a refused set changed the layer to %v", got) + } + + // A set without placement replaces it too, and the history keeps the layer whole. + if err := inv.SetSettings(ctx, "anchor", "library", map[string]any{"log": "debug"}); err != nil { + t.Fatal(err) + } + if rows := inv.QueryForTest(t, `select module from placement where node is not null`); len(rows) != 0 { + t.Fatalf("a set without placement left it: %v", rows) + } + past, err := inv.SettingsHistory(ctx, "anchor", "library") + if err != nil || len(past) != 1 || past[0].Values["places"] == nil || past[0].Values["log"] != "warn" { + t.Fatalf("the replaced layer is not kept whole: %+v, %v", past, err) + } + + // The merge gate's way in keeps a layer as the facts carry it, whole: split the same way. + if err := inv.KeepSettings(ctx, "anchor", "library", map[string]any{"log": "info", + "places": map[string]any{"config": "/var/lib/library/config"}}); err != nil { + t.Fatal(err) + } + if rows := inv.QueryForTest(t, `select values from settings where values ? 'places'`); len(rows) != 0 { + t.Fatalf("a kept layer put placement among the settings: %v", rows) + } + for _, node := range []string{"", "anchor"} { + if err := inv.ClearSettings(ctx, node, "library"); err != nil { + t.Fatal(err) + } + } + if rows := inv.QueryForTest(t, `select module from placement`); len(rows) != 0 { + t.Fatalf("a clear left the placement: %v", rows) + } +} + +func roundTrip(t *testing.T, v map[string]any) map[string]any { + t.Helper() + b, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + out := map[string]any{} + if err := json.Unmarshal(b, &out); err != nil { + t.Fatal(err) + } + return out +}