diff --git a/internal/inventory/catalogue.go b/internal/inventory/catalogue.go index 6ce3740c..615319be 100644 --- a/internal/inventory/catalogue.go +++ b/internal/inventory/catalogue.go @@ -117,6 +117,11 @@ func (i *Inventory) RegisterModule(ctx context.Context, m catalogue.Manifest, fr } } } + // **The manifest the mesh holds carries each file's text** (novox/hq ADR 0349 §2): one still naming a template + // would compose a file with no text, so it is refused by name — a build by a builder that predates templates. + if why := catalogue.HeldTemplateProblem(m); why != "" { + return fmt.Errorf("%s is not registered: %s", m.Module, why) + } raw, err := json.Marshal(m) if err != nil { return err @@ -839,7 +844,18 @@ func (i *Inventory) KeepSettings(ctx context.Context, nodeName, module string, v } func (i *Inventory) setSettings(ctx context.Context, nodeName, module string, values map[string]any, judge bool, setBy string) error { - raw, err := json.Marshal(values) + // **Placement is not config** (novox/hq ADR 0348 clause 11, issue 651): the layer is said whole, as + // it always was, and its placement keys are kept beside the assignment, never in the settings row. + // Both are written in one transaction, and the layer a set replaces is the whole of both. + config, placement := catalogue.SplitPlacement(values) + if config == nil { + config = map[string]any{} + } + raw, err := json.Marshal(config) + if err != nil { + return err + } + placed, err := placementJSON(placement) if err != nil { return err } @@ -881,6 +897,9 @@ func (i *Inventory) setSettings(ctx context.Context, nodeName, module string, va do update set values = excluded.values, set_at = now(), set_by = excluded.set_by`, module, raw, nullable(setBy)); err != nil { return wrapModule(err, module) } + if err := keepPlacement(ctx, tx, nil, module, placed, setBy); err != nil { + return err + } return tx.Commit(ctx) } node, err := i.NodeByName(ctx, nodeName) @@ -912,6 +931,9 @@ func (i *Inventory) setSettings(ctx context.Context, nodeName, module string, va if err != nil { return wrapModule(err, module) } + if err := keepPlacement(ctx, tx, &node.ID, module, placed, setBy); err != nil { + return err + } // A given port replaces what the mesh assigned for that port: the assignment is given back, // so the number is free for the next module rather than held for ever in the name of a port // that now lives elsewhere. @@ -942,21 +964,15 @@ func (i *Inventory) judgeSettings(ctx context.Context, nodeName, module string, return err } where, from, adopted = nodeName, nodeName, node.Adopted - var meshWide []byte - err = i.store.Pool().QueryRow(ctx, - `select values from settings where module = $1 and node is null`, module).Scan(&meshWide) - if err != nil && !errors.Is(err, pgx.ErrNoRows) { + under, has, err := i.Layer(ctx, "", module) + if err != nil { return err } - if len(meshWide) > 0 { - var under map[string]any - if err := json.Unmarshal(meshWide, &under); err != nil { - return err - } - layers = append(layers, catalogue.Layer{From: catalogue.MeshWideLayer, Values: under}) + if has { + layers = append(layers, catalogue.LayerOf(catalogue.MeshWideLayer, under)) } } - layers = append(layers, catalogue.Layer{From: from, Values: values}) + layers = append(layers, catalogue.LayerOf(from, values)) if err := catalogue.JudgeSettings(m, layers, adopted); err != nil { return fmt.Errorf("refused: %s on %s cannot compose with the layer %q — %w", module, where, from, err) } @@ -1089,7 +1105,8 @@ func refuseGivenCollisions(ctx context.Context, tx pgx.Tx, nodeID any, node, mod } func wrapModule(err error, module string) error { - if err != nil && strings.Contains(err.Error(), "settings_module_fkey") { + if err != nil && (strings.Contains(err.Error(), "settings_module_fkey") || + strings.Contains(err.Error(), "placement_module_fkey")) { return fmt.Errorf("%w: %s", ErrNoSuchModule, module) } return err @@ -1130,6 +1147,11 @@ func (i *Inventory) ClearSettingsBy(ctx context.Context, nodeName, module, clear `delete from settings where module = $1 and node is not distinct from $2::uuid`, module, nodeID); err != nil { return err } + // And its placement, which is the same layer as a person says it (novox/hq ADR 0348 clause 11). + if _, err := tx.Exec(ctx, + `delete from placement where module = $1 and node is not distinct from $2::uuid`, module, nodeID); err != nil { + return err + } return tx.Commit(ctx) } @@ -1142,10 +1164,13 @@ func (i *Inventory) SettingsFor(ctx context.Context, nodeName, module string) ([ if err != nil { return nil, err } + // Each layer's placement beside it, from the placement the controller holds (novox/hq ADR 0348 + // clause 11): a layer's Values hold its config values only, its Placement where things are. rows, err := i.store.Pool().Query(ctx, - `select node is null, values from settings - where module = $1 and (node is null or node = $2) - order by node is null desc`, module, node.ID) + `select s.node is null, s.values, p.values from settings s + left join placement p on p.module = s.module and p.node is not distinct from s.node + where s.module = $1 and (s.node is null or s.node = $2) + order by s.node is null desc`, module, node.ID) if err != nil { return nil, err } @@ -1154,23 +1179,62 @@ func (i *Inventory) SettingsFor(ctx context.Context, nodeName, module string) ([ var layers []catalogue.Layer for rows.Next() { var meshWide bool - var raw []byte - if err := rows.Scan(&meshWide, &raw); err != nil { + var raw, placed []byte + if err := rows.Scan(&meshWide, &raw, &placed); err != nil { return nil, err } values := map[string]any{} if err := json.Unmarshal(raw, &values); err != nil { return nil, err } + var placement map[string]any + if len(placed) > 0 { + if err := json.Unmarshal(placed, &placement); err != nil { + return nil, err + } + } from := nodeName if meshWide { from = catalogue.MeshWideLayer } - layers = append(layers, catalogue.Layer{From: from, Values: values}) + layers = append(layers, catalogue.Layer{From: from, Values: values, Placement: placement}) } return layers, rows.Err() } +// placementJSON is a layer's placement as the placement table keeps it, or nil when it has none. +func placementJSON(placement map[string]any) ([]byte, error) { + if len(placement) == 0 { + return nil, nil + } + return json.Marshal(placement) +} + +// keepPlacement writes one layer's placement beside its settings row, in the write's transaction: the +// placement the layer says, or none when it says none — a set replaces the layer whole, placement too. +func keepPlacement(ctx context.Context, tx pgx.Tx, nodeID *string, module string, placed []byte, setBy string) error { + if placed == nil { + _, err := tx.Exec(ctx, + `delete from placement where module = $1 and node is not distinct from $2::uuid`, module, nodeID) + return err + } + var err error + if nodeID == nil { + _, err = tx.Exec(ctx, + `insert into placement (node, module, values, set_by) values (null, $1, $2, $3) + on conflict (module) where node is null + do update set values = excluded.values, set_at = now(), set_by = excluded.set_by`, + module, placed, nullable(setBy)) + } else { + _, err = tx.Exec(ctx, + `insert into placement (node, module, values, set_by) values ($1, $2, $3, $4) + on conflict (node, module) where node is not null + do update set values = excluded.values, set_at = now(), set_by = excluded.set_by`, + *nodeID, module, placed, nullable(setBy)) + } + return wrapModule(err, module) +} + // PinProvision records which provider a machine gets a provision from: a module, and the node it // runs on — both, always (novox/hq #258). A provision comes from a module, and the same module on // two machines is two answers, so neither half alone says which. diff --git a/internal/inventory/fortest.go b/internal/inventory/fortest.go index 77aec9cc..cbd7f73a 100644 --- a/internal/inventory/fortest.go +++ b/internal/inventory/fortest.go @@ -2,6 +2,7 @@ package inventory import ( "context" + "encoding/json" "fmt" "os" "strings" @@ -75,3 +76,94 @@ func (i *Inventory) RemoveNodeForTest(ctx context.Context, name string) (bool, e } return tag.RowsAffected() == 1, nil } + +// ForTestBefore is ForTest with the store migrated only below migration `number`, and a function that runs the +// rest: for a test of a migration that moves data, which seeds the store as the mesh holds it before the +// migration, migrates, and reads what the migration made of it (novox/hq issue 651). +func ForTestBefore(t *testing.T, number int) (*Inventory, func()) { + t.Helper() + admin := os.Getenv("MESH_TEST_POSTGRES") + if admin == "" { + t.Skip("no MESH_TEST_POSTGRES; run `make check` to raise one") + } + name := fmt.Sprintf("invb_%d_%s", time.Now().UnixNano()%1_000_000, + strings.ToLower(strings.NewReplacer("/", "", "-", "").Replace(t.Name()))) + if len(name) > 60 { + name = name[:60] + } + all, err := Migrations() + if err != nil { + t.Fatal(err) + } + var before []store.Migration + for _, m := range all { + if m.Number < number { + before = append(before, m) + } + } + store.CreateMigratedForTest(t, admin, name, before) + busPasswordCostForTest.Store(int32(bcrypt.MinCost)) + cut := strings.LastIndex(admin, "/") + t.Setenv(store.Variable(Name), admin[:cut]+"/"+name+"?sslmode=disable") + inv, err := Open(t.Context()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + inv.Close() + c, err := pgx.Connect(context.Background(), admin) + if err != nil { + return + } + defer c.Close(context.Background()) + _, _ = c.Exec(context.Background(), "drop database if exists "+name+" with (force)") + }) + if err := inv.Ready(t.Context(), 20*time.Second); err != nil { + t.Fatal(err) + } + return inv, func() { + t.Helper() + if _, err := inv.store.Migrate(t.Context(), all); err != nil { + t.Fatal(err) + } + } +} + +// ExecForTest runs one statement against a test's store, for a fixture no verb of the mesh writes: a row as an +// older migration left it. +func (i *Inventory) ExecForTest(t *testing.T, sql string, args ...any) { + t.Helper() + if _, err := i.store.Pool().Exec(t.Context(), sql, args...); err != nil { + t.Fatalf("%s: %v", sql, err) + } +} + +// QueryForTest reads rows of a test's store, each row's columns as JSON text, for a test that checks what a +// migration left. +func (i *Inventory) QueryForTest(t *testing.T, sql string, args ...any) [][]string { + t.Helper() + rows, err := i.store.Pool().Query(t.Context(), sql, args...) + if err != nil { + t.Fatalf("%s: %v", sql, err) + } + defer rows.Close() + var out [][]string + for rows.Next() { + values, err := rows.Values() + if err != nil { + t.Fatal(err) + } + row := make([]string, len(values)) + for k, v := range values { + row[k] = fmt.Sprint(v) + if b, err := json.Marshal(v); err == nil { + row[k] = string(b) + } + } + out = append(out, row) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + return out +} diff --git a/internal/inventory/migrations/0097-placement-is-not-config.sql b/internal/inventory/migrations/0097-placement-is-not-config.sql new file mode 100644 index 00000000..553cf6d6 --- /dev/null +++ b/internal/inventory/migrations/0097-placement-is-not-config.sql @@ -0,0 +1,46 @@ +-- Placement is not config (novox/hq ADR 0348 clause 11, issue 651). +-- +-- Where a module's directories are on a machine (`places`), where the operator's data it reads is +-- (`accesses`) and which found networks a taken container keeps (`networks`) were kept as keys of a +-- settings layer. They are facts of an assignment that the controller decides, never config values, and +-- the config values are leaving for the mesh-config module (ADR 0348 clause 9) while placement stays with +-- the controller. So they get a table of their own beside the assignment, and every settings layer's +-- placement keys move into it, unchanged. +-- +-- Scoped exactly as the layer they came from: one machine's, or the whole mesh's (node null) for a +-- module's every assignment, which is how a mesh-wide `places` or `accesses` layer composes today. The +-- composer reads placement from here only; the `settings` verb still takes the three keys and routes +-- them here, so nothing a person types changes. +create table placement ( + node uuid references node(id) on delete cascade, + module text not null references module(name) on delete cascade, + -- Only `places`, `accesses` and `networks`, as the settings layer held them. + values jsonb not null, + set_at timestamptz not null default now(), + set_by text +); +create unique index placement_for_the_mesh on placement (module) where node is null; +create unique index placement_for_a_node on placement (node, module) where node is not null; + +-- Why a layer went into the history, beside how (`replaced_by`): said for a layer this migration moved. +alter table settings_history add column why text; + +-- Each layer that held a placement key is kept whole in the history first, as a `settings history` reads it. +insert into settings_history (node, module, values, set_at, replaced_by, set_by, why) +select node, module, values, set_at, 'moved', set_by, 'moved to placement, ADR 0348 clause 11' + from settings + where values ?| array['places', 'accesses', 'networks']; + +insert into placement (node, module, values, set_at, set_by) +select node, module, + (select jsonb_object_agg(e.key, e.value) from jsonb_each(values) as e + where e.key in ('places', 'accesses', 'networks')), + set_at, set_by + from settings + where values ?| array['places', 'accesses', 'networks']; + +-- The layer keeps its other keys, when it was set and who set it; a layer that held only placement stays, +-- empty, so a layer that existed still exists. +update settings + set values = values - 'places' - 'accesses' - 'networks' + where values ?| array['places', 'accesses', 'networks']; diff --git a/internal/inventory/mirrors.go b/internal/inventory/mirrors.go index c33db1c0..a7ee7535 100644 --- a/internal/inventory/mirrors.go +++ b/internal/inventory/mirrors.go @@ -26,26 +26,42 @@ type AssignmentMirrored struct { } // DeclaredConfigMirrored is one registered module's config declarations as the declared-config mirror holds -// them: the module, and each key its manifest's `settings` declares with that declaration's listed fields. -// An empty object when the module declares none: registered, declaring nothing. +// them: the module, and each key its manifest declares — under `config`, or under its old name `settings` for a +// module not yet moved (ADR 0348 §1) — with that declaration's listed fields. An empty object when the module +// declares none: registered, declaring nothing. type DeclaredConfigMirrored struct { Module string `json:"module"` Settings map[string]DeclaredMirrored `json:"settings"` } // DeclaredMirrored is one key's declaration as the mirror holds it: **its listed fields and no other** -// (ADR 0289 §12). Today a declaration is kind, default and why (catalogue.SettingDeclaration); a field the -// manifest gains reaches the bus only once it is added here, deliberately, and a test fails until it is -// listed or refused (ADR 0348 clause 4's fields arrive with package K1). A default is never a secret: the -// catalogue refuses a default for a key whose name says secret. +// (ADR 0289 §12), and every field a declaration has (catalogue.SettingDeclaration), under the same names. The +// mirror holds what the module's manifest declares about its config values, and mesh-config validates and +// resolves from it (ADR 0356 §4, ADR 0348 §4, §9), so it carries the whole declaration: kind, default and why +// of the settings form, and type, description, validation, trusted, predefined, required and runtime of the +// config form. None is a secret: a secret lives in the vault, never in config, and the catalogue refuses a +// default for a key whose name says secret. A field the manifest gains reaches the bus only once it is added +// here, deliberately: a test fails until it is listed. The config form's fields are left out when unset, so a +// module not yet moved is mirrored as it was before they were carried. type DeclaredMirrored struct { - Kind string `json:"kind"` - Default json.RawMessage `json:"default"` - Why string `json:"why"` + Kind string `json:"kind"` + Default json.RawMessage `json:"default"` + Why string `json:"why"` + Type string `json:"type,omitempty"` + Description string `json:"description,omitempty"` + Enum json.RawMessage `json:"enum,omitempty"` + Minimum *float64 `json:"minimum,omitempty"` + Maximum *float64 `json:"maximum,omitempty"` + Pattern string `json:"pattern,omitempty"` + Trusted bool `json:"trusted,omitempty"` + Predefined string `json:"predefined,omitempty"` + Required bool `json:"required,omitempty"` + Runtime bool `json:"runtime,omitempty"` } // DeclaredMirroredFields are the fields of a declaration the mirror carries, by their manifest names. -var DeclaredMirroredFields = []string{"kind", "default", "why"} +var DeclaredMirroredFields = []string{"kind", "default", "why", "type", "description", "enum", "minimum", "maximum", + "pattern", "trusted", "predefined", "required", "runtime"} // AssignmentsMirror is every assignment's mirrored value, keyed `.`. func (i *Inventory) AssignmentsMirror(ctx context.Context) (map[string][]byte, error) { @@ -74,7 +90,7 @@ func (i *Inventory) AssignmentsMirror(ctx context.Context) (map[string][]byte, e // DeclaredConfigMirror is every registered module's mirrored config declarations, keyed by the module. func (i *Inventory) DeclaredConfigMirror(ctx context.Context) (map[string][]byte, error) { rows, err := i.store.Pool().Query(ctx, - `select name, coalesce(manifest->'settings', '{}'::jsonb) from module`) + `select name, coalesce(manifest->'config', manifest->'settings', '{}'::jsonb) from module`) if err != nil { return nil, err } @@ -88,19 +104,9 @@ func (i *Inventory) DeclaredConfigMirror(ctx context.Context) (map[string][]byte if err := rows.Scan(&d.Module, &raw); err != nil { return nil, err } - var declared map[string]DeclaredMirrored - if err := json.Unmarshal(raw, &declared); err != nil { + if d.Settings, err = declaredMirrored(raw); err != nil { return nil, fmt.Errorf("the declarations of %s cannot be read: %w", d.Module, err) } - d.Settings = map[string]DeclaredMirrored{} - for key, decl := range declared { - if len(decl.Default) > 0 { - if decl.Default, err = canonical(decl.Default); err != nil { - return nil, err - } - } - d.Settings[key] = decl - } value, err := json.Marshal(d) if err != nil { return nil, err @@ -110,6 +116,31 @@ func (i *Inventory) DeclaredConfigMirror(ctx context.Context) (map[string][]byte return out, rows.Err() } +// declaredMirrored is a manifest's config declarations, keyed by the value, as the mirror holds them: each +// with its listed fields, a default and an enum encoded the one way. +func declaredMirrored(raw []byte) (map[string]DeclaredMirrored, error) { + var declared map[string]DeclaredMirrored + if err := json.Unmarshal(raw, &declared); err != nil { + return nil, err + } + out := map[string]DeclaredMirrored{} + for key, decl := range declared { + var err error + if len(decl.Default) > 0 { + if decl.Default, err = canonical(decl.Default); err != nil { + return nil, err + } + } + if len(decl.Enum) > 0 { + if decl.Enum, err = canonical(decl.Enum); err != nil { + return nil, err + } + } + out[key] = decl + } + return out, nil +} + // canonical is a JSON document encoded the one way: objects' keys sorted, numbers as written. func canonical(raw []byte) ([]byte, error) { dec := json.NewDecoder(bytes.NewReader(raw))