diff --git a/cmd/mesh-builder/main.go b/cmd/mesh-builder/main.go index e47b9ed7..2c21504b 100644 --- a/cmd/mesh-builder/main.go +++ b/cmd/mesh-builder/main.go @@ -327,6 +327,7 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri } result.Against = built.Against result.SourceFingerprint = built.Source + result.Predefined = built.Predefined result.Trunk, result.OnTrunk, result.Branches = built.Trunk, built.OnTrunk, built.Branches for _, r := range built.Read { result.Read = append(result.Read, link.ReadRepository{Repository: r.Repository, Ref: r.Ref}) diff --git a/cmd/mesh-controller/build.go b/cmd/mesh-controller/build.go index 23cdcfd3..d4a64a38 100644 --- a/cmd/mesh-controller/build.go +++ b/cmd/mesh-controller/build.go @@ -696,6 +696,20 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu } } } + // **The list of predefined keys arrives with a build of the catalogue** (novox/hq issue 630): the + // builder reads predefined.json at the catalogue repository's root at the commit built, and it is kept + // before the module is registered, so a preference falling back to a key added in the same commit is + // judged against the list that adds it. Only from the catalogue: another repository names no list. + if len(result.Predefined) > 0 && isCatalogueRepository(result.Repository) { + list, err := catalogue.ParsePredefined(result.Predefined) + if err != nil { + return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On, + manifest.Module, short(result.Commit), err) + } + if err := inv.SetPredefined(ctx, list, result.Repository, result.Commit); err != nil { + return manifest, kept, err + } + } if err := inv.RegisterModule(ctx, manifest, recorded); err != nil { if errors.Is(err, inventory.ErrSuperseded) { return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", @@ -1025,3 +1039,16 @@ func buildLog(ctx context.Context, id string) error { } return nil } + +// catalogueRepository is the repository the catalogue is: the one whose predefined.json is the list of +// predefined keys (novox/hq issue 630). +const catalogueRepository = "mesh-catalog" + +// isCatalogueRepository is whether a repository, as a path on the forge or a URL, is the catalogue. +func isCatalogueRepository(repository string) bool { + trimmed := strings.TrimSuffix(strings.TrimSuffix(strings.TrimSpace(repository), "/"), ".git") + if i := strings.LastIndex(trimmed, "/"); i >= 0 { + trimmed = trimmed[i+1:] + } + return trimmed == catalogueRepository +} diff --git a/cmd/mesh-controller/check.go b/cmd/mesh-controller/check.go index 16e16ba6..c3f772ee 100644 --- a/cmd/mesh-controller/check.go +++ b/cmd/mesh-controller/check.go @@ -47,6 +47,13 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error { shelf := catalogue.Shelf{} faulted := map[string]bool{} failed := 0 + // The list of predefined keys a preference may fall back to (novox/hq issue 630): the predefined.json + // at the root of the repository the manifests are in, found above the first of them. + list, listAt, listErr := predefinedAbove(paths[0]) + if listErr != nil { + fmt.Fprintf(out, "%s: %v\n", listAt, listErr) + failed++ + } for _, path := range paths { raw, err := os.ReadFile(path) if err != nil { @@ -92,6 +99,20 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error { failed += len(named) faulted[m.Module] = true } + // A preference falls back only to a key on the list, with its type (novox/hq issue 630). + if named := catalogue.PredefinedNamed(m); len(named) > 0 { + if listAt == "" { + fmt.Fprintf(out, "note: %s: its preferences fall back to %s, and no %s was found above it; "+ + "registration judges them against the list the mesh holds\n", path, strings.Join(named, ", "), + catalogue.PredefinedFile) + } else if wrong := catalogue.PredefinedProblems(m, list); len(wrong) > 0 { + for _, p := range wrong { + fmt.Fprintf(out, "%s: %s\n", path, p) + } + failed += len(wrong) + faulted[m.Module] = true + } + } shelf[m.Module] = m } @@ -285,3 +306,27 @@ func manifestsUnder(dir string) ([]string, error) { sort.Strings(found) return found, err } + +// predefinedAbove reads the predefined.json found in the directory of a manifest or one above it, up to +// the repository's root (the directory holding .git). The path it was found at is "" when there is none. +func predefinedAbove(manifest string) (catalogue.Predefined, string, error) { + dir, err := filepath.Abs(filepath.Dir(manifest)) + if err != nil { + return nil, "", nil + } + for { + at := filepath.Join(dir, catalogue.PredefinedFile) + if raw, err := os.ReadFile(at); err == nil { + list, err := catalogue.ParsePredefined(raw) + return list, at, err + } + if _, err := os.Stat(filepath.Join(dir, ".git")); err == nil { + return nil, "", nil + } + up := filepath.Dir(dir) + if up == dir { + return nil, "", nil + } + dir = up + } +} diff --git a/cmd/mesh-controller/config.go b/cmd/mesh-controller/config.go new file mode 100644 index 00000000..cd09a289 --- /dev/null +++ b/cmd/mesh-controller/config.go @@ -0,0 +1,383 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "sort" + "strings" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +// configCommand reads and sets config values (novox/hq issue 630). A value resolves from four sources, the +// last set winning: the module's default, mesh config, node config, and the module's own value on a node +// (its module-assignment config). Mesh and node config hold values only for the predefined keys, the +// catalogue's list; a module's own value is set on one node, never for the whole mesh. +// +// config [list] [--node ] [--module ] +// config set [--node [--module ]] --why +// config clear [--node [--module ]] --why +// config history [--node ] +// +// A set or clear is recorded in the hand-act log with the caller and why, kept in the history, and takes +// effect at the next push of the machines it reaches, which it names — as `settings set` does. +func configCommand(ctx context.Context, args []string) error { + sub := "list" + if len(args) > 0 && !strings.HasPrefix(args[0], "-") { + sub, args = args[0], args[1:] + } + set := flag.NewFlagSet("config", flag.ContinueOnError) + node := set.String("node", "", "one node: its node config, or with --module the module's own value there") + module := set.String("module", "", "a module: its own value on --node (module-assignment config)") + acts := addHandActFlags(set) + positionals, err := parseAround(set, args) + if err != nil { + return err + } + if *module != "" && *node == "" && sub != "list" { + return fmt.Errorf("config: %s names no node, and there is no module layer for the whole mesh — an "+ + "assignment is one module on one node (novox/hq issue 630). Give node, or set mesh config for a "+ + "predefined key without module. Nothing was changed", *module) + } + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + inv := open.inventory + + switch sub { + case "list": + if len(positionals) > 0 { + return errors.New("config list [--node ] [--module ]") + } + return configList(ctx, inv, *node, *module) + case "history": + if len(positionals) != 1 || *module != "" { + return errors.New("config history [--node ] — a module's own values are in `settings show --node --history`") + } + past, err := inv.ConfigHistory(ctx, positionals[0], *node) + if err != nil { + return err + } + where := configWhere(*node, "") + if len(past) == 0 { + fmt.Printf("%s, %s: no value has been replaced\n", positionals[0], where) + return nil + } + for _, p := range past { + v, _ := json.Marshal(p.Value) + fmt.Printf("%s, %s: %s until %s (%s", positionals[0], where, v, + p.ReplacedAt.Local().Format("2006-01-02 15:04:05"), p.ReplacedBy) + if p.ReplacedWhy != "" { + fmt.Printf(": %s", p.ReplacedWhy) + } + fmt.Printf("); %s, because %s\n", orUnknown(p.SetBy), orUnknown(p.Why)) + } + return nil + case "set", "clear": + default: + return fmt.Errorf("config has no %q; it has list, set, clear and history", sub) + } + + // set and clear. + if err := acts.require("config " + sub); err != nil { + return err + } + why := strings.TrimSpace(*acts.why) + var key string + var value any + switch { + case sub == "set" && len(positionals) == 2: + key = positionals[0] + if err := json.Unmarshal([]byte(positionals[1]), &value); err != nil { + // A bare word is a string: `config set theme dark` means "dark". + value = positionals[1] + } + if value == nil { + return errors.New("config set takes a value; to remove one, config clear") + } + case sub == "clear" && len(positionals) == 1: + key = positionals[0] + default: + return errors.New("config set [--node [--module ]] --why , or " + + "config clear [--node [--module ]] --why ") + } + if *node != "" { + if _, err := inv.NodeByName(ctx, *node); err != nil { + return err + } + } + where := configWhere(*node, *module) + + // The module's own value on one node: a preference it declares, of its type, judged as `settings set` + // judges a layer, its trusted keys refused through a verb exactly as there (novox/hq issue 339, 340). + if *module != "" { + shelf, err := inv.Catalogue(ctx) + if err != nil { + return err + } + m, known := shelf[*module] + if !known { + return fmt.Errorf("%w: %s", inventory.ErrNoSuchModule, *module) + } + p, declared := catalogue.PreferencesOf(m)[key] + if !declared { + return fmt.Errorf("config: %s declares no preference %q; config sets a module's preferences (%s) — "+ + "any other setting is the settings verb's. Nothing was changed", *module, key, + preferencesOrNone(sortedPreferenceKeys(m))) + } + if value != nil { + if err := catalogue.ConfigValueProblem(fmt.Sprintf("%s's preference %q", *module, key), p.Type, value); err != nil { + return err + } + } + before, _, err := inv.Layer(ctx, *node, *module) + if err != nil { + return err + } + after := map[string]any{} + for k, v := range before { + after[k] = v + } + if value == nil { + delete(after, key) + } else { + after[key] = value + } + if err := refuseTerminalSettingsThroughAVerb(ctx, inv, before, after, *module, *node); err != nil { + return err + } + acts.record(ctx, "config", configActArgs(sub, positionals, *node, *module)) + was, had, err := inv.SetModuleValue(ctx, *node, *module, key, value, setByWords()+": "+why) + if err != nil { + return err + } + sayConfigChange(key, where, value, was, had) + fmt.Printf(" takes effect at the next push: run `plan %s --diff` to see it, `push %s` to send it\n", *node, *node) + return nil + } + + // Mesh config or node config: a predefined key only, judged before the act is recorded. + if value != nil { + if err := inv.JudgeConfig(ctx, key, *node, value); err != nil { + return err + } + } + acts.record(ctx, "config", configActArgs(sub, positionals, *node, "")) + before, had := configValueOf(ctx, inv, key, *node) + if value == nil { + removed, err := inv.ClearConfig(ctx, key, *node, setByWords(), why) + if err != nil { + return err + } + if !removed { + fmt.Printf("%s, %s: no value was set; nothing changed\n", key, where) + return nil + } + } else if err := inv.SetConfig(ctx, key, *node, value, setByWords(), why); err != nil { + return err + } + sayConfigChange(key, where, value, before, had) + reached, err := reachedBy(ctx, inv, key, *node) + if err != nil { + return err + } + if len(reached) == 0 { + fmt.Println(" no module assigned there falls back to it yet; nothing to push") + return nil + } + fmt.Printf(" takes effect at the next push of %s: run `push %s` to send it\n", strings.Join(reached, ", "), + strings.Join(reached, " ")) + return nil +} + +// configList lists the predefined keys with their mesh and node config, then each module's preferences +// with the value on each node it is assigned to and where it comes from. +func configList(ctx context.Context, inv *inventory.Inventory, node, module string) error { + if node != "" { + if _, err := inv.NodeByName(ctx, node); err != nil { + return err + } + } + list, err := inv.Predefined(ctx) + if err != nil { + return err + } + values, err := inv.Config(ctx) + if err != nil { + return err + } + var b strings.Builder + if len(list) == 0 { + fmt.Fprintf(&b, "no predefined keys: the list arrives with a build of the catalogue carrying %s\n", + catalogue.PredefinedFile) + } else { + fmt.Fprintf(&b, "predefined keys (the catalogue's %s), with mesh config and node config:\n", catalogue.PredefinedFile) + } + for _, key := range list.Keys() { + k := list[key] + fmt.Fprintf(&b, " %s (%s): %s\n", key, k.Type, k.Why) + held := false + for _, v := range values { + if v.Key != key || (node != "" && v.Node != "" && v.Node != node) { + continue + } + held = true + shown, _ := json.Marshal(v.Value) + fmt.Fprintf(&b, " %s: %s — %s, because %s\n", configWhere(v.Node, ""), shown, orUnknown(v.SetBy), v.Why) + } + if !held { + b.WriteString(" no mesh or node config: each module's default\n") + } + } + for _, key := range inventory.ConfigKeysSorted(values) { + if _, on := list[key]; on { + continue + } + for _, v := range values { + if v.Key == key { + shown, _ := json.Marshal(v.Value) + fmt.Fprintf(&b, " %s is no longer a predefined key, and %s still holds %s for it: it reaches nothing; "+ + "config clear %s%s\n", key, configWhere(v.Node, ""), shown, key, nodeFlag(v.Node)) + } + } + } + + entries, err := inv.Catalogued(ctx) + if err != nil { + return err + } + var listed []preferencesOf + for _, e := range entries { + m := e.Manifest + if len(catalogue.PreferencesOf(m)) == 0 || (module != "" && m.Module != module) { + continue + } + if node != "" && !containsString(e.On, node) { + continue + } + p := preferencesOf{Manifest: m, On: map[string][]catalogue.SettingSource{}} + for _, n := range e.On { + if node != "" && n != node { + continue + } + p.Nodes = append(p.Nodes, n) + layers, err := inv.SettingsFor(ctx, n, m.Module) + if err != nil { + return err + } + p.On[n] = catalogue.Effective(m, layers) + } + listed = append(listed, p) + } + b.WriteString("\npreferences, and on each node the value and where it comes from (default, mesh config, " + + "node config, or the module on the node):\n") + b.WriteString(describePreferences(listed)) + fmt.Print(b.String()) + return nil +} + +// configWhere is how a config value's place is said. +func configWhere(node, module string) string { + switch { + case module != "": + return module + " on " + node + case node != "": + return catalogue.NodeConfigLayer(node) + } + return catalogue.MeshConfigLayer +} + +// configValueOf is the value mesh config (node "") or a node's config holds for a key. +func configValueOf(ctx context.Context, inv *inventory.Inventory, key, node string) (any, bool) { + values, err := inv.Config(ctx) + if err != nil { + return nil, false + } + for _, v := range values { + if v.Key == key && v.Node == node { + return v.Value, true + } + } + return nil, false +} + +// reachedBy is the nodes a value of mesh config (node "") or a node's config for a key reaches: those +// running a module whose preference falls back to it. +func reachedBy(ctx context.Context, inv *inventory.Inventory, key, node string) ([]string, error) { + entries, err := inv.Catalogued(ctx) + if err != nil { + return nil, err + } + nodes := map[string]bool{} + for _, e := range entries { + if !containsString(catalogue.PredefinedNamed(e.Manifest), key) { + continue + } + for _, n := range e.On { + if node == "" || n == node { + nodes[n] = true + } + } + } + out := make([]string, 0, len(nodes)) + for n := range nodes { + out = append(out, n) + } + sort.Strings(out) + return out, nil +} + +func sayConfigChange(key, where string, value, before any, had bool) { + was := "nothing" + if had { + w, _ := json.Marshal(before) + was = string(w) + } + if value == nil { + fmt.Printf("%s, %s: cleared (was %s); the next source down applies\n", key, where, was) + return + } + now, _ := json.Marshal(value) + fmt.Printf("%s, %s: %s (was %s); the value it replaced is kept\n", key, where, now, was) +} + +func configActArgs(sub string, positionals []string, node, module string) []string { + args := append([]string{sub}, positionals...) + if node != "" { + args = append(args, "--node", node) + } + if module != "" { + args = append(args, "--module", module) + } + return args +} + +func sortedPreferenceKeys(m catalogue.Manifest) []string { + declared := catalogue.PreferencesOf(m) + keys := make([]string, 0, len(declared)) + for k := range declared { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +func orUnknown(s string) string { + if strings.TrimSpace(s) == "" { + return "not kept" + } + return s +} + +func preferencesOrNone(keys []string) string { + if len(keys) == 0 { + return "it declares none" + } + return strings.Join(keys, ", ") +} diff --git a/cmd/mesh-controller/config_test.go b/cmd/mesh-controller/config_test.go new file mode 100644 index 00000000..4b79cc2c --- /dev/null +++ b/cmd/mesh-controller/config_test.go @@ -0,0 +1,187 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// The config verb's line: a listing without key, a set or clear with why, and a module value refused +// without a node (novox/hq issue 630). +func TestTheConfigVerbsLine(t *testing.T) { + for _, c := range []struct { + args map[string]any + want string + }{ + {map[string]any{}, "config list"}, + {map[string]any{"node": "g14", "module": "dunst"}, "config list --node g14 --module dunst"}, + {map[string]any{"key": "font-size", "value": "12", "why": "bigger"}, "config set font-size 12 --why bigger"}, + {map[string]any{"key": "font-size", "value": "13", "node": "g14", "why": "dense"}, + "config set font-size 13 --node g14 --why dense"}, + {map[string]any{"key": "font-size", "value": "14", "node": "g14", "module": "dunst", "why": "x"}, + "config set font-size 14 --node g14 --module dunst --why x"}, + {map[string]any{"key": "font-size", "clear": "true", "node": "g14", "why": "back"}, + "config clear font-size --node g14 --why back"}, + {map[string]any{"key": "font-size", "history": "true"}, "config history font-size"}, + } { + argv, err := argvFor("config", c.args) + if err != nil || strings.Join(argv, " ") != c.want { + t.Errorf("%v: %v %v, want %s", c.args, argv, err, c.want) + } + } + for want, args := range map[string]map[string]any{ + "no module layer for the whole mesh": {"key": "font-size", "value": "12", "module": "dunst", "why": "x"}, + "why": {"key": "font-size", "value": "12"}, + "names its key": {"value": "12"}, + "not both": {"key": "font-size", "value": "12", "clear": "true", "why": "x"}, + } { + if _, err := argvFor("config", args); err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("%v: %v, want %q", args, err, want) + } + } +} + +// End to end through the verb, as the operator's acceptance reads it: cleared → the default; mesh font-size, +// then the node's, then dunst's own on the node — each in the node's dunstrc as composed (novox/hq issue 630). +func TestConfigReachesTheComposedFileFromEachSource(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + if err := open.inventory.SetPredefined(ctx, catalogue.Predefined{ + "font-size": {Type: catalogue.TypeNumber, Why: "the size text is drawn at"}}, "", ""); err != nil { + t.Fatal(err) + } + register(t, open, catalogue.Manifest{Module: "dunst", Version: "1", + Preferences: map[string]catalogue.Preference{ + "font-size": {Type: catalogue.TypeNumber, Default: float64(10), Predefined: "font-size", Why: "100 DPI"}, + }, + Resources: []map[string]any{{"id": "dunstrc", "type": "file", "path": "/etc/dunst/dunstrc", "mode": "0644", "trusted": false, + "content": "font = Inter ${setting:font-size}\n"}}}) + for _, n := range []string{"laptop", "anchor"} { + if _, err := assign(ctx, open, n, "dunst"); err != nil { + t.Fatal(err) + } + } + dunstrc := func(node string) string { + t.Helper() + gens, err := generators(ctx, open) + if err != nil { + t.Fatal(err) + } + declared, problems, err := composedAndValidated(ctx, open, node, gens, Allocating) + if err != nil { + t.Fatalf("%s does not compose: %v %v", node, err, problems) + } + for _, r := range declared.Resources { + if r["path"] == "/etc/dunst/dunstrc" { + return fmt.Sprint(r["content"]) + } + } + t.Fatalf("%s has no dunstrc: %v", node, problems) + return "" + } + want := func(node, size string) { + t.Helper() + if got := dunstrc(node); got != "font = Inter "+size+"\n" { + t.Fatalf("%s's dunstrc: %q, want size %s", node, got, size) + } + } + verb := func(args map[string]any) error { + t.Helper() + return throughVerb(t, "config", args) + } + want("laptop", "10") + + if err := verb(map[string]any{"key": "font-size", "value": "11", "why": "a larger default"}); err != nil { + t.Fatal(err) + } + want("laptop", "11") + want("anchor", "11") + if err := verb(map[string]any{"key": "font-size", "value": "13", "node": "laptop", "why": "a dense screen"}); err != nil { + t.Fatal(err) + } + want("laptop", "13") + want("anchor", "11") + if err := verb(map[string]any{"key": "font-size", "value": "15", "node": "laptop", "module": "dunst", + "why": "dunst alone larger"}); err != nil { + t.Fatal(err) + } + want("laptop", "15") + + listing := stdoutOf(t, func() error { return verb(map[string]any{}) }) + for _, said := range []string{"font-size (number): the size text is drawn at", "mesh config: 11", + "node config (laptop): 13", "laptop: 15 (dunst on laptop)", "anchor: 11 (mesh config)", + "falls back to the predefined key font-size"} { + if !strings.Contains(listing, said) { + t.Errorf("the listing does not say %q:\n%s", said, listing) + } + } + + // Cleared, each falls back to the next source down. + for _, step := range []struct { + args map[string]any + size string + }{ + {map[string]any{"key": "font-size", "clear": "true", "node": "laptop", "module": "dunst", "why": "x"}, "13"}, + {map[string]any{"key": "font-size", "clear": "true", "node": "laptop", "why": "x"}, "11"}, + {map[string]any{"key": "font-size", "clear": "true", "why": "x"}, "10"}, + } { + if err := verb(step.args); err != nil { + t.Fatal(err) + } + want("laptop", step.size) + } + + // Refused: a key not on the list for mesh config, a module value without a node, a type not the key's. + if err := verb(map[string]any{"key": "colour", "value": "blue", "why": "x"}); err == nil || + !strings.Contains(err.Error(), "predefined") { + t.Fatalf("mesh config for a key not on the list: %v", err) + } + if err := runLine(t, []string{"config", "set", "font-size", "12", "--module", "dunst", "--why", "x"}); err == nil || + !strings.Contains(err.Error(), "no module layer for the whole mesh") { + t.Fatalf("a module value without a node: %v", err) + } + if err := verb(map[string]any{"key": "font-size", "value": `"large"`, "node": "laptop", "module": "dunst", + "why": "x"}); err == nil || !strings.Contains(err.Error(), "is a number") { + t.Fatalf("a string for a number: %v", err) + } + if err := verb(map[string]any{"key": "nothing", "value": "1", "node": "laptop", "module": "dunst", + "why": "x"}); err == nil || !strings.Contains(err.Error(), "declares no preference") { + t.Fatalf("a module value for a key it does not declare: %v", err) + } +} + +// The module check reads the catalogue's predefined.json at the repository's root and refuses a preference +// falling back to a key not on it (novox/hq issue 630). +func TestTheModuleCheckJudgesPreferencesAgainstTheCataloguesList(t *testing.T) { + root := t.TempDir() + for _, dir := range []string{".git", "modules/dunst"} { + if err := os.MkdirAll(filepath.Join(root, dir), 0o755); err != nil { + t.Fatal(err) + } + } + manifest := filepath.Join(root, "modules/dunst/module.json") + write := func(path, body string) { + t.Helper() + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + write(manifest, `{"module": "dunst", "version": "1", + "preferences": {"font-size": {"type": "number", "default": 10, "predefined": "font-size", "why": "100 DPI"}}, + "resources": [{"id": "rc", "type": "file", "path": "/etc/rc", "mode": "0644", "trusted": false, + "content": "font ${setting:font-size}"}]}`) + write(filepath.Join(root, "predefined.json"), `{"theme": {"type": "string", "why": "light or dark"}}`) + var out strings.Builder + if err := moduleCheck([]string{manifest}, &out); err == nil || !strings.Contains(out.String(), "not on the catalogue's list") { + t.Fatalf("a key not on the list passed: %v\n%s", err, out.String()) + } + write(filepath.Join(root, "predefined.json"), `{"font-size": {"type": "number", "why": "the size text is drawn at"}}`) + out.Reset() + if err := moduleCheck([]string{manifest}, &out); err != nil { + t.Fatalf("a key on the list was refused: %v\n%s", err, out.String()) + } +} diff --git a/cmd/mesh-controller/facts.go b/cmd/mesh-controller/facts.go index 50b891cf..e0b92c53 100644 --- a/cmd/mesh-controller/facts.go +++ b/cmd/mesh-controller/facts.go @@ -263,6 +263,37 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot f.Controller.Commit = c.Commit } + // The list of predefined keys, and mesh and node config (novox/hq issue 630). + predefined, err := inv.Predefined(ctx) + if err != nil { + return snapshot.Facts{}, err + } + for key, k := range predefined { + if f.Predefined == nil { + f.Predefined = map[string]snapshot.PredefinedKey{} + } + f.Predefined[key] = snapshot.PredefinedKey{Type: k.Type, Why: k.Why} + } + config, err := inv.Config(ctx) + if err != nil { + return snapshot.Facts{}, err + } + nodeConfig := map[string]map[string]any{} + meshConfig := map[string]any{} + for _, c := range config { + if c.Node == "" { + meshConfig[c.Key] = c.Value + continue + } + if nodeConfig[c.Node] == nil { + nodeConfig[c.Node] = map[string]any{} + } + nodeConfig[c.Node][c.Key] = c.Value + } + if len(meshConfig) > 0 { + f.Config = scrub.Values(meshConfig) + } + gens, gensErr := generators(ctx, open) hostShelf := shelf[hostModule] meshWide := map[string]bool{} @@ -280,6 +311,9 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot if n.HostVersion != "" { engines[n.HostVersion] = true } + if values := nodeConfig[n.Name]; len(values) > 0 { + m.Config = scrub.Values(values) + } m.System = systemOf(hostShelf, n.HostVersion) m.Libc = libcOf(m.System) reported, err := inv.DescribedOf(ctx, n.Name) @@ -345,6 +379,10 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot return snapshot.Facts{}, err } for _, layer := range layers { + if layer.Config { + // Mesh and node config are said once, beside the machine and the mesh (novox/hq issue 630). + continue + } if layer.From == catalogue.MeshWideLayer { if !meshWide[module] { meshWide[module] = true diff --git a/cmd/mesh-controller/facts_test.go b/cmd/mesh-controller/facts_test.go index d5b3ac63..5e3eb51e 100644 --- a/cmd/mesh-controller/facts_test.go +++ b/cmd/mesh-controller/facts_test.go @@ -31,7 +31,7 @@ func aMeshWithSecrets(t *testing.T) (*stores, []string) { } } secrets := []string{"Hunter2-Is-Not-A-Password-9f8e7d", "0123456789abcdefABCDEF0123456789zz"} - if err := open.inventory.SetSettings(ctx, "", "files", map[string]any{ + if err := open.inventory.KeepSettings(ctx, "", "files", map[string]any{ "admin_password": secrets[0], "upstream": "10.77.0.9", "hub": "anchor"}); err != nil { t.Fatal(err) } diff --git a/cmd/mesh-controller/handacts.go b/cmd/mesh-controller/handacts.go index 147a6dca..49decc69 100644 --- a/cmd/mesh-controller/handacts.go +++ b/cmd/mesh-controller/handacts.go @@ -89,6 +89,8 @@ var handActVerbs = []handActVerb{ {Verb: handActIssued, Decision: "a credential the provider's software issued, kept for its consumers (ADR 0305)"}, {Verb: "hand-act drill", Decision: "a drill is a person's deliberate test of the mesh, never a repair"}, // A person's decisions by design. + // A config value is what a person chooses the mesh's modules use (novox/hq issue 630), never a repair. + {Verb: "config", Decision: "a config value is a person's choice of what the modules use, not a repair (issue 630)"}, {Verb: "retire approve", Decision: "nothing is retired past its bound without a person (ADR 0230)"}, {Verb: "retire reject", Decision: "keeping a consumer active is a person's word (ADR 0230)"}, {Verb: "cleanup delete", Decision: "nothing retired is deleted without a person (ADR 0230)"}, diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index 7269d761..4ddc6143 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -171,6 +171,8 @@ func run() error { return adoptCommand(ctx, args[1:]) case "settings": return settingsCommand(ctx, args[1:]) + case "config": + return configCommand(ctx, args[1:]) case "secret": return secretCommand(ctx, args[1:]) case "operator": diff --git a/cmd/mesh-controller/merge_gate.go b/cmd/mesh-controller/merge_gate.go index ddcfd609..0e614fd2 100644 --- a/cmd/mesh-controller/merge_gate.go +++ b/cmd/mesh-controller/merge_gate.go @@ -296,6 +296,24 @@ func judgeChange(ctx context.Context, in mergeCheckInput) (mergeVerdict, error) } v.Failures = append(v.Failures, failures...) } + // The list of predefined keys a head's catalogue tree carries, which the change composes with + // (novox/hq issue 630); the mesh's own list otherwise. + var changedList catalogue.Predefined + for _, h := range heads { + if h.Tree == "" { + continue + } + raw, err := os.ReadFile(filepath.Join(h.Tree, catalogue.PredefinedFile)) + if err != nil { + continue + } + list, err := catalogue.ParsePredefined(raw) + if err != nil { + v.Failures = append(v.Failures, err.Error()) + continue + } + changedList = list + } if len(in.group) > 0 { say("a delivery group: %d head(s) composed together", len(heads)) } @@ -319,7 +337,7 @@ func judgeChange(ctx context.Context, in mergeCheckInput) (mergeVerdict, error) // Both compositions, each in a store of its own. say("composing every machine as the mesh is") - before, _, notes, err := composeEveryMachine(ctx, in, base, sources, nil) + before, _, notes, err := composeEveryMachine(ctx, in, base, sources, nil, nil) if err != nil { return v, fmt.Errorf("the mesh as the snapshot says it is could not be raised: %w", err) } @@ -332,7 +350,7 @@ func judgeChange(ctx context.Context, in mergeCheckInput) (mergeVerdict, error) } } sort.Strings(added) - after, tried, notes, err := composeEveryMachine(ctx, in, change, sources, added) + after, tried, notes, err := composeEveryMachine(ctx, in, change, sources, added, changedList) if err != nil { return v, fmt.Errorf("the mesh with the change could not be raised: %w", err) } @@ -748,13 +766,13 @@ func standInBuild(m catalogue.Manifest) (catalogue.Manifest, error) { // grant to compose — and answers each machine's composition. Notes are what of the snapshot could not // be raised: a refusal to register a module, keep a setting or a pin. func composeEveryMachine(ctx context.Context, in mergeCheckInput, shelf map[string]catalogue.Manifest, - sources map[string]inventory.Source, trials []string) (map[string]mergeComposed, map[string]mergeTrial, []string, error) { + sources map[string]inventory.Source, trials []string, list catalogue.Predefined) (map[string]mergeComposed, map[string]mergeTrial, []string, error) { open, drop, err := throwawayStores(ctx, in.admin) if err != nil { return nil, nil, nil, err } defer drop() - notes, err := raiseFromFacts(ctx, open, in.facts, shelf, sources) + notes, err := raiseFromFacts(ctx, open, in.facts, shelf, sources, list) if err != nil { return nil, nil, notes, err } @@ -950,9 +968,22 @@ func throwawayStores(ctx context.Context, admin string) (*stores, func(), error) // for each standing in for the one it holds, its assignments, the seats it holds, its pins and its // settings, and a stand-in for every secret a person gave. func raiseFromFacts(ctx context.Context, open *stores, f snapshot.Facts, shelf map[string]catalogue.Manifest, - sources map[string]inventory.Source) ([]string, error) { + sources map[string]inventory.Source, list catalogue.Predefined) ([]string, error) { inv := open.inventory var notes []string + // The list of predefined keys before any module, which registration judges its preferences against + // (novox/hq issue 630): the change's own when its catalogue tree carries one, the mesh's otherwise. + if list == nil && len(f.Predefined) > 0 { + list = catalogue.Predefined{} + for key, k := range f.Predefined { + list[key] = catalogue.PredefinedKey{Type: k.Type, Why: k.Why} + } + } + if len(list) > 0 { + if err := inv.SetPredefined(ctx, list, "", ""); err != nil { + notes = append(notes, fmt.Sprintf("the list of predefined keys is not kept: %s", oneLine(err.Error()))) + } + } names := make([]string, 0, len(shelf)) for name := range shelf { names = append(names, name) @@ -1045,6 +1076,20 @@ func raiseFromFacts(ctx context.Context, open *stores, f snapshot.Facts, shelf m } } } + // Mesh and node config, as the mesh holds them (novox/hq issue 630). + for _, key := range sortedAnyKeys(f.Config) { + if err := inv.SetConfig(ctx, key, "", f.Config[key], "the facts snapshot", "raised from the facts snapshot"); err != nil { + notes = append(notes, fmt.Sprintf("the mesh config of %s is not kept: %s", key, oneLine(err.Error()))) + } + } + for _, m := range f.Machines { + for _, key := range sortedAnyKeys(m.Config) { + if err := inv.SetConfig(ctx, key, m.Name, m.Config[key], "the facts snapshot", "raised from the facts snapshot"); err != nil { + notes = append(notes, fmt.Sprintf("%s's node config of %s is not kept: %s", m.Described(), key, + oneLine(err.Error()))) + } + } + } for _, s := range f.Settings { if err := inv.KeepSettings(ctx, "", s.Module, standInPaths(s.Values)); err != nil { notes = append(notes, fmt.Sprintf("the mesh's settings of %s are not kept: %s", s.Module, oneLine(err.Error()))) @@ -1312,3 +1357,13 @@ func firstOr(items, otherwise []string) string { } return "it says nothing more" } + +// sortedAnyKeys is a map's keys, sorted. +func sortedAnyKeys(in map[string]any) []string { + keys := make([]string, 0, len(in)) + for k := range in { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/cmd/mesh-controller/modules.go b/cmd/mesh-controller/modules.go index 66804655..aeac0218 100644 --- a/cmd/mesh-controller/modules.go +++ b/cmd/mesh-controller/modules.go @@ -571,18 +571,14 @@ func settingsCommand(ctx context.Context, args []string) error { return err } m, ok := known[positionals[0]] - if !ok || len(m.Settings) == 0 { + if !ok || len(catalogue.PreferencesOf(m)) == 0 { return nil } var layers []catalogue.Layer if *node != "" { - if mesh, has, err := inv.Layer(ctx, "", positionals[0]); err != nil { + // Every source on the node: mesh config, node config, then the module's own (novox/hq issue 630). + if layers, err = inv.SettingsFor(ctx, *node, positionals[0]); err != nil { return err - } else if has { - layers = append(layers, catalogue.Layer{From: catalogue.MeshWideLayer, Values: mesh}) - } - if has { - layers = append(layers, catalogue.Layer{From: *node, Values: values}) } } else if has { layers = append(layers, catalogue.Layer{From: catalogue.MeshWideLayer, Values: values}) @@ -612,7 +608,7 @@ func settingsCommand(ctx context.Context, args []string) error { var listed []preferencesOf for _, e := range entries { m := e.Manifest - if len(m.Settings) == 0 || (only != "" && m.Module != only) { + if len(catalogue.PreferencesOf(m)) == 0 || (only != "" && m.Module != only) { continue } if *node != "" && !containsString(e.On, *node) { @@ -682,7 +678,11 @@ func describeEffective(module, where string, values []catalogue.SettingSource) s fmt.Fprintf(&b, "%s on %s, every value and where it comes from:\n", module, where) for _, v := range values { value, _ := json.Marshal(v.Value) - fmt.Fprintf(&b, " %s = %s (%s", v.Key, value, v.From) + from := v.From + if v.From == where && where != "the whole mesh" { + from = valueSource(module, where, v) + } + fmt.Fprintf(&b, " %s = %s (%s", v.Key, value, from) if v.HasDefault && !v.FromDefault { d, _ := json.Marshal(v.Default) fmt.Fprintf(&b, "; the default is %s", d) @@ -723,28 +723,27 @@ func describePreferences(modules []preferencesOf) string { on = "on " + strings.Join(p.Nodes, ", ") } fmt.Fprintf(&b, "%s (%s)\n", p.Manifest.Module, on) - keys := make([]string, 0, len(p.Manifest.Settings)) - for k := range p.Manifest.Settings { + declared := catalogue.PreferencesOf(p.Manifest) + keys := make([]string, 0, len(declared)) + for k := range declared { keys = append(keys, k) } sort.Strings(keys) for _, k := range keys { - d := p.Manifest.Settings[k] + d := declared[k] def, _ := json.Marshal(d.Default) - fmt.Fprintf(&b, " %s, default %s: %s\n", k, def, d.Why) + falls := "" + if d.Predefined != "" { + falls = ", falls back to the predefined key " + d.Predefined + } + fmt.Fprintf(&b, " %s, default %s%s: %s\n", k, def, falls, d.Why) for _, n := range p.Nodes { for _, s := range p.On[n] { if s.Key != k { continue } v, _ := json.Marshal(s.Value) - from := s.From - if s.FromDefault { - from = catalogue.DefaultLayer - } else if s.From != catalogue.MeshWideLayer { - from = "the node" - } - fmt.Fprintf(&b, " %s: %s (%s)\n", n, v, from) + fmt.Fprintf(&b, " %s: %s (%s)\n", n, v, valueSource(p.Manifest.Module, n, s)) } } } @@ -752,6 +751,18 @@ func describePreferences(modules []preferencesOf) string { return b.String() } +// valueSource is where a value on a node comes from, as the four sources are said (novox/hq issue 630): +// "default", "mesh config", "node config ()", or " on " for the module's own value. +func valueSource(module, node string, s catalogue.SettingSource) string { + switch { + case s.FromDefault: + return catalogue.DefaultLayer + case s.From == catalogue.MeshConfigLayer, s.From == catalogue.NodeConfigLayer(node), s.From == catalogue.MeshWideLayer: + return s.From + } + return module + " on " + node +} + // nodeFlag is ` --node ` for a machine's layer, nothing for the whole mesh's. func nodeFlag(node string) string { if node == "" { diff --git a/cmd/mesh-controller/operator_wait.go b/cmd/mesh-controller/operator_wait.go index 48e8bae4..d006e501 100644 --- a/cmd/mesh-controller/operator_wait.go +++ b/cmd/mesh-controller/operator_wait.go @@ -88,7 +88,7 @@ func checkWait(module, machine string, w inventory.Wait, f operatorWaitFacts) er case w.Provision != "": return checkProvisionWait(module, machine, w, m, f) case w.Setting != "": - if _, declared := m.Settings[w.Setting]; !declared { + if !catalogue.DeclaresPreference(m, w.Setting) { return fmt.Errorf("says it waits for the setting %s, which %s does not declare", w.Setting, module) } return nil diff --git a/cmd/mesh-controller/probe_agent_root.go b/cmd/mesh-controller/probe_agent_root.go index 95111959..d9b0245b 100644 --- a/cmd/mesh-controller/probe_agent_root.go +++ b/cmd/mesh-controller/probe_agent_root.go @@ -211,7 +211,7 @@ func (r *rootReader) executeServed(ctx context.Context, machine string) (bool, s } holders++ var setting *any - if _, declared := e.Manifest.Settings[loginShellVerb]; declared { + if catalogue.DeclaresPreference(e.Manifest, loginShellVerb) { layers, err := r.settings(ctx, machine, e.Manifest.Module) if err != nil { served = true diff --git a/cmd/mesh-controller/proposals.go b/cmd/mesh-controller/proposals.go index 6dd4d000..5a5a1416 100644 --- a/cmd/mesh-controller/proposals.go +++ b/cmd/mesh-controller/proposals.go @@ -542,6 +542,11 @@ const ( // proposeCommand is `settings propose`, on this controller's stores and bus. func proposeCommand(ctx context.Context, module, node, valuesArg string, clear, replace bool) error { + if node == "" && !clear { + // There is no module layer for the whole mesh (novox/hq issue 630); clearing one left over still works. + return fmt.Errorf("%s: there is no module layer for the whole mesh — an assignment is one module on one "+ + "node (novox/hq issue 630). Propose it for one node with --node. Nothing was proposed", module) + } var values map[string]any if !clear { if valuesArg == "" { diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index ff43179d..3660438d 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -881,6 +881,48 @@ func (a *verbArguments) commandLine() ([]string, error) { argv = append(argv, "--adopted") } return argv, nil + case "config": + // Config values from four sources (novox/hq issue 630): mesh config and node config for a predefined + // key, and a module's own value on one node. A listing without key; a set or clear says why. + key := str("key") + if str("module") != "" && str("node") == "" && (key != "" || on("clear")) { + return nil, errors.New("config: there is no module layer for the whole mesh — an assignment is one module " + + "on one node (novox/hq issue 630); give node, or leave module out to set mesh config for a predefined key") + } + place := func(argv []string) []string { + if n := str("node"); n != "" { + argv = append(argv, "--node", n) + } + if m := str("module"); m != "" { + argv = append(argv, "--module", m) + } + return argv + } + switch { + case key == "": + if str("value") != "" || on("clear") || on("history") || str("why") != "" { + return nil, errors.New("config: a set, a clear or a history names its key") + } + return place([]string{"config", "list"}), nil + case on("history"): + if str("value") != "" || on("clear") || str("module") != "" || str("why") != "" { + return nil, errors.New("config: history reads a key's mesh or node config, and takes only node") + } + return place([]string{"config", "history", key}), nil + case on("clear") && str("value") != "": + return nil, errors.New("config: a value or clear, not both") + case on("clear"): + if err := need("why"); err != nil { + return nil, err + } + return append(place([]string{"config", "clear", key}), "--why", str("why")), nil + case str("value") != "": + if err := need("why"); err != nil { + return nil, err + } + return append(place([]string{"config", "set", key, str("value")}), "--why", str("why")), nil + } + return nil, errors.New("config: with key, give value, clear or history") case "settings": // Every module's preferences, their defaults and each machine's value (novox/hq ADR 0262): // the one interface for them, so no module builds a settings tool of its own. Asked for by @@ -1614,6 +1656,7 @@ var commandReadForms = map[string]func(rest []string) bool{ "node": func(r []string) bool { return subIn(r, "list", "show") }, "module": func(r []string) bool { return subIn(r, "list") }, "settings": func(r []string) bool { return subIn(r, "show", "preferences", "proposals") }, + "config": func(r []string) bool { return flagsOnly(r) || subIn(r, "list", "history") }, "retire": func(r []string) bool { return subIn(r, "list") }, "cleanup": func(r []string) bool { return subIn(r, "list") }, "delivery": func(r []string) bool { return subIn(r, "plan", "walks") }, diff --git a/cmd/mesh-controller/settings_effective_test.go b/cmd/mesh-controller/settings_effective_test.go index f248e2b4..bf024d3b 100644 --- a/cmd/mesh-controller/settings_effective_test.go +++ b/cmd/mesh-controller/settings_effective_test.go @@ -14,7 +14,7 @@ func TestSettingsSayWhereEachValueComesFrom(t *testing.T) { {Key: "width", Value: float64(250), From: catalogue.DefaultLayer, FromDefault: true, Default: float64(250), HasDefault: true}, }) want := "dunst on laptop, every value and where it comes from:\n" + - " font-size = 13 (laptop; the default is 10)\n" + + " font-size = 13 (dunst on laptop; the default is 10)\n" + " width = 250 (default)\n" if got != want { t.Fatalf("said\n%s\nwant\n%s", got, want) @@ -66,7 +66,7 @@ func TestPreferencesSayEachMachinesValueAndItsSource(t *testing.T) { want := "dunst (on desk, laptop)\n" + " font-size, default 10: readable at 100 DPI\n" + " desk: 10 (default)\n" + - " laptop: 16 (the node)\n" + + " laptop: 16 (dunst on laptop)\n" + " width, default 250: forty characters\n" + " desk: 300 (the mesh)\n" + " laptop: 250 (default)\n" @@ -96,7 +96,7 @@ func TestPreferencesListedFromTheStores(t *testing.T) { t.Fatal(err) } all := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"preferences"}) }) - if !strings.Contains(all, "notes (on laptop)") || !strings.Contains(all, "laptop: 16 (the node)") || + if !strings.Contains(all, "notes (on laptop)") || !strings.Contains(all, "laptop: 16 (notes on laptop)") || !strings.Contains(all, "font-size, default 10: readable at 100 DPI") { t.Fatalf("the listing:\n%s", all) } diff --git a/cmd/mesh-controller/terminal_settings_test.go b/cmd/mesh-controller/terminal_settings_test.go index 783ca251..c354ac6e 100644 --- a/cmd/mesh-controller/terminal_settings_test.go +++ b/cmd/mesh-controller/terminal_settings_test.go @@ -263,8 +263,18 @@ func TestTheAgentsManagedSettingsAreRefusedThroughAVerb(t *testing.T) { if err := atTheTerminal(t, "settings", "set", "claude-code", allow, "--node", "laptop"); err != nil { t.Fatalf("the managed settings at the terminal: %v", err) } - if err := atTheTerminal(t, "settings", "set", "claude-code", server); err != nil { - t.Fatalf("a tool server for the mesh at the terminal: %v", err) + // There is no module layer for the whole mesh (novox/hq issue 630), at the terminal either; one the mesh + // still holds from before is kept as it is, and is the terminal's as before. + if err := atTheTerminal(t, "settings", "set", "claude-code", server); err == nil || + !strings.Contains(err.Error(), "no module layer for the whole mesh") { + t.Fatalf("a tool server for the whole mesh at the terminal: %v", err) + } + var held map[string]any + if err := json.Unmarshal([]byte(server), &held); err != nil { + t.Fatal(err) + } + if err := open.inventory.KeepSettings(ctx, "", "claude-code", held); err != nil { + t.Fatalf("a whole-mesh layer held from before: %v", err) } kept := layer("laptop") // Through a verb they are neither changed, dropped nor cleared. @@ -359,15 +369,15 @@ func TestOnlyATrustedFilesNamedKeyIsRefusedThroughAVerb(t *testing.T) { if _, err := assign(ctx, open, "anchor", "mailer"); err != nil { t.Fatal(err) } - if err := throughVerb(t, "settings", map[string]any{"module": "mailer", "values": `{"tags":{"todo":{"name":"To do"}}}`}); err != nil { + if err := throughVerb(t, "settings", map[string]any{"module": "mailer", "node": "anchor", "values": `{"tags":{"todo":{"name":"To do"}}}`}); err != nil { t.Fatalf("a key of a file nothing trusts is the verb's: %v", err) } - err := throughVerb(t, "settings", map[string]any{"module": "mailer", "replace": "true", + err := throughVerb(t, "settings", map[string]any{"module": "mailer", "node": "anchor", "replace": "true", "values": `{"tags":{"todo":{"name":"To do"}},"accounts":{"mail":{"address":"me@mail.example"}}}`}) if err == nil || !strings.Contains(err.Error(), "controller's terminal") || !strings.Contains(err.Error(), "accounts") { t.Fatalf("a key of a trusted file is the terminal's: %v", err) } - values, _, err := open.inventory.Layer(ctx, "", "mailer") + values, _, err := open.inventory.Layer(ctx, "anchor", "mailer") if err != nil { t.Fatal(err) } diff --git a/internal/builder/builder.go b/internal/builder/builder.go index 586b375f..61df585a 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -98,6 +98,11 @@ type Result struct { // recipe and the package it compiles; for any other, nothing is said and the whole of what the build // sees stays its source, as before. Sources []BuildSource + + // Predefined is the repository's list of predefined keys (catalogue.PredefinedFile at its root) as + // read at the commit built, when it has one (novox/hq issue 630): what the controller keeps as the + // list mesh and node config hold values for. Empty for a repository without one. + Predefined json.RawMessage } // BuildSource is the build source a build read in one repository (novox/hq ADR 0267). @@ -233,6 +238,25 @@ func build(ctx context.Context, run Runner, publish Publisher, say("manifest", "INVALID: %v", err) return Result{}, err } + // The repository's list of predefined keys, at its root (novox/hq issue 630): read and judged here, + // where a changed list is first read, and handed on for the controller to keep. A preference that + // falls back to a key the list does not hold, or with another type, is refused here as at registration. + var predefined json.RawMessage + if raw, err := os.ReadFile(filepath.Join(tree, catalogue.PredefinedFile)); err == nil { + list, err := catalogue.ParsePredefined(raw) + if err != nil { + say("manifest", "INVALID: %v", err) + return Result{}, err + } + if problems := catalogue.PredefinedProblems(manifest, list); len(problems) > 0 { + err := fmt.Errorf("%s", strings.Join(problems, "; ")) + say("manifest", "INVALID: %v", err) + return Result{}, err + } + predefined, _ = json.Marshal(list) + } else if !errors.Is(err, os.ErrNotExist) { + return Result{}, fmt.Errorf("%s's %s cannot be read: %w", repository, catalogue.PredefinedFile, err) + } say("manifest", "%s v%s — %d artifact(s)", manifest.Module, manifest.Version, artifactCount(manifest)) // What it is made from, for its source fingerprint: the module's own tree first. @@ -365,7 +389,7 @@ func build(ctx context.Context, run Runner, publish Publisher, } return Result{Manifest: resolved, Commit: commit, Built: built, Against: against(within, manifest, stoodOn), Read: readBy(manifest), Source: fingerprint, - Trunk: trunk, OnTrunk: onTrunk, Branches: branches, Sources: sources}, nil + Trunk: trunk, OnTrunk: onTrunk, Branches: branches, Sources: sources, Predefined: predefined}, nil } // branchesHolding is every branch of a fresh clone's origin the commit is on, without `origin/`. diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index 866daf78..6e110155 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -530,6 +530,12 @@ type Manifest struct { // stays refused by name until a layer sets it. The mesh's layer, then the node's, override it. Settings map[string]SettingDeclaration `json:"settings,omitempty"` + // Preferences are the values this module reads that a machine may tune (novox/hq issue 630): each + // key's type, default and why, and the predefined key it falls back to, if any. The value resolves + // from the default, then mesh config and node config for the predefined key, then this module's own + // value on the node. `settings` entries of kind preference are read as preferences while they move. + Preferences map[string]Preference `json:"preferences,omitempty"` + // unknown is the first key this manifest has that this controller does not know, at any depth, when // it was read from the store (novox/hq ADR 0262): a manifest a newer controller registered. // ParseManifest refuses it; the store's catalogue still loads, and the module is left out of every diff --git a/internal/catalogue/preferences.go b/internal/catalogue/preferences.go new file mode 100644 index 00000000..8acd6ad2 --- /dev/null +++ b/internal/catalogue/preferences.go @@ -0,0 +1,270 @@ +package catalogue + +import ( + "bytes" + "encoding/json" + "fmt" + "regexp" + "sort" + "strings" +) + +// A config value resolves from four sources, the last set winning (novox/hq issue 630): +// +// 1. the default in the module's manifest; +// 2. mesh config: a mesh-wide value for a predefined key, such as `font-size`; +// 3. node config: a value for a predefined key for one node; +// 4. module-assignment config: that module's own value on that node, the most specific. +// +// An assignment is one module on one node: there is no module layer for the whole mesh. Mesh and node +// config hold values only for the keys on a fixed, reviewable list — the catalogue's `predefined.json` — +// that any module may fall back to, by naming one as its preference's `predefined`. A preference without +// `predefined` reads only its default and its module-assignment config. A preference is never trusted. + +// Preference is one value a module reads that a machine may tune, as the manifest declares it under +// `preferences`. +type Preference struct { + // Type is "number", "string" or "boolean". + Type string `json:"type"` + // Default is the value when no source sets it, of Type. + Default any `json:"default"` + // Predefined names the key on the catalogue's list this preference falls back to: mesh config, + // then node config, for that key. Empty reads only the default and the module-assignment config. + Predefined string `json:"predefined,omitempty"` + // Why this default: one sentence for whoever wonders whether to change it. + Why string `json:"why"` +} + +// The types a preference and a predefined key may have. +const ( + TypeNumber = "number" + TypeString = "string" + TypeBoolean = "boolean" +) + +// PredefinedFile is the catalogue repository's root file holding the predefined keys. +const PredefinedFile = "predefined.json" + +// PredefinedKey is one key mesh and node config may hold a value for. +type PredefinedKey struct { + Type string `json:"type"` + Why string `json:"why"` +} + +// Predefined is the list of predefined keys, by key. +type Predefined map[string]PredefinedKey + +// Keys is the list's keys, sorted. +func (p Predefined) Keys() []string { + keys := make([]string, 0, len(p)) + for k := range p { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +// predefinedName is what a predefined key looks like: the shape of a setting key. +var predefinedName = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]*$`) + +// ParsePredefined reads `predefined.json` strictly, and refuses a key whose name, type or why is wrong. +func ParsePredefined(raw []byte) (Predefined, error) { + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + var list Predefined + if err := decoder.Decode(&list); err != nil { + return nil, fmt.Errorf("%s is not a list of predefined keys ({\"\": {\"type\": …, \"why\": …}}): %w", + PredefinedFile, err) + } + var problems []string + for _, key := range list.Keys() { + k := list[key] + if !predefinedName.MatchString(key) { + problems = append(problems, fmt.Sprintf("%q is not a usable key: lower-case letters, digits, dots, dashes and underscores", key)) + } + if meshWords[key] { + problems = append(problems, fmt.Sprintf("%q is a key the mesh reads itself", key)) + } + if !knownType(k.Type) { + problems = append(problems, fmt.Sprintf("%q has the type %q; a type is %q, %q or %q", key, k.Type, + TypeNumber, TypeString, TypeBoolean)) + } + if strings.TrimSpace(k.Why) == "" { + problems = append(problems, fmt.Sprintf("%q has no why: say in one sentence what the key is for", key)) + } + } + if len(problems) > 0 { + return nil, fmt.Errorf("%s: %s", PredefinedFile, strings.Join(problems, "; ")) + } + if list == nil { + list = Predefined{} + } + return list, nil +} + +func knownType(t string) bool { + return t == TypeNumber || t == TypeString || t == TypeBoolean +} + +// TypeOf is the type a JSON value has, as a preference names it, or "" for one that is none of them. +func TypeOf(v any) string { + switch v.(type) { + case float64, int, int64, json.Number: + return TypeNumber + case string: + return TypeString + case bool: + return TypeBoolean + } + return "" +} + +// PreferencesOf is every preference a module declares: those under `preferences`, and — while the +// catalogue moves to them — every `settings` entry of kind preference, its type read from its default. +func PreferencesOf(m Manifest) map[string]Preference { + if len(m.Preferences) == 0 && len(m.Settings) == 0 { + return nil + } + out := make(map[string]Preference, len(m.Preferences)+len(m.Settings)) + for key, d := range m.Settings { + out[key] = Preference{Type: TypeOf(d.Default), Default: d.Default, Why: d.Why} + } + for key, p := range m.Preferences { + out[key] = p + } + return out +} + +// DeclaresPreference is whether a module declares a preference by this key. +func DeclaresPreference(m Manifest, key string) bool { + _, has := m.Preferences[key] + if !has { + _, has = m.Settings[key] + } + return has +} + +// PredefinedProblems is every way a module's preferences disagree with the list of predefined keys: a +// `predefined` not on it, or a type other than the list's. +func PredefinedProblems(m Manifest, list Predefined) []string { + var problems []string + keys := make([]string, 0, len(m.Preferences)) + for k := range m.Preferences { + keys = append(keys, k) + } + sort.Strings(keys) + for _, key := range keys { + p := m.Preferences[key] + if p.Predefined == "" { + continue + } + k, on := list[p.Predefined] + if !on { + listed := "the list holds none" + if len(list) > 0 { + listed = "the list holds " + strings.Join(list.Keys(), ", ") + } + problems = append(problems, fmt.Sprintf("%s's preference %q falls back to the predefined key %q, which "+ + "is not on the catalogue's list of predefined keys (%s; %s)", m.Module, key, p.Predefined, PredefinedFile, + listed)) + continue + } + if k.Type != p.Type { + problems = append(problems, fmt.Sprintf("%s's preference %q is a %s, and the predefined key %q it falls "+ + "back to is a %s", m.Module, key, p.Type, p.Predefined, k.Type)) + } + } + return problems +} + +// Where a value from mesh or node config is said to come from. +const MeshConfigLayer = "mesh config" + +// NodeConfigLayer is what the node config of one node is called where a value's source is said. +func NodeConfigLayer(node string) string { return "node config (" + node + ")" } + +// ConfigLayers are the layers mesh config and node config make for one module: each of its preferences +// that names a predefined key, given that key's value. Laid between the module's defaults and its +// module-assignment config; like the defaults they fill `${setting:}` only. +func ConfigLayers(m Manifest, mesh, node map[string]any, nodeName string) []Layer { + var out []Layer + for _, source := range []struct { + from string + values map[string]any + }{{MeshConfigLayer, mesh}, {NodeConfigLayer(nodeName), node}} { + values := map[string]any{} + for key, p := range m.Preferences { + if p.Predefined == "" { + continue + } + if v, set := source.values[p.Predefined]; set { + values[key] = v + } + } + if len(values) > 0 { + out = append(out, Layer{From: source.from, Values: values, Config: true}) + } + } + return out +} + +// ConfigValueProblem refuses a value of the wrong type for a predefined key or a preference. +func ConfigValueProblem(what string, want string, v any) error { + if got := TypeOf(v); got != want { + shown, _ := json.Marshal(v) + return fmt.Errorf("%s is a %s, and %s is %s", what, want, shown, aOrNothing(got)) + } + return nil +} + +func aOrNothing(t string) string { + if t == "" { + return "none of a number, a string or a boolean" + } + return "a " + t +} + +// preferenceProblems is every way a module's `preferences` are wrong in themselves. +func preferenceProblems(m Manifest) []string { + var problems []string + keys := make([]string, 0, len(m.Preferences)) + for k := range m.Preferences { + keys = append(keys, k) + } + sort.Strings(keys) + for _, key := range keys { + p := m.Preferences[key] + say := func(format string, args ...any) { + problems = append(problems, fmt.Sprintf("%s's preference %q: ", m.Module, key)+fmt.Sprintf(format, args...)) + } + if _, twice := m.Settings[key]; twice { + say("declared under settings too; declare it once, under preferences") + } + if !knownType(p.Type) { + say("the type is %q; a type is %q, %q or %q", p.Type, TypeNumber, TypeString, TypeBoolean) + } else if p.Default == nil { + say("no default: a preference has one") + } else if got := TypeOf(p.Default); got != p.Type { + shown, _ := json.Marshal(p.Default) + say("it is a %s, and its default %s is %s", p.Type, shown, aOrNothing(got)) + } + if p.Predefined != "" && !predefinedName.MatchString(p.Predefined) { + say("%q is not a predefined key's name", p.Predefined) + } + } + return problems +} + +// PredefinedNamed is every predefined key a module's preferences fall back to, sorted. +func PredefinedNamed(m Manifest) []string { + seen := map[string]bool{} + var out []string + for _, p := range m.Preferences { + if p.Predefined != "" && !seen[p.Predefined] { + seen[p.Predefined] = true + out = append(out, p.Predefined) + } + } + sort.Strings(out) + return out +} diff --git a/internal/catalogue/preferences_test.go b/internal/catalogue/preferences_test.go new file mode 100644 index 00000000..160b991a --- /dev/null +++ b/internal/catalogue/preferences_test.go @@ -0,0 +1,117 @@ +package catalogue + +import ( + "strings" + "testing" +) + +// The list of predefined keys is read strictly: a type and a why for each key (novox/hq issue 630). +func TestThePredefinedListIsReadStrictly(t *testing.T) { + list, err := ParsePredefined([]byte(`{"font-size": {"type": "number", "why": "the size text is drawn at"}, + "theme": {"type": "string", "why": "light or dark"}}`)) + if err != nil || len(list) != 2 || list["font-size"].Type != TypeNumber { + t.Fatalf("a good list: %v %v", list, err) + } + for raw, want := range map[string]string{ + `{"font-size": {"type": "integer", "why": "x"}}`: `the type "integer"`, + `{"font-size": {"type": "number"}}`: "no why", + `{"Font Size": {"type": "number", "why": "x"}}`: "not a usable key", + `{"font-size": {"type": "number", "why": "x", "a": 1}}`: "unknown field", + `{"ports": {"type": "number", "why": "x"}}`: "the mesh reads itself", + } { + if _, err := ParsePredefined([]byte(raw)); err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("%s: %v, want %q", raw, err, want) + } + } +} + +// A manifest declares preferences with a type, a default of that type, a why and maybe a predefined key; +// a `settings` entry of kind preference is read as one, its type from its default. +func TestAManifestDeclaresPreferences(t *testing.T) { + file := `"resources": [{"id": "rc", "type": "file", "path": "/etc/rc", "mode": "0644", "trusted": false, + "content": "size = ${setting:font-size} width = ${setting:width}"}]` + m, err := ParseManifest([]byte(`{"module": "dunst", "version": "1", + "preferences": {"font-size": {"type": "number", "default": 10, "predefined": "font-size", "why": "100 DPI"}}, + "settings": {"width": {"kind": "preference", "default": 250, "why": "forty characters"}}, ` + file + `}`)) + if err != nil { + t.Fatal(err) + } + prefs := PreferencesOf(m) + if prefs["font-size"].Predefined != "font-size" || prefs["width"].Type != TypeNumber || prefs["width"].Default != float64(250) { + t.Fatalf("the preferences: %+v", prefs) + } + if got := PredefinedNamed(m); len(got) != 1 || got[0] != "font-size" { + t.Fatalf("the predefined keys named: %v", got) + } + for body, want := range map[string]string{ + `"preferences": {"font-size": {"type": "number", "default": "10", "why": "x"}, "width": {"type": "number", "default": 1, "why": "x"}}`: `its default "10" is a string`, + `"preferences": {"font-size": {"type": "float", "default": 10, "why": "x"}, "width": {"type": "number", "default": 1, "why": "x"}}`: `the type is "float"`, + `"preferences": {"font-size": {"type": "number", "why": "x"}, "width": {"type": "number", "default": 1, "why": "x"}}`: "no default", + `"preferences": {"font-size": {"type": "number", "default": 10}, "width": {"type": "number", "default": 1, "why": "x"}}`: "no why", + } { + if _, err := ParseManifest([]byte(`{"module": "dunst", "version": "1", ` + body + `, ` + file + `}`)); err == nil || + !strings.Contains(err.Error(), want) { + t.Errorf("%s: %v, want %q", body, err, want) + } + } +} + +// A preference falls back only to a key on the list, with the list's type. +func TestAPreferencesPredefinedKeyIsOnTheListWithItsType(t *testing.T) { + m := Manifest{Module: "dunst", Preferences: map[string]Preference{ + "font-size": {Type: TypeNumber, Default: float64(10), Predefined: "font-size", Why: "x"}}} + if p := PredefinedProblems(m, Predefined{"font-size": {Type: TypeNumber, Why: "x"}}); len(p) != 0 { + t.Fatalf("on the list: %v", p) + } + if p := PredefinedProblems(m, Predefined{"theme": {Type: TypeString, Why: "x"}}); len(p) != 1 || + !strings.Contains(p[0], "not on the catalogue's list") || !strings.Contains(p[0], "theme") { + t.Fatalf("not on the list: %v", p) + } + if p := PredefinedProblems(m, Predefined{"font-size": {Type: TypeString, Why: "x"}}); len(p) != 1 || + !strings.Contains(p[0], "is a string") { + t.Fatalf("another type: %v", p) + } +} + +// Mesh and node config fill `${setting:}` between the default and the module's own layer, and — like +// the default — never reach a mergeable file. +func TestConfigLayersFillAPlaceholderAndNoMergeableFile(t *testing.T) { + m := Manifest{Module: "dunst", Preferences: map[string]Preference{ + "font-size": {Type: TypeNumber, Default: float64(10), Predefined: "font-size", Why: "x"}, + "width": {Type: TypeNumber, Default: float64(250), Why: "x"}}} + fill := func(layers []Layer) string { + t.Helper() + r := map[string]any{"id": "rc", "type": "file", "content": "font ${setting:font-size} width ${setting:width}"} + if err := settingInto(r, WithDefaults(m, layers), m.Module); err != nil { + t.Fatal(err) + } + return r["content"].(string) + } + mesh := map[string]any{"font-size": float64(11), "width": float64(999)} + node := map[string]any{"font-size": float64(13)} + if got := fill(nil); got != "font 10 width 250" { + t.Fatalf("the defaults: %s", got) + } + if got := fill(ConfigLayers(m, mesh, nil, "g14")); got != "font 11 width 250" { + t.Fatalf("mesh config, which a preference without predefined does not read: %s", got) + } + both := ConfigLayers(m, mesh, node, "g14") + if got := fill(both); got != "font 13 width 250" { + t.Fatalf("node config over mesh config: %s", got) + } + own := append(both, Layer{From: "g14", Values: map[string]any{"font-size": float64(15)}}) + if got := fill(own); got != "font 15 width 250" { + t.Fatalf("the module's own value over both: %s", got) + } + sources := map[string]string{} + for _, s := range Effective(m, both) { + sources[s.Key] = s.From + } + if sources["font-size"] != NodeConfigLayer("g14") || sources["width"] != DefaultLayer { + t.Fatalf("where each comes from: %v", sources) + } + merged, err := ApplySettings(map[string]any{"id": "conf", "type": "file", "merge": MergeJSON, "content": "{}"}, both) + if err != nil || merged["content"] != "{}\n" { + t.Fatalf("config reached a mergeable file: %v %v", merged["content"], err) + } +} diff --git a/internal/catalogue/setting_defaults.go b/internal/catalogue/setting_defaults.go index 1bc72f67..d8ebb75b 100644 --- a/internal/catalogue/setting_defaults.go +++ b/internal/catalogue/setting_defaults.go @@ -114,15 +114,23 @@ func operatorsWord(key string) string { return "" } -// SettingProblems is every way a definition's setting defaults are wrong, in its own words. +// SettingProblems is every way a definition's setting defaults and preferences are wrong, in its own +// words. The list of predefined keys is not needed here; PredefinedProblems judges against it. func SettingProblems(m Manifest) []string { - if len(m.Settings) == 0 { + declared := PreferencesOf(m) + if len(declared) == 0 { return nil } used := settingKeysUsedBy(m) - var problems []string - for _, key := range sortedSettingKeys(m.Settings) { - d := m.Settings[key] + problems := preferenceProblems(m) + keys := make([]string, 0, len(declared)) + for k := range declared { + keys = append(keys, k) + } + sort.Strings(keys) + for _, key := range keys { + d := declared[key] + _, old := m.Settings[key] say := func(format string, args ...any) { problems = append(problems, fmt.Sprintf("%s's setting %q: ", m.Module, key)+fmt.Sprintf(format, args...)) } @@ -134,23 +142,27 @@ func SettingProblems(m Manifest) []string { say("the mesh reads %q itself, and a module gives it no default", key) continue } - if d.Kind != KindPreference { - say("kind is %q, and only a %q has a default (novox/hq ADR 0262)", d.Kind, KindPreference) + if old && m.Settings[key].Kind != KindPreference { + say("kind is %q, and only a %q has a default (novox/hq ADR 0262)", m.Settings[key].Kind, KindPreference) } if word := operatorsWord(key); word != "" { say("a key naming %q is the operator's value, and has no default — it is the "+ "assignment's, never the definition's (novox/hq ADR 0112, ADR 0262)", word) } - switch v := d.Default.(type) { - case string: - if strings.TrimSpace(v) == "" { - say("an empty default is no default; give the value, or declare nothing") + if old { + switch v := d.Default.(type) { + case string: + if strings.TrimSpace(v) == "" { + say("an empty default is no default; give the value, or declare nothing") + } + case float64, bool: + case nil: + say("no default: a key without one is the operator's, and is not declared here") + default: + say("a default is a string, a number or a boolean, and this is %T", d.Default) } - case float64, bool: - case nil: - say("no default: a key without one is the operator's, and is not declared here") - default: - say("a default is a string, a number or a boolean, and this is %T", d.Default) + } else if v, isString := d.Default.(string); isString && strings.TrimSpace(v) == "" { + say("an empty default is no default; give the value, or declare nothing") } if strings.TrimSpace(d.Why) == "" { say("no why: say in one sentence why this default") @@ -164,11 +176,12 @@ func SettingProblems(m Manifest) []string { // Defaults is the layer a module's own defaults make, or nothing when it gives none. func Defaults(m Manifest) (Layer, bool) { - if len(m.Settings) == 0 { + declared := PreferencesOf(m) + if len(declared) == 0 { return Layer{}, false } values := map[string]any{} - for key, d := range m.Settings { + for key, d := range declared { if d.Default != nil { values[key] = d.Default } @@ -189,7 +202,7 @@ func WithDefaults(m Manifest, layers []Layer) []Layer { type SettingSource struct { Key string Value any - // From is DefaultLayer, MeshWideLayer or the node's name. + // From is DefaultLayer, MeshConfigLayer, NodeConfigLayer(node), MeshWideLayer or the node's name. From string // FromDefault is whether the value is the module's default, whatever From reads. FromDefault bool @@ -199,10 +212,11 @@ type SettingSource struct { } // Effective is every key a module gives a default or a layer sets, with its value and where it came -// from: the default, then the mesh's layer, then the node's — later wins. +// from: the default, then mesh config, then node config, then the module's own layer on the node — +// later wins (novox/hq issue 630). func Effective(m Manifest, layers []Layer) []SettingSource { byKey := map[string]*SettingSource{} - for key, d := range m.Settings { + for key, d := range PreferencesOf(m) { byKey[key] = &SettingSource{Key: key, Value: d.Default, From: DefaultLayer, FromDefault: true, Default: d.Default, HasDefault: true} } diff --git a/internal/catalogue/setting_into.go b/internal/catalogue/setting_into.go index ee04ee18..8f2b6002 100644 --- a/internal/catalogue/setting_into.go +++ b/internal/catalogue/setting_into.go @@ -188,7 +188,7 @@ func wholeID(v string) (uint64, bool) { func OwnerSettingProblems(m Manifest) []string { var problems []string asked := settingKeysAskedOutsideOwners(m) - for k := range m.Settings { + for k := range PreferencesOf(m) { asked[k] = true } for _, r := range m.Resources { diff --git a/internal/catalogue/setting_takes.go b/internal/catalogue/setting_takes.go index 0cb7fca4..f239eac0 100644 --- a/internal/catalogue/setting_takes.go +++ b/internal/catalogue/setting_takes.go @@ -43,7 +43,7 @@ func onlyTaken(layers []Layer, taken map[string]bool) []Layer { values[key] = value } } - kept = append(kept, Layer{From: layer.From, Values: values, Default: layer.Default}) + kept = append(kept, Layer{From: layer.From, Values: values, Default: layer.Default, Config: layer.Config}) } return kept } @@ -83,7 +83,7 @@ func TakenSettingsProblems(m Manifest) []string { // Said only when none of the module's mergeable files names its keys: when one does, §2 already // names every file that does not, and a file is not reported twice. func WrittenSettingsProblems(m Manifest) []string { - if len(m.Settings) == 0 { + if len(PreferencesOf(m)) == 0 { return nil } var silent []string diff --git a/internal/catalogue/settings.go b/internal/catalogue/settings.go index 71357453..aa89f294 100644 --- a/internal/catalogue/settings.go +++ b/internal/catalogue/settings.go @@ -37,6 +37,9 @@ type Layer struct { // Default marks the layer a module's own defaults make (novox/hq ADR 0262). Marked rather than // recognised by From, which is a node's name for a node's layer, and a node may be called anything. Default bool + // Config marks a layer of mesh config or node config (novox/hq issue 630): values for predefined keys, + // given to the preferences that fall back to them. Like the defaults, it fills `${setting:}` only. + Config bool } // ApplySettings produces a resource's final content from the module's own and the layers over it. @@ -136,7 +139,7 @@ func overridden(base map[string]any, layers []Layer, what string) (map[string]an values[key] = value } } - kept = append(kept, Layer{From: layer.From, Values: values, Default: layer.Default}) + kept = append(kept, Layer{From: layer.From, Values: values, Default: layer.Default, Config: layer.Config}) } merged, err := settle(base, kept, nil, what) if err != nil { @@ -173,7 +176,7 @@ func settle(base map[string]any, layers []Layer, protected map[string]bool, what map[string]any, error) { merged := deepCopy(base) for _, layer := range layers { - if layer.Default { + if layer.Default || layer.Config { // A module's own defaults fill ${setting:} only (novox/hq ADR 0262). A mergeable // file's content already is its defaults, and a contribution or served fact declares its // own; laid on here, a default would reach every mergeable file of its module (issue 168). @@ -363,7 +366,7 @@ func UnusedSettings(m Manifest, layers []Layer) []string { } // A key the module gives a default reaches what asks for it (novox/hq ADR 0262): setting it // overrides the default, which is how a running module gains a setting without a gap between. - for key := range m.Settings { + for key := range PreferencesOf(m) { lands[key] = true } for _, values := range m.Contributes { diff --git a/internal/catalogue/verbs.go b/internal/catalogue/verbs.go index 8b13beae..62a9bc42 100644 --- a/internal/catalogue/verbs.go +++ b/internal/catalogue/verbs.go @@ -299,10 +299,33 @@ var ControllerVerbs = []Verb{ "adopted": "\"true\" when the machine is in use and joins adopted", }, nil, "adopted"), Replaces: []string{"mesh-controller token issue", "wg set"}}, - {Name: "settings", Description: "Read or set what an assignment is configured with: a module's settings for the whole " + - "mesh, or for one machine. Without values or clear, answers the layer as it stands — read it before setting it — " + + {Name: "config", Description: "Read or set config values (novox/hq issue 630). A module's preference resolves from " + + "four sources, the last set winning: its default; mesh config, a mesh-wide value for a predefined key; node " + + "config, a value for a predefined key on one node; and the module's own value on that node. Mesh and node " + + "config hold values only for the predefined keys, the catalogue's predefined.json, which a preference names " + + "as its predefined; there is no module layer for the whole mesh. Without key: the predefined keys with their " + + "mesh and node config, then each module's preferences with the value on each node and its source (default, " + + "mesh config, node config (), on ); node and module narrow it. With key and value (JSON; " + + "a bare word is a string): sets mesh config, or node config with node, or with module and node the module's own " + + "value on that node — refused for a key not on the list, a value of the wrong type, and a module without " + + "node. With clear instead of value: removes it, and the next source down applies. A set or clear says why, " + + "is recorded in the hand-act log with the caller, keeps the value it replaced in the history, and takes effect " + + "at the next push of the machines it reaches, which the answer names. With history: a key's past mesh or node " + + "config values.", + Input: schema(map[string]string{ + "key": "a predefined key (mesh or node config), or with module one of that module's preferences", + "value": "the value as JSON (12, \"dark\", true); a bare word is a string", + "node": "one node: its node config, or with module that module's value there", + "module": "a module: its own value on node; needs node", + "clear": "\"true\" to remove the value instead of setting it", + "history": "\"true\": the key's past mesh config values, or with node its node config's", + "why": "why it is set or cleared, kept with the value and in the hand-act log; required to set or clear", + }, nil, "clear", "history")}, + {Name: "settings", Description: "Read or set what an assignment is configured with: a module's settings on one " + + "machine; there is no module layer for the whole mesh (novox/hq issue 630) — a set or proposal without node is " + + "refused, and a mesh-wide value of a predefined key is the config verb's. Without values or clear, answers the layer as it stands — read it before setting it — " + "then every value the module gives a default or a layer sets, with where it comes from: the module's default, " + - "the mesh, or the machine (novox/hq ADR 0262); with list \"preferences\", or with no module, every " + + "mesh config, node config, or the module on the machine (novox/hq ADR 0262, issue 630); with list \"preferences\", or with no module, every " + "module's preferences and each machine's value; " + "with history, the layers it replaced. Setting replaces that layer whole and answers each key it adds (+), " + "changes (~) and removes (-); a set that would remove a key is refused unless replace says it is meant " + @@ -317,7 +340,7 @@ var ControllerVerbs = []Verb{ Input: schema(map[string]string{ "module": "the module's name; with list, only that module's", "values": "the settings as a JSON object, for set or propose", - "node": "one machine; the whole mesh when absent", + "node": "one machine; needed to set or propose values (no module layer for the whole mesh); without it, clear removes a whole-mesh layer left over", "clear": "\"true\" to remove the layer instead of setting it, or to propose its removal; not with values", "replace": "\"true\": with values, the set (or the proposal) is meant to remove the keys the layer had and it does not name", "history": "\"true\": without values or clear, the layers this one replaced, the latest first", diff --git a/internal/facts/facts.go b/internal/facts/facts.go index be55f445..5f61e744 100644 --- a/internal/facts/facts.go +++ b/internal/facts/facts.go @@ -59,6 +59,10 @@ type Facts struct { Modules []Module `json:"modules"` // Settings are the mesh-wide layer of every module's settings, scrubbed. Settings []Settings `json:"settings,omitempty"` + // Predefined is the list of predefined keys the mesh holds, and Config the mesh config's value for + // each it sets, scrubbed (novox/hq issue 630). A machine's node config is on the machine. + Predefined map[string]PredefinedKey `json:"predefined,omitempty"` + Config map[string]any `json:"config,omitempty"` // Edges are the build dependencies between modules, as the mesh recorded them — what a merge's // rebuild width is computed from. Edges []Edge `json:"edges,omitempty"` @@ -143,6 +147,9 @@ type Machine struct { Pins []Pin `json:"pins,omitempty"` // Settings are its own layer of each module's settings, scrubbed. Settings []Settings `json:"settings,omitempty"` + // Config is the node config of the machine: its value for each predefined key it sets, scrubbed + // (novox/hq issue 630). + Config map[string]any `json:"config,omitempty"` // Accepted are the secrets a person gave the mesh for modules here, by name only: the mesh cannot // make them, so a check composing this machine gives each a stand-in instead of refusing it. Accepted []Accepted `json:"accepted,omitempty"` @@ -173,6 +180,12 @@ type Accepted struct { Local string `json:"local,omitempty"` } +// PredefinedKey is one key of the list of predefined keys. +type PredefinedKey struct { + Type string `json:"type"` + Why string `json:"why"` +} + // Settings is one layer of one module's settings. type Settings struct { Module string `json:"module"` diff --git a/internal/inventory/catalogue.go b/internal/inventory/catalogue.go index 6ce3740c..78e02e5f 100644 --- a/internal/inventory/catalogue.go +++ b/internal/inventory/catalogue.go @@ -117,6 +117,18 @@ func (i *Inventory) RegisterModule(ctx context.Context, m catalogue.Manifest, fr } } } + // **A preference falls back only to a key on the catalogue's list, and with its type** (novox/hq issue + // 630): mesh and node config hold values for the predefined keys alone, so a `predefined` the list does + // not hold is a fallback nobody can ever set. + if names := catalogue.PredefinedNamed(m); len(names) > 0 { + list, err := i.Predefined(ctx) + if err != nil { + return err + } + if problems := catalogue.PredefinedProblems(m, list); len(problems) > 0 { + return fmt.Errorf("%s is not registered: %s", m.Module, strings.Join(problems, "; ")) + } + } raw, err := json.Marshal(m) if err != nil { return err @@ -853,6 +865,14 @@ func (i *Inventory) setSettings(ctx context.Context, nodeName, module string, va } else if _, err := i.declared(ctx, module); err != nil { return fmt.Errorf("%w: %s", ErrNoSuchModule, module) } + if nodeName == "" && judge { + // **There is no module layer for the whole mesh** (novox/hq issue 630): an assignment is one module + // on one node. What every node should share is mesh config, for a predefined key. A layer the mesh + // already holds is still kept as it is (KeepSettings, the merge gate raising a store from facts). + return fmt.Errorf("%s: there is no module layer for the whole mesh — an assignment is one module on "+ + "one node (novox/hq issue 630). Set it on each node with --node, or, for a predefined key, as mesh "+ + "config with the config verb. Nothing was changed", module) + } if nodeName == "" { // A port is a fact about one machine (novox/hq ADR 0100). Refused here, in composition's // words: stored, it refuses every node running the module at composition, and the mesh @@ -1133,7 +1153,8 @@ func (i *Inventory) ClearSettingsBy(ctx context.Context, nodeName, module, clear return tx.Commit(ctx) } -// SettingsFor is the layers that apply to one module on one node, in the order they are applied. +// SettingsFor is the layers that apply to one module on one node, in the order they are applied: mesh +// config, node config (both marked Config), then the module's own layer on the node. // // The mesh's first, then the node's, so a node that differs is expressed by differing rather // than by restating everything the rest of the mesh already says. @@ -1151,7 +1172,12 @@ func (i *Inventory) SettingsFor(ctx context.Context, nodeName, module string) ([ } defer rows.Close() - var layers []catalogue.Layer + // Mesh config, then node config, for the predefined keys the module's preferences fall back to + // (novox/hq issue 630): under the module's own layers, over its defaults. + layers, err := i.configLayers(ctx, node.ID, nodeName, module) + if err != nil { + return nil, err + } for rows.Next() { var meshWide bool var raw []byte diff --git a/internal/inventory/config.go b/internal/inventory/config.go new file mode 100644 index 00000000..45ae2097 --- /dev/null +++ b/internal/inventory/config.go @@ -0,0 +1,339 @@ +package inventory + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + "github.com/jackc/pgx/v5" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// Mesh config and node config (novox/hq issue 630): a value for a predefined key, for the whole mesh or for +// one node, that every module whose preference names the key falls back to. The list of predefined keys is +// the catalogue's predefined.json, kept here as the last build of the catalogue carried it. + +// ErrNotPredefined is a key mesh or node config was asked to hold that is not on the list. +var ErrNotPredefined = errors.New("not a predefined key") + +// SetPredefined replaces the list of predefined keys with the one a build of `repository` at `commit` +// carried. Replaced whole: the file is a statement of the whole list. +func (i *Inventory) SetPredefined(ctx context.Context, list catalogue.Predefined, repository, commit string) error { + tx, err := i.store.Pool().Begin(ctx) + if err != nil { + return err + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + if _, err := tx.Exec(ctx, `delete from predefined_key`); err != nil { + return err + } + for _, key := range list.Keys() { + k := list[key] + if _, err := tx.Exec(ctx, + `insert into predefined_key (key, type, why, repository, commit) values ($1, $2, $3, $4, $5)`, + key, k.Type, k.Why, nullable(repository), nullable(commit)); err != nil { + return err + } + } + return tx.Commit(ctx) +} + +// Predefined is the list of predefined keys the mesh holds; empty before a build of the catalogue carried one. +func (i *Inventory) Predefined(ctx context.Context) (catalogue.Predefined, error) { + rows, err := i.store.Pool().Query(ctx, `select key, type, why from predefined_key`) + if err != nil { + return nil, err + } + defer rows.Close() + out := catalogue.Predefined{} + for rows.Next() { + var key string + var k catalogue.PredefinedKey + if err := rows.Scan(&key, &k.Type, &k.Why); err != nil { + return nil, err + } + out[key] = k + } + return out, rows.Err() +} + +// ConfigValue is one value of mesh config (Node "") or node config. +type ConfigValue struct { + Key string + Node string + Value any + SetAt time.Time + SetBy string + Why string +} + +// Config is every value of mesh config and node config, by key, the mesh's first, then by node. +func (i *Inventory) Config(ctx context.Context) ([]ConfigValue, error) { + rows, err := i.store.Pool().Query(ctx, + `select c.key, coalesce(n.name, ''), c.value, c.set_at, coalesce(c.set_by, ''), c.why + from config c left join node n on n.id = c.node + order by c.key, c.node is not null, n.name`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []ConfigValue + for rows.Next() { + var v ConfigValue + var raw []byte + if err := rows.Scan(&v.Key, &v.Node, &raw, &v.SetAt, &v.SetBy, &v.Why); err != nil { + return nil, err + } + if err := json.Unmarshal(raw, &v.Value); err != nil { + return nil, err + } + out = append(out, v) + } + return out, rows.Err() +} + +// SetConfig sets a predefined key's value for the whole mesh (node "") or for one node. Refused for a key not +// on the list and for a value not of the key's type. The value it replaces is kept in config_history. +func (i *Inventory) SetConfig(ctx context.Context, key, nodeName string, value any, setBy, why string) error { + if strings.TrimSpace(why) == "" { + return errors.New("config says why: one sentence, kept with the value") + } + if err := i.JudgeConfig(ctx, key, nodeName, value); err != nil { + return err + } + raw, err := json.Marshal(value) + if err != nil { + return err + } + nodeID, err := i.layerNode(ctx, nodeName) + if err != nil { + return err + } + tx, err := i.store.Pool().Begin(ctx) + if err != nil { + return err + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + if _, err := tx.Exec(ctx, keepConfigSQL, key, nodeID, "set", why); err != nil { + return err + } + if _, err := tx.Exec(ctx, + `delete from config where key = $1 and node is not distinct from $2::uuid`, key, nodeID); err != nil { + return err + } + if _, err := tx.Exec(ctx, + `insert into config (key, node, value, set_by, why) values ($1, $2::uuid, $3, $4, $5)`, + key, nodeID, raw, nullable(setBy), why); err != nil { + return err + } + return tx.Commit(ctx) +} + +// JudgeConfig refuses, before anything is done, a value mesh or node config may not hold: a key not on the +// list of predefined keys, or a value not of the key's type. +func (i *Inventory) JudgeConfig(ctx context.Context, key, nodeName string, value any) error { + list, err := i.Predefined(ctx) + if err != nil { + return err + } + k, on := list[key] + if !on { + return notPredefined(key, nodeName, list) + } + return catalogue.ConfigValueProblem(fmt.Sprintf("the predefined key %q", key), k.Type, value) +} + +// ClearConfig removes a predefined key's value for the whole mesh (node "") or for one node, keeping it in +// config_history; false when there was none. A key no longer on the list may still be cleared. +func (i *Inventory) ClearConfig(ctx context.Context, key, nodeName, clearedBy, why string) (bool, error) { + if strings.TrimSpace(why) == "" { + return false, errors.New("config says why: one sentence, kept with the history") + } + nodeID, err := i.layerNode(ctx, nodeName) + if err != nil { + return false, err + } + tx, err := i.store.Pool().Begin(ctx) + if err != nil { + return false, err + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + if _, err := tx.Exec(ctx, keepConfigSQL, key, nodeID, "clear", why); err != nil { + return false, err + } + if clearedBy != "" { + if _, err := tx.Exec(ctx, + `update config_history set replaced_by = 'clear by ' || $3 + where key = $1 and node is not distinct from $2::uuid + and replaced_at = (select max(replaced_at) from config_history + where key = $1 and node is not distinct from $2::uuid)`, + key, nodeID, clearedBy); err != nil { + return false, err + } + } + tag, err := tx.Exec(ctx, `delete from config where key = $1 and node is not distinct from $2::uuid`, key, nodeID) + if err != nil { + return false, err + } + return tag.RowsAffected() > 0, tx.Commit(ctx) +} + +// PastConfig is one value of mesh or node config that was replaced or cleared. +type PastConfig struct { + Value any + SetAt time.Time + SetBy string + Why string + ReplacedAt time.Time + ReplacedBy string + ReplacedWhy string +} + +// ConfigHistory is the values a predefined key had for the whole mesh (node "") or one node, latest first. +func (i *Inventory) ConfigHistory(ctx context.Context, key, nodeName string) ([]PastConfig, error) { + nodeID, err := i.layerNode(ctx, nodeName) + if err != nil { + return nil, err + } + rows, err := i.store.Pool().Query(ctx, + `select value, coalesce(set_at, 'epoch'), coalesce(set_by, ''), coalesce(why, ''), replaced_at, replaced_by, + coalesce(replaced_why, '') + from config_history where key = $1 and node is not distinct from $2::uuid order by replaced_at desc`, + key, nodeID) + if err != nil { + return nil, err + } + defer rows.Close() + var out []PastConfig + for rows.Next() { + var p PastConfig + var raw []byte + if err := rows.Scan(&raw, &p.SetAt, &p.SetBy, &p.Why, &p.ReplacedAt, &p.ReplacedBy, &p.ReplacedWhy); err != nil { + return nil, err + } + if err := json.Unmarshal(raw, &p.Value); err != nil { + return nil, err + } + out = append(out, p) + } + return out, rows.Err() +} + +// keepConfigSQL copies the value about to be replaced or cleared into the history, with why it goes. +const keepConfigSQL = `insert into config_history (key, node, value, set_at, set_by, why, replaced_by, replaced_why) + select key, node, value, set_at, set_by, why, $3, $4 from config + where key = $1 and node is not distinct from $2::uuid` + +func notPredefined(key, nodeName string, list catalogue.Predefined) error { + where := "mesh config" + if nodeName != "" { + where = "node config" + } + listed := "the list holds none yet: it arrives with a build of the catalogue carrying " + catalogue.PredefinedFile + if len(list) > 0 { + listed = "the list holds " + strings.Join(list.Keys(), ", ") + } + return fmt.Errorf("%w: %s holds values only for the predefined keys, and %q is not one (%s); a module's own "+ + "value is set with module and node", ErrNotPredefined, where, key, listed) +} + +// configLayers are the layers mesh config and node config make for one module on one node: nothing when +// neither holds a value, or the module names no predefined key. +func (i *Inventory) configLayers(ctx context.Context, nodeID, nodeName, module string) ([]catalogue.Layer, error) { + rows, err := i.store.Pool().Query(ctx, + `select key, node is null, value from config where node is null or node = $1`, nodeID) + if err != nil { + return nil, err + } + defer rows.Close() + mesh, node := map[string]any{}, map[string]any{} + for rows.Next() { + var key string + var meshWide bool + var raw []byte + if err := rows.Scan(&key, &meshWide, &raw); err != nil { + return nil, err + } + var v any + if err := json.Unmarshal(raw, &v); err != nil { + return nil, err + } + if meshWide { + mesh[key] = v + } else { + node[key] = v + } + } + if err := rows.Err(); err != nil { + return nil, err + } + if len(mesh)+len(node) == 0 { + return nil, nil + } + // Only the module's preferences, read from its stored definition: a whole catalogue per module per + // node would be read for every machine planned. + var raw []byte + err = i.store.Pool().QueryRow(ctx, `select manifest -> 'preferences' from module where name = $1`, module).Scan(&raw) + if errors.Is(err, pgx.ErrNoRows) || len(raw) == 0 || string(raw) == "null" { + return nil, nil + } + if err != nil { + return nil, err + } + var prefs map[string]catalogue.Preference + if err := json.Unmarshal(raw, &prefs); err != nil { + return nil, fmt.Errorf("%s's preferences cannot be read: %w", module, err) + } + return catalogue.ConfigLayers(catalogue.Manifest{Module: module, Preferences: prefs}, mesh, node, nodeName), nil +} + +// SetModuleValue sets or clears one key of a module's own layer on one node — its module-assignment config — +// keeping the rest of the layer, judged as `settings set` judges it, with the layer it replaces kept. +// value nil clears the key; the layer goes when no key is left. +func (i *Inventory) SetModuleValue(ctx context.Context, nodeName, module, key string, value any, setBy string) ( + before any, had bool, err error) { + if nodeName == "" { + return nil, false, errors.New("there is no module layer for the whole mesh: a module's own value is " + + "set on one node (novox/hq issue 630); for every node, set mesh config for a predefined key") + } + layer, has, err := i.Layer(ctx, nodeName, module) + if err != nil { + return nil, false, err + } + values := map[string]any{} + for k, v := range layer { + values[k] = v + } + before, had = values[key] + if value == nil { + if !had { + return nil, false, nil + } + delete(values, key) + if len(values) == 0 && has { + return before, had, i.ClearSettingsBy(ctx, nodeName, module, setBy) + } + } else { + values[key] = value + } + return before, had, i.SetSettingsBy(ctx, nodeName, module, values, setBy) +} + +// ConfigKeysSorted is the keys of a set of config values, sorted, for saying them. +func ConfigKeysSorted(values []ConfigValue) []string { + seen := map[string]bool{} + var out []string + for _, v := range values { + if !seen[v.Key] { + seen[v.Key] = true + out = append(out, v.Key) + } + } + sort.Strings(out) + return out +} diff --git a/internal/inventory/config_test.go b/internal/inventory/config_test.go new file mode 100644 index 00000000..749d4d05 --- /dev/null +++ b/internal/inventory/config_test.go @@ -0,0 +1,274 @@ +package inventory + +import ( + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// dunstLike is a module whose font-size falls back to the predefined key font-size (novox/hq issue 630). +func dunstLike() catalogue.Manifest { + return catalogue.Manifest{Module: "dunst", Version: "1", + Preferences: map[string]catalogue.Preference{ + "font-size": {Type: catalogue.TypeNumber, Default: float64(10), Predefined: "font-size", Why: "readable at 100 DPI"}, + "width": {Type: catalogue.TypeNumber, Default: float64(250), Why: "forty characters"}, + }, + Resources: []map[string]any{{"id": "rc", "type": "file", "path": "/etc/dunstrc", "mode": "0644", + "content": "font = Inter ${setting:font-size}\nwidth = ${setting:width}\n"}}} +} + +func fontList() catalogue.Predefined { + return catalogue.Predefined{"font-size": {Type: catalogue.TypeNumber, Why: "the size text is drawn at"}} +} + +// effectiveOn is a module's value for a key on a node, and its source, as composition reads them. +func effectiveOn(t *testing.T, inv *Inventory, node, key string) (any, string) { + t.Helper() + layers, err := inv.SettingsFor(t.Context(), node, "dunst") + if err != nil { + t.Fatal(err) + } + for _, s := range catalogue.Effective(dunstLike(), layers) { + if s.Key == key { + if s.FromDefault { + return s.Value, catalogue.DefaultLayer + } + return s.Value, s.From + } + } + t.Fatalf("no %s", key) + return nil, "" +} + +// Each of the four sources in turn, the most specific winning, and clearing each falls back to the next +// one down (novox/hq issue 630). +func TestAConfigValueComesFromFourSourcesTheMostSpecificWinning(t *testing.T) { + inv := fresh(t) + ctx := t.Context() + for _, n := range []string{"g14", "shanks"} { + if _, err := inv.AddNode(ctx, n); err != nil { + t.Fatal(err) + } + } + if err := inv.SetPredefined(ctx, fontList(), "novox/mesh-catalog", "c1"); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(ctx, dunstLike(), Source{}); err != nil { + t.Fatal(err) + } + for _, n := range []string{"g14", "shanks"} { + if _, err := inv.Assign(ctx, n, "dunst"); err != nil { + t.Fatal(err) + } + } + want := func(node string, value float64, from string) { + t.Helper() + v, f := effectiveOn(t, inv, node, "font-size") + if v != value || f != from { + t.Fatalf("font-size on %s: %v from %q, want %v from %q", node, v, f, value, from) + } + } + want("g14", 10, "default") + + if err := inv.SetConfig(ctx, "font-size", "", float64(11), "the operator", "a larger default everywhere"); err != nil { + t.Fatal(err) + } + want("g14", 11, "mesh config") + want("shanks", 11, "mesh config") + + if err := inv.SetConfig(ctx, "font-size", "g14", float64(13), "the operator", "a dense screen"); err != nil { + t.Fatal(err) + } + want("g14", 13, "node config (g14)") + want("shanks", 11, "mesh config") + + if _, _, err := inv.SetModuleValue(ctx, "g14", "dunst", "font-size", float64(15), "the operator"); err != nil { + t.Fatal(err) + } + want("g14", 15, "g14") + want("shanks", 11, "mesh config") + + // A preference without predefined reads only its default and the module's own value. + if v, f := effectiveOn(t, inv, "g14", "width"); v != float64(250) || f != "default" { + t.Fatalf("width: %v from %s", v, f) + } + + // Clearing each falls back to the next source down. + if _, _, err := inv.SetModuleValue(ctx, "g14", "dunst", "font-size", nil, "the operator"); err != nil { + t.Fatal(err) + } + want("g14", 13, "node config (g14)") + if removed, err := inv.ClearConfig(ctx, "font-size", "g14", "the operator", "back to the mesh's"); err != nil || !removed { + t.Fatalf("clear node config: %v %v", removed, err) + } + want("g14", 11, "mesh config") + if removed, err := inv.ClearConfig(ctx, "font-size", "", "the operator", "back to the default"); err != nil || !removed { + t.Fatalf("clear mesh config: %v %v", removed, err) + } + want("g14", 10, "default") + if _, has, _ := inv.Layer(ctx, "g14", "dunst"); has { + t.Fatal("the module's layer was left empty rather than removed") + } + + // What each replaced is kept, with why. + past, err := inv.ConfigHistory(ctx, "font-size", "g14") + if err != nil || len(past) != 1 || past[0].Value != float64(13) || past[0].Why != "a dense screen" || + past[0].ReplacedBy != "clear by the operator" || past[0].ReplacedWhy != "back to the mesh's" { + t.Fatalf("node config history: %+v %v", past, err) + } +} + +// Mesh and node config hold values for predefined keys only, of their type; a module value needs a node. +func TestConfigRefusesAKeyNotPredefinedAWrongTypeAndAModuleValueForTheWholeMesh(t *testing.T) { + inv := fresh(t) + ctx := t.Context() + if _, err := inv.AddNode(ctx, "g14"); err != nil { + t.Fatal(err) + } + err := inv.SetConfig(ctx, "font-size", "", float64(12), "the operator", "bigger") + if !errors.Is(err, ErrNotPredefined) || !strings.Contains(err.Error(), "the list holds none yet") { + t.Fatalf("a key before any list: %v", err) + } + if err := inv.SetPredefined(ctx, fontList(), "", ""); err != nil { + t.Fatal(err) + } + err = inv.SetConfig(ctx, "colour", "g14", "blue", "the operator", "blue") + if !errors.Is(err, ErrNotPredefined) || !strings.Contains(err.Error(), "node config") || + !strings.Contains(err.Error(), "font-size") { + t.Fatalf("a key not on the list: %v", err) + } + if err := inv.SetConfig(ctx, "font-size", "", "large", "the operator", "bigger"); err == nil || + !strings.Contains(err.Error(), "is a number") { + t.Fatalf("a string for a number: %v", err) + } + if err := inv.SetConfig(ctx, "font-size", "", float64(12), "the operator", ""); err == nil { + t.Fatal("a value without why was kept") + } + if err := inv.SetConfig(ctx, "font-size", "nowhere", float64(12), "the operator", "x"); err == nil { + t.Fatal("node config for a node the mesh does not know was kept") + } + if _, _, err := inv.SetModuleValue(ctx, "", "dunst", "font-size", float64(12), "the operator"); err == nil || + !strings.Contains(err.Error(), "no module layer for the whole mesh") { + t.Fatalf("a module value for the whole mesh: %v", err) + } + if values, err := inv.Config(ctx); err != nil || len(values) != 0 { + t.Fatalf("a refused value was kept: %v %v", values, err) + } +} + +// Registration refuses a preference whose predefined key is not on the list, or whose type differs. +func TestRegistrationJudgesPreferencesAgainstThePredefinedList(t *testing.T) { + inv := fresh(t) + ctx := t.Context() + err := inv.RegisterModule(ctx, dunstLike(), Source{}) + if err == nil || !strings.Contains(err.Error(), `predefined key "font-size"`) { + t.Fatalf("a predefined key before any list: %v", err) + } + if err := inv.SetPredefined(ctx, catalogue.Predefined{"font-size": {Type: catalogue.TypeString, Why: "x"}}, "", ""); err != nil { + t.Fatal(err) + } + err = inv.RegisterModule(ctx, dunstLike(), Source{}) + if err == nil || !strings.Contains(err.Error(), "is a number") || !strings.Contains(err.Error(), "is a string") { + t.Fatalf("a type other than the list's: %v", err) + } + if err := inv.SetPredefined(ctx, fontList(), "", ""); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(ctx, dunstLike(), Source{}); err != nil { + t.Fatal(err) + } +} + +// The module layer for the whole mesh is moved into each assigned node's layer by migration 0096: the mesh's +// values under the node's, maps merged key by key, every replaced layer kept in the history (novox/hq issue 630). +func TestTheWholeMeshModuleLayerMovesIntoEachAssignedNode(t *testing.T) { + inv := fresh(t) + ctx := t.Context() + for _, n := range []string{"novox", "g14", "shanks"} { + if _, err := inv.AddNode(ctx, n); err != nil { + t.Fatal(err) + } + } + m := catalogue.Manifest{Module: "artifact-store-tools", Version: "1", + Resources: []map[string]any{{"id": "conf", "type": "file", "path": "/etc/t.json", "content": "{}", "merge": "json"}}} + if err := inv.RegisterModule(ctx, m, Source{}); err != nil { + t.Fatal(err) + } + for _, n := range []string{"novox", "g14"} { + if _, err := inv.Assign(ctx, n, "artifact-store-tools"); err != nil { + t.Fatal(err) + } + } + // As held before the migration: a whole-mesh layer, and g14's own layer over it. + if err := inv.KeepSettings(ctx, "", "artifact-store-tools", map[string]any{ + "older-store-names": "registry-api.novox.be", "nested": map[string]any{"a": 1, "b": 1}}); err != nil { + t.Fatal(err) + } + if err := inv.SetSettings(ctx, "g14", "artifact-store-tools", map[string]any{ + "nested": map[string]any{"b": 2}, "own": true}); err != nil { + t.Fatal(err) + } + + migrations, err := Migrations() + if err != nil { + t.Fatal(err) + } + var moving string + for _, mig := range migrations { + if mig.Number == 96 { + moving = mig.SQL[strings.Index(mig.SQL, "create function pg_temp"):] + } + } + if moving == "" { + t.Fatal("migration 0096 moves no whole-mesh layer") + } + tx, err := inv.store.Pool().Begin(ctx) + if err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(ctx, moving); err != nil { + t.Fatal(err) + } + if err := tx.Commit(ctx); err != nil { + t.Fatal(err) + } + + layer := func(node string) string { + t.Helper() + values, _, err := inv.Layer(ctx, node, "artifact-store-tools") + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(values) + return string(raw) + } + if got := layer("novox"); got != `{"nested":{"a":1,"b":1},"older-store-names":"registry-api.novox.be"}` { + t.Fatalf("novox's layer: %s", got) + } + if got := layer("g14"); got != `{"nested":{"a":1,"b":2},"older-store-names":"registry-api.novox.be","own":true}` { + t.Fatalf("g14's layer, the mesh's under its own: %s", got) + } + if got := layer("shanks"); got != "null" && got != "{}" { + t.Fatalf("a node the module is not assigned to gained a layer: %s", got) + } + if _, has, _ := inv.Layer(ctx, "", "artifact-store-tools"); has { + t.Fatal("the whole-mesh layer is still there") + } + mesh, err := inv.SettingsHistory(ctx, "", "artifact-store-tools") + if err != nil || len(mesh) != 1 || mesh[0].ReplacedBy != "moved-to-nodes" || + mesh[0].Values["older-store-names"] != "registry-api.novox.be" { + t.Fatalf("the whole-mesh layer's history: %+v %v", mesh, err) + } + past, err := inv.SettingsHistory(ctx, "g14", "artifact-store-tools") + if err != nil || len(past) != 1 || past[0].ReplacedBy != "moved-to-nodes" || past[0].Values["own"] != true { + t.Fatalf("g14's replaced layer: %+v %v", past, err) + } + // And composition reads the same values on each node as before. + layers, err := inv.SettingsFor(ctx, "g14", "artifact-store-tools") + if err != nil || len(layers) != 1 { + t.Fatalf("g14's layers: %+v %v", layers, err) + } +} diff --git a/internal/inventory/forget_test.go b/internal/inventory/forget_test.go index b28e317c..5283b77f 100644 --- a/internal/inventory/forget_test.go +++ b/internal/inventory/forget_test.go @@ -39,7 +39,7 @@ func TestRegisteringAModuleAgainKeepsWhatTheMeshHoldsForIt(t *testing.T) { if err := inv.RegisterModule(ctx, m, Source{}); err != nil { t.Fatal(err) } - if err := inv.SetSettings(ctx, "", "step-ca", map[string]any{"issuer": "the mesh"}); err != nil { + if err := inv.KeepSettings(ctx, "", "step-ca", map[string]any{"issuer": "the mesh"}); err != nil { t.Fatal(err) } if err := inv.SetSettings(ctx, "anchor", "step-ca", map[string]any{"port": 9000}); err != nil { @@ -162,7 +162,7 @@ func TestDiscardingAModuleSaysWhatWentWithIt(t *testing.T) { if err := inv.RegisterModule(ctx, withOwnSecret(manifest("step-ca", nil, nil), "password"), Source{}); err != nil { t.Fatal(err) } - if err := inv.SetSettings(ctx, "", "step-ca", map[string]any{"issuer": "the mesh"}); err != nil { + if err := inv.KeepSettings(ctx, "", "step-ca", map[string]any{"issuer": "the mesh"}); err != nil { t.Fatal(err) } if err := inv.AcceptSecretForModule(ctx, "anchor", "step-ca", "password", "sealed"); err != nil { diff --git a/internal/inventory/migrations/0096-config-values-come-from-four-sources.sql b/internal/inventory/migrations/0096-config-values-come-from-four-sources.sql new file mode 100644 index 00000000..db667c34 --- /dev/null +++ b/internal/inventory/migrations/0096-config-values-come-from-four-sources.sql @@ -0,0 +1,92 @@ +-- A config value comes from four sources, the last set winning (novox/hq issue 630): the module's +-- default, mesh config, node config, and the module's own value on the node. Mesh config and node config +-- hold values only for the predefined keys, a fixed list kept in the catalogue (`predefined.json`). An +-- assignment is one module on one node: there is no module layer for the whole mesh any more. + +-- The list of predefined keys, as the catalogue's predefined.json last said it. Replaced whole when a +-- build of the catalogue carries the file; `repository` and `commit` say which. +create table predefined_key ( + key text primary key, + type text not null check (type in ('number', 'string', 'boolean')), + why text not null, + repository text, + commit text, + listed_at timestamptz not null default now() +); + +-- Mesh config (node null) and node config: one value per predefined key, and per node. Not a foreign key +-- to predefined_key: a key the list drops keeps its value, said as reaching nothing, rather than losing it. +create table config ( + key text not null, + node uuid references node(id) on delete cascade, + value jsonb not null, + set_at timestamptz not null default now(), + set_by text, + why text not null +); +create unique index config_for_the_mesh on config (key) where node is null; +create unique index config_for_a_node on config (key, node) where node is not null; + +-- Every value that was replaced or cleared, with who set it and why, and who replaced it and why. +create table config_history ( + key text not null, + node uuid references node(id) on delete cascade, + value jsonb not null, + set_at timestamptz, + set_by text, + why text, + replaced_at timestamptz not null default now(), + -- set · clear + replaced_by text not null, + replaced_why text +); +create index config_history_by_key on config_history (key, node, replaced_at desc); + +-- The module layer for the whole mesh goes: each one is laid under the layer of every node the module is +-- assigned to — the mesh's values first, the node's over them, maps merged key by key as composition +-- merged them — so every machine composes what it did. Every layer replaced, and the mesh-wide layer +-- itself, is kept in settings_history (ADR 0217) with the reason `moved-to-nodes`. A mesh-wide layer of +-- a module assigned nowhere is kept in the history only: no machine read it. +create function pg_temp.config_630_merge(under jsonb, over jsonb) returns jsonb language plpgsql immutable as $$ +declare + merged jsonb; + k text; +begin + if jsonb_typeof(under) is distinct from 'object' or jsonb_typeof(over) is distinct from 'object' then + return over; + end if; + merged := under; + for k in select jsonb_object_keys(over) loop + if merged ? k then + merged := jsonb_set(merged, array[k], pg_temp.config_630_merge(merged -> k, over -> k)); + else + merged := merged || jsonb_build_object(k, over -> k); + end if; + end loop; + return merged; +end +$$; + +insert into settings_history (node, module, values, set_at, replaced_by, set_by) +select t.node, t.module, t.values, t.set_at, 'moved-to-nodes', t.set_by + from settings t + join settings s on s.module = t.module and s.node is null + where t.node is not null + and exists (select 1 from assignment a where a.node = t.node and a.module = t.module); + +insert into settings_history (node, module, values, set_at, replaced_by, set_by) +select null, module, values, set_at, 'moved-to-nodes', set_by from settings where node is null; + +insert into settings (node, module, values, set_at, set_by) +select a.node, s.module, s.values, now(), + coalesce(s.set_by || '; ', '') || 'moved from the whole mesh''s layer to this node''s (novox/hq issue 630)' + from settings s + join assignment a on a.module = s.module + where s.node is null +on conflict (node, module) where node is not null +do update set values = pg_temp.config_630_merge(excluded.values, settings.values), + set_at = now(), + set_by = coalesce(settings.set_by || '; ', '') || + 'the whole mesh''s layer laid under it (novox/hq issue 630)'; + +delete from settings where node is null; diff --git a/internal/inventory/settings_judged_test.go b/internal/inventory/settings_judged_test.go index 1563fa17..6fb84b5d 100644 --- a/internal/inventory/settings_judged_test.go +++ b/internal/inventory/settings_judged_test.go @@ -40,15 +40,17 @@ func TestASettingIsJudgedWhereItIsStored(t *testing.T) { } // A key that reaches nothing is refused too, where the spelling can still be fixed; a module // with a mergeable file takes any key. - err = inv.SetSettings(ctx, "", "plain", map[string]any{"colour": "blue"}) + err = inv.SetSettings(ctx, "anchor", "plain", map[string]any{"colour": "blue"}) if err == nil || !strings.Contains(err.Error(), "reach nothing") || !strings.Contains(err.Error(), `"colour"`) { t.Fatalf("a stray key was stored: %v", err) } - if err := inv.SetSettings(ctx, "", "web", map[string]any{"colour": "blue"}); err != nil { - t.Fatal(err) + // There is no module layer for the whole mesh (novox/hq issue 630): a new one is refused. + err = inv.SetSettings(ctx, "", "web", map[string]any{"colour": "blue"}) + if err == nil || !strings.Contains(err.Error(), "no module layer for the whole mesh") { + t.Fatalf("a module layer for the whole mesh was stored: %v", err) } - // The mesh-wide layer is under the node's when the node's is judged. - if err := inv.SetSettings(ctx, "anchor", "web", map[string]any{catalogue.PortsSetting: map[string]any{"8080": 10000}}); err != nil { + if err := inv.SetSettings(ctx, "anchor", "web", map[string]any{"colour": "blue", + catalogue.PortsSetting: map[string]any{"8080": 10000}}); err != nil { t.Fatal(err) } // A kept network is for an adopted machine only (rule 4). diff --git a/internal/inventory/unseen_test.go b/internal/inventory/unseen_test.go index 73a98031..3fcfe5d1 100644 --- a/internal/inventory/unseen_test.go +++ b/internal/inventory/unseen_test.go @@ -56,10 +56,10 @@ func TestTheLayerASetReplacesIsKeptAndReadable(t *testing.T) { t.Errorf("the replaced layer still has what the set dropped: %+v", past[1]) } // The mesh-wide layer keeps its own history, apart from the node's. - if err := inv.SetSettings(ctx, "", "web", map[string]any{"colour": "green"}); err != nil { + if err := inv.KeepSettings(ctx, "", "web", map[string]any{"colour": "green"}); err != nil { t.Fatal(err) } - if err := inv.SetSettings(ctx, "", "web", map[string]any{"colour": "grey"}); err != nil { + if err := inv.KeepSettings(ctx, "", "web", map[string]any{"colour": "grey"}); err != nil { t.Fatal(err) } mesh, err := inv.SettingsHistory(ctx, "", "web") diff --git a/internal/link/build.go b/internal/link/build.go index 8262b8b9..7a26489c 100644 --- a/internal/link/build.go +++ b/internal/link/build.go @@ -250,6 +250,11 @@ type BuildResult struct { // the repository's default is the branch it follows. Branches []string `json:"branches,omitempty"` + // Predefined is the built repository's list of predefined keys, its root predefined.json at the commit + // built, when it has one (novox/hq issue 630). The controller keeps it, when it registers the build, + // as the list mesh and node config hold values for. Empty from a builder that predates it. + Predefined json.RawMessage `json:"predefined,omitempty"` + // SourceFingerprint is what the build was made from, hashed (novox/hq issue 280): the module's // tree at the commit, the trees of the contexts it read, its bases and toolchains by digest. Two // builds with one fingerprint are one build, however their digests differ — an image is not diff --git a/module.json b/module.json index 4ceb785d..ad7e591f 100644 --- a/module.json +++ b/module.json @@ -69,6 +69,7 @@ "issue", "token", "settings", + "config", "command", "queue", "cancel",