diff --git a/RELEASE-57.md b/RELEASE-57.md new file mode 100644 index 00000000..140ccff6 --- /dev/null +++ b/RELEASE-57.md @@ -0,0 +1,6 @@ +# Release 57: config and provisioning + +This branch carries the work of hq design 57 (ADR 0348, ADR 0349) in this repository. Each package +merges here through its own reviewed pull request; the branch reaches main as one pull request, in one +delivery group with the branch of the same name in mesh-controller, mesh-host, mesh-sdk, mesh-catalog +and hq, after a rehearsal in the lab. This file is deleted by the last commit before that pull request. diff --git a/cmd/mesh-builder/main.go b/cmd/mesh-builder/main.go index ff1e1062..b23b862a 100644 --- a/cmd/mesh-builder/main.go +++ b/cmd/mesh-builder/main.go @@ -428,6 +428,11 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri for _, s := range built.Sources { result.Sources = append(result.Sources, link.BuildSource{Repository: s.Repository, Ref: s.Ref, Paths: s.Paths}) } + // The list of predefined keys the manifest was judged against, for the controller to publish + // (novox/hq ADR 0348 §7). + if p := built.Predefined; p != nil { + result.Predefined = &link.PredefinedList{Keys: p.Keys, Generation: p.Generation} + } say("built", built.Manifest.Module+" from "+short(built.Commit)) } } diff --git a/cmd/mesh-controller/adoption.go b/cmd/mesh-controller/adoption.go index 411ae05d..afa90a07 100644 --- a/cmd/mesh-controller/adoption.go +++ b/cmd/mesh-controller/adoption.go @@ -424,8 +424,9 @@ func comparisonOf(module string, c comparison, opts takeOptions) (preview string } // And its settings on this machine, composed against its definition (rule 1, rule 6). for _, layer := range c.layers { - keys := make([]string, 0, len(layer.Values)) - for k := range layer.Values { + said := layer.Whole() + keys := make([]string, 0, len(said)) + for k := range said { keys = append(keys, k) } sort.Strings(keys) diff --git a/cmd/mesh-controller/apply_failing.go b/cmd/mesh-controller/apply_failing.go index 6daf960d..986207ce 100644 --- a/cmd/mesh-controller/apply_failing.go +++ b/cmd/mesh-controller/apply_failing.go @@ -38,15 +38,61 @@ func watchApplyFailing(f *signalFacts) []conditions.Observation { } } var out []conditions.Observation + said := map[string]bool{} for _, d := range f.doing { if !d.Wrong() || d.Since == nil || f.now.Sub(*d.Since) <= applyEvery || !heard[d.Node] { continue } out = append(out, applyFailing(d, f.now)) + said[d.Node] = true + } + // **A module's files the provisioner failed to place are a failed apply too** (novox/hq ADR 0362 §3): its client's + // newest word, judged by the same bound — the client says a failure once, and fills again on every change — + // on a machine whose node-engine applied what it was sent. + failed := map[string][]inventory.ProvReport{} + for _, r := range f.provFailed { + if r.Status != inventory.ProvFailed || f.now.Sub(r.At) <= applyEvery || !heard[r.Node] || said[r.Node] { + continue + } + failed[r.Node] = append(failed[r.Node], r) + } + for _, node := range sortedNames(failed) { + out = append(out, provisioningFailing(node, failed[node], f.now)) } return out } +// provisioningFailing is one machine's S21 condition for the modules whose files its provisioner client failed to +// place. +func provisioningFailing(m string, failed []inventory.ProvReport, now time.Time) conditions.Observation { + sort.Slice(failed, func(a, b int) bool { return failed[a].Module < failed[b].Module }) + var names, words []string + since := failed[0].At + for _, r := range failed { + names = append(names, r.Module) + words = append(words, r.Module+": "+oneLine(orNoReason(r.Reason))) + if r.At.Before(since) { + since = r.At + } + } + o := conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Kind: kindApplyFailing, Machine: m, + Severity: conditions.Warning, Said: strings.Join(words, "; "), + Resolved: m + " applies what it is sent again"} + o.Summary = fmt.Sprintf("%s's provisioner has failed to place the files of %s since %s (bound %s)", m, + strings.Join(names, ", "), since.UTC().Format("2006-01-02 15:04 MST"), ago(applyEvery)) + o.Headline = m + " is not applying its update" + o.Explanation = fmt.Sprintf("The provisioner on %s has not been able to write the files of %s for %s. The "+ + "details say why.", m, strings.Join(names, ", "), humanDuration(now.Sub(since))) + return o +} + +func orNoReason(s string) string { + if s == "" { + return "no reason given" + } + return s +} + // applyFailing is one machine's S21 condition. func applyFailing(d inventory.Doing, now time.Time) conditions.Observation { m := d.Node diff --git a/cmd/mesh-controller/build.go b/cmd/mesh-controller/build.go index 2997292d..5a6802e1 100644 --- a/cmd/mesh-controller/build.go +++ b/cmd/mesh-controller/build.go @@ -183,8 +183,10 @@ func buildFrom(result link.BuildResult) inventory.Build { kept.Manifest = recordedManifest(result.Manifest, announced) // The module name comes from the manifest, which only exists when the build got that far. if len(kept.Manifest) > 0 { - if m, err := catalogue.ParseManifest(kept.Manifest); err == nil { - kept.Module = m.Module + if list, err := judgedWith(result); err == nil { + if m, err := catalogue.ParseManifestWith(kept.Manifest, list); err == nil { + kept.Module = m.Module + } } } return kept @@ -603,6 +605,11 @@ func saysWhenThePolicyActs(ctx context.Context, inv *inventory.Inventory, module // written with the same values. func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResult) ( catalogue.Manifest, inventory.Build, error) { + // The list of predefined keys as the store keeps it now: another controller process may have kept a newer one. + if err := loadPredefined(ctx, inv); err != nil { + return catalogue.Manifest{}, inventory.Build{}, fmt.Errorf("the list of predefined keys kept cannot be "+ + "read: %w", err) + } kept := buildFrom(result) if err := inv.RecordBuild(ctx, kept); err != nil { return catalogue.Manifest{}, kept, err @@ -613,7 +620,15 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu return catalogue.Manifest{}, kept, fmt.Errorf("%s could not build %s:\n%s", result.On, result.Repository, result.Failed) } - manifest, err := catalogue.ParseManifest(kept.Manifest) + // Judged first against the list the build was judged against: the catalogue's at the commit built (novox/hq + // ADR 0348 §7), so a module naming a key its own commit adds is read; and again below against the list the + // mesh publishes, before it is registered. + list, err := judgedWith(result) + if err != nil { + return catalogue.Manifest{}, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On, + result.Repository, short(result.Commit), err) + } + manifest, err := catalogue.ParseManifestWith(kept.Manifest, list) if err != nil { return catalogue.Manifest{}, kept, fmt.Errorf("%s built %s and what came back is not a manifest: %w", result.On, result.Repository, err) @@ -687,6 +702,23 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu "back: it is recorded and not registered again — a newer build is", result.On, manifest.Module, short(result.Commit)) } + // **The newest trunk build's list of predefined keys is published** (novox/hq ADR 0348 §7): a build of the + // catalogue that may be registered keeps its list when its commit is newer than the one kept. Then the module + // is judged against the list the mesh publishes, as every manifest read again is: one naming a key that list + // does not hold, or of another type, is refused here and never enters the catalogue. + if carried, err := carriedList(result); err != nil { + return manifest, kept, err + } else if carried != nil { + if err := publishPredefined(ctx, inv, result, carried); err != nil { + fmt.Printf("%s: the list of predefined keys it carried is not published: %v\n", result.ID, err) + } + } + if manifest.NamesPredefined() { + if _, err := catalogue.ParseManifest(kept.Manifest); err != nil { + return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: judged against the "+ + "list of predefined keys the mesh publishes, %w", result.On, manifest.Module, short(result.Commit), err) + } + } // **A build whose source is unchanged is never a move** (novox/hq issue 280): a rebuild made from // what the build the mesh stands on was made from registers that build's artifacts at the new // commit, so no machine is sent a new digest for a source nobody changed — an image is not diff --git a/cmd/mesh-controller/check.go b/cmd/mesh-controller/check.go index 16e16ba6..0aeebb18 100644 --- a/cmd/mesh-controller/check.go +++ b/cmd/mesh-controller/check.go @@ -47,6 +47,24 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error { shelf := catalogue.Shelf{} faulted := map[string]bool{} failed := 0 + // The list of predefined keys at the root of the repository above each manifest (novox/hq ADR 0348 §7), read + // once per repository: a manifest naming a key is judged against the list its own tree holds. + lists := map[string]catalogue.PredefinedList{} + listOf := func(path string) (catalogue.PredefinedList, error) { + root := catalogue.RepositoryRoot(filepath.Dir(path)) + if root == "" { + return nil, nil + } + if list, read := lists[root]; read { + return list, nil + } + list, err := catalogue.ReadPredefined(root) + if err != nil { + return nil, err + } + lists[root] = list + return list, nil + } for _, path := range paths { raw, err := os.ReadFile(path) if err != nil { @@ -54,7 +72,18 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error { failed++ continue } - m, err := catalogue.ParseManifest(raw) + list, err := listOf(path) + if err != nil { + fmt.Fprintf(out, "%s: %v\n", path, err) + failed++ + continue + } + m, err := catalogue.ParseManifestWith(raw, list) + if err == nil { + // Each template's text read from the tree beside the manifest, and judged as inline content is + // (novox/hq ADR 0349 §2): a missing template is refused by name. + m, err = catalogue.WithTemplatesFrom(m, filepath.Dir(path)) + } if err != nil { fmt.Fprintf(out, "%s: %v\n", path, err) failed++ @@ -215,6 +244,12 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error { checks = append(checks, fmt.Sprintf("%v by %s", r["id"], catalogue.HealthWords(h))) } } + // And its own runtime's, where it runs as its own account (novox/hq ADR 0366). + if own, has := catalogue.OwnRuntimeHealth(m); has { + if h, _, _ := catalogue.ReadHealth(own); h.Kind != "" { + checks = append(checks, fmt.Sprintf("%v by %s", own["id"], catalogue.HealthWords(h))) + } + } if len(checks) > 0 { fmt.Fprintf(out, ", ready: %s", strings.Join(checks, "; ")) } diff --git a/cmd/mesh-controller/config_move.go b/cmd/mesh-controller/config_move.go new file mode 100644 index 00000000..1b65621d --- /dev/null +++ b/cmd/mesh-controller/config_move.go @@ -0,0 +1,616 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "reflect" + "sort" + "strings" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// **The move of config** (novox/hq ADR 0348 §9 and its consequences, design 57 K5, issue 664): every config value a +// settings layer holds is set in mesh-config, one of the moves of data a release runs once after its delivery, after +// the move of placement. A module's layer on a node becomes its own value on that node (`module...`). +// A whole-mesh layer has no module value for the whole mesh (ADR 0348 §3): its value becomes mesh config for the +// predefined key it falls back to where that module alone falls back to the key (claude-code's managed settings +// become `agent-managed-settings`), and otherwise the module's own value on each node it is assigned to, laid under +// that node's layer as composition lays it. A key a layer still says by its file's name (nodered's `timeZone`) is +// set under its config key (`time-zone`, issue 751). +// +// **Every node composes byte for byte as before.** The move sets the values, reads every module's resolved values +// back through mesh-config and compares them with what the layers lay today; only when they agree does it empty the +// layers it moved (each kept whole in its history first) and turn on the switch (inventory.FinishConfigMove), after +// which the controller reads mesh-config where it read the layers. A module still in the settings form keeps its +// layers and is said so. Anything that cannot move as it composes refuses the whole move, by name, before anything +// is set. Asked again, it sets nothing that is set already. + +// configLevel is where mesh-config keeps a value: mesh config, node config, or a module's own value on a node. +type configLevel struct { + Level string `json:"level"` + Node string `json:"node,omitempty"` + Module string `json:"module,omitempty"` +} + +func (l configLevel) valueKey(key string) string { + switch l.Level { + case "node": + return "node." + l.Node + "." + key + case "module": + return "module." + l.Node + "." + l.Module + "." + key + } + return "mesh." + key +} + +func (l configLevel) String() string { + switch l.Level { + case "node": + return "node config on " + l.Node + case "module": + return l.Module + "'s own value on " + l.Node + } + return "mesh config" +} + +// configSet is one value the move sets in mesh-config, and the layers it comes from. +type configSet struct { + configLevel + Key string `json:"key"` + Value any `json:"value"` + Trusted bool `json:"trusted,omitempty"` + // From is the layers it comes from, as ` on : `. + From []string `json:"from"` + // Already is true when mesh-config holds this value at this level already: nothing is set. + Already bool `json:"already,omitempty"` +} + +// configMoveAnswer is what the move of config says it did, or would do. +type configMoveAnswer struct { + // DryRun is true when nothing was set: asked without --confirm. + DryRun bool `json:"dry-run"` + Sets []configSet `json:"sets"` + // Moved is each layer whose values the move sets, emptied once they read back. + Moved []string `json:"moved"` + // Stays is each layer of a module still in the settings form: kept, and composed from as before. + Stays []string `json:"stays,omitempty"` + // Unread is each layer that composes nothing (its module is not assigned where it says): kept in the history, + // emptied, and set nowhere. + Unread []string `json:"unread,omitempty"` + // Problems is why the move refuses, each by name; nothing is set while there is one. + Problems []string `json:"problems,omitempty"` + // Switched is true once the controller reads mesh-config where it read the layers. + Switched bool `json:"switched"` +} + +// configMovePlan is the move worked out from the layers, before anything is asked of mesh-config. +type configMovePlan struct { + answer configMoveAnswer + // moved is the layers the move empties once the values read back. + moved []inventory.LayerToMove + // expected is, for each module the move reaches, keyed `.`, every value composition lays over + // its files today: what mesh-config must resolve for it from mesh config, node config or its own value. + expected map[string]map[string]any + // modules is each module in the config form, by the nodes it is assigned to: whose resolved values are read back. + modules map[string][]string +} + +func whereOf(node string) string { + if node == "" { + return "the whole mesh" + } + return node +} + +// planConfigMove works out the move from every layer that holds a value, the catalogue, and where each module is +// assigned. +func planConfigMove(layers []inventory.LayerToMove, shelf map[string]catalogue.Manifest, + on map[string][]string) configMovePlan { + p := configMovePlan{answer: configMoveAnswer{Sets: []configSet{}, Moved: []string{}}, + expected: map[string]map[string]any{}, modules: map[string][]string{}} + problem := func(format string, a ...any) { + p.answer.Problems = append(p.answer.Problems, fmt.Sprintf(format, a...)) + } + for name, m := range shelf { + if m.DeclaresConfig() { + p.modules[name] = on[name] + } + } + + byModule := map[string]*layersOfModule{} + var names []string + for _, l := range layers { + said := fmt.Sprintf("%s on %s", l.Module, whereOf(l.Node)) + m, known := shelf[l.Module] + if !known || !m.DeclaresConfig() { + p.answer.Stays = append(p.answer.Stays, said+" (its module is in the settings form)") + continue + } + if _, placement := catalogue.SplitPlacement(l.Values); len(placement) > 0 { + problem("%s still holds placement (%s): run move-placement first", said, strings.Join(movedKeys(placement), ", ")) + continue + } + assigned := on[l.Module] + if (l.Node == "" && len(assigned) == 0) || (l.Node != "" && !containsString(assigned, l.Node)) { + p.answer.Unread = append(p.answer.Unread, said+" (the module is not assigned there, so it composes nothing)") + p.moved = append(p.moved, l) + continue + } + // The layer's keys as config keys: a key the file names otherwise is set under its config key. + as := catalogue.TakenAs(m) + values := map[string]any{} + from := map[string]string{} + for _, k := range movedKeys(l.Values) { + key := k + if _, declared := m.Settings[k]; !declared { + if renamed, ok := as[k]; ok { + key = renamed + } else { + problem("%s says %q, which %s does not declare: declare it, or clear it from the layer", said, k, l.Module) + continue + } + } + if _, both := values[key]; both { + // The layer says the key and the file's name for it: the config key wins, as composition has it (issue 751). + if key != k { + continue + } + } + values[key] = l.Values[k] + from[key] = said + ": " + k + } + om := byModule[l.Module] + if om == nil { + om = &layersOfModule{nodes: map[string]map[string]any{}, saidAt: map[string]map[string]string{}} + byModule[l.Module] = om + names = append(names, l.Module) + } + om.saidAt[l.Node] = from + if l.Node == "" { + om.mesh = values + } else { + om.nodes[l.Node] = values + } + p.moved = append(p.moved, l) + p.answer.Moved = append(p.answer.Moved, said) + } + sort.Strings(names) + + // Which predefined key each module falls back to, so a whole-mesh value goes to mesh config only where no other + // module's value moves with it. + fallingBack := map[string][]string{} + for name, m := range shelf { + for _, d := range m.Settings { + if d.Predefined != "" && !containsString(fallingBack[d.Predefined], name) { + fallingBack[d.Predefined] = append(fallingBack[d.Predefined], name) + } + } + } + + for _, name := range names { + m, om := shelf[name], byModule[name] + asked := catalogue.AskedKeys(m) + keys := map[string]bool{} + for k := range om.mesh { + keys[k] = true + } + for _, values := range om.nodes { + for k := range values { + keys[k] = true + } + } + for _, key := range movedKeys(keys) { + d := m.Settings[key] + meshValue, meshSays := om.mesh[key] + alone := d.Predefined != "" && len(fallingBack[d.Predefined]) == 1 + if meshSays && alone { + p.set(configLevel{Level: "mesh"}, d.Predefined, meshValue, d, om.said("", key)) + } + for _, node := range on[name] { + nodeValue, nodeSays := om.nodes[node][key] + value := meshValue + switch { + case nodeSays && meshSays: + _, under := meshValue.(map[string]any) + _, over := nodeValue.(map[string]any) + if under && over && asked[key] { + problem("%s on %s lays an object for %q over the whole mesh's, and a placeholder takes it whole "+ + "where its file merges the two: say the value on %s whole, then move", name, node, key, node) + continue + } + value = catalogue.LaidOver(meshValue, nodeValue) + case nodeSays: + value = nodeValue + case !meshSays: + continue + } + p.expect(node, name, key, value) + if nodeSays || !alone { + saidBy := om.said(node, key) + if meshSays { + saidBy = append(om.said("", key), saidBy...) + } + p.set(configLevel{Level: "module", Node: node, Module: name}, key, value, d, saidBy) + } + } + } + } + sort.Strings(p.answer.Problems) + return p +} + +// layersOfModule is one module's layers as the move reads them: its whole-mesh layer's config values, each node's, +// and where each key was said, by node ("" for the whole mesh). +type layersOfModule struct { + mesh map[string]any + nodes map[string]map[string]any + saidAt map[string]map[string]string +} + +func (om *layersOfModule) said(node, key string) []string { + if s, ok := om.saidAt[node][key]; ok { + return []string{s} + } + return nil +} + +// set is one value the move sets, judged against its declaration as mesh-config will judge it. +func (p *configMovePlan) set(level configLevel, key string, value any, d catalogue.SettingDeclaration, from []string) { + p.answer.Sets = append(p.answer.Sets, configSet{configLevel: level, Key: key, Value: value, Trusted: d.Trusted, + From: from}) + for _, wrong := range catalogue.ValueProblems(d, value) { + p.answer.Problems = append(p.answer.Problems, fmt.Sprintf("%s at %s: %s", key, level, wrong)) + } +} + +func (p *configMovePlan) expect(node, module, key string, value any) { + k := node + "." + module + if p.expected[k] == nil { + p.expected[k] = map[string]any{} + } + p.expected[k][key] = value +} + +func movedKeys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// configService is mesh-config as the move asks it: what it holds at each level, what it resolved, and a value set +// as the controller carries it from its terminal. +type configService interface { + // Values is every value set, by its key in mesh-config_values (`mesh.`, `node..`, + // `module...`). + Values(ctx context.Context) (map[string]json.RawMessage, error) + // Resolved is every resolved value. + Resolved(ctx context.Context) ([]resolvedEntry, error) + // Set sets one value at its level, by whom and why. + Set(ctx context.Context, level configLevel, key string, value any, by, why string) error +} + +// runConfigMove is the move against a store and mesh-config: a dry run says what it would set. +func runConfigMove(ctx context.Context, inv *inventory.Inventory, config configService, apply bool, by, + why string) (configMoveAnswer, error) { + layers, holding, err := inv.LayersToMove(ctx) + if err != nil { + return configMoveAnswer{}, err + } + entries, err := inv.Catalogued(ctx) + if err != nil { + return configMoveAnswer{}, err + } + shelf := map[string]catalogue.Manifest{} + on := map[string][]string{} + for _, e := range entries { + shelf[e.Manifest.Module] = e.Manifest + on[e.Manifest.Module] = e.On + } + p := planConfigMove(layers, shelf, on) + if len(holding) > 0 && len(p.answer.Problems) == 0 { + p.answer.Problems = append(p.answer.Problems, "a layer still holds placement ("+strings.Join(holding, "; ")+ + "): run move-placement first") + } + moved, err := inv.ConfigMoved(ctx) + if err != nil { + return configMoveAnswer{}, err + } + p.answer.Switched = moved.Moved + if config == nil { + return p.answer, errors.New("mesh-config cannot be asked, so nothing was moved") + } + held, err := config.Values(ctx) + if err != nil { + return p.answer, fmt.Errorf("what mesh-config holds cannot be read, so nothing was moved: %w", err) + } + for i := range p.answer.Sets { + s := &p.answer.Sets[i] + if was, ok := held[s.valueKey(s.Key)]; ok && sameJSON(was, s.Value) { + s.Already = true + } + } + p.answer.DryRun = !apply + if !apply || len(p.answer.Problems) > 0 { + if apply { + return p.answer, fmt.Errorf("the move of config refuses, and nothing was set: %s", + strings.Join(p.answer.Problems, "; ")) + } + return p.answer, nil + } + if len(p.moved) == 0 && moved.Moved { + return p.answer, nil + } + for _, s := range p.answer.Sets { + if s.Already { + continue + } + said := "moved from the settings layer of " + strings.Join(s.From, " and ") + " (issue 664): " + why + if err := config.Set(ctx, s.configLevel, s.Key, s.Value, by, said); err != nil { + return p.answer, fmt.Errorf("%s at %s was not set, and no layer was emptied; ask the move again: %w", + s.Key, s.configLevel, err) + } + } + // Read back through mesh-config before anything points at it (design 57): every module the move reaches, and + // every other in the config form, must resolve exactly what its layers lay today. + if differs, err := readsBack(ctx, config, p); err != nil { + return p.answer, err + } else if len(differs) > 0 { + return p.answer, fmt.Errorf("mesh-config does not resolve what the layers lay today, so no layer was emptied and "+ + "the controller still composes from the layers: %s", strings.Join(differs, "; ")) + } + if err := inv.FinishConfigMove(ctx, p.moved, by, why); err != nil { + return p.answer, err + } + p.answer.Switched = true + return p.answer, nil +} + +// readsBack compares what mesh-config resolved with what the layers lay, for every module in the config form on +// every node it is assigned to, and says each difference. A value the layers do not lay and mesh-config resolves from +// a level would be laid once the switch is on: a difference too. +func readsBack(ctx context.Context, config configService, p configMovePlan) ([]string, error) { + var resolved []resolvedEntry + var err error + // mesh-config writes the resolved values with each set; a pass that missed one is put right by its next + // resolution, so a difference is read again for a little while before it is said. + for wait := 0; ; wait++ { + if resolved, err = config.Resolved(ctx); err != nil { + return nil, fmt.Errorf("mesh-config's resolved values cannot be read back, so no layer was emptied: %w", err) + } + differs, err := differences(resolved, p) + if err != nil || len(differs) == 0 || wait >= readBackTries { + return differs, err + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(readBackEvery): + } + } +} + +// How often, and how many times, the move reads mesh-config's resolved values back before it says they differ. +var ( + readBackEvery = 2 * time.Second + readBackTries = 15 +) + +func differences(resolved []resolvedEntry, p configMovePlan) ([]string, error) { + var differs []string + for _, module := range movedKeys(p.modules) { + for _, node := range p.modules[module] { + laid, err := configLaid(resolved, node, module) + if err != nil { + return nil, err + } + want := p.expected[node+"."+module] + if want == nil { + want = map[string]any{} + } + for _, k := range movedKeys(mergedKeys(laid, want)) { + got, has := laid[k] + w, wants := want[k] + switch { + case !wants: + differs = append(differs, fmt.Sprintf("%s on %s resolves %s from a level no layer says; clear it "+ + "there, or set it in a layer first", module, node, k)) + case !has: + differs = append(differs, fmt.Sprintf("%s on %s does not resolve %s from the value moved", module, node, k)) + case !reflect.DeepEqual(normalJSON(got), normalJSON(w)): + differs = append(differs, fmt.Sprintf("%s on %s resolves %s as %s, and its layers lay %s", module, + node, k, compactJSON(got), compactJSON(w))) + } + } + } + } + return differs, nil +} + +func mergedKeys(a, b map[string]any) map[string]bool { + out := map[string]bool{} + for k := range a { + out[k] = true + } + for k := range b { + out[k] = true + } + return out +} + +func normalJSON(v any) any { + raw, _ := json.Marshal(v) + var out any + _ = json.Unmarshal(raw, &out) + return out +} + +func compactJSON(v any) string { + raw, _ := json.Marshal(v) + return string(raw) +} + +func sameJSON(raw json.RawMessage, v any) bool { + var held any + if err := json.Unmarshal(raw, &held); err != nil { + return false + } + return reflect.DeepEqual(held, normalJSON(v)) +} + +// busConfig is mesh-config on this controller's bus: its buckets read directly, a value set through its verb in +// the controller's own name, so mesh-config takes a trusted value as given at the controller's terminal (ADR 0348 §8). +type busConfig struct{} + +func (busConfig) Values(ctx context.Context) (map[string]json.RawMessage, error) { + out := map[string]json.RawMessage{} + err := onTheBus(func(conn *nats.Conn) error { + js, err := jetstream.New(conn) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(ctx, 20*time.Second) + defer cancel() + kv, err := js.KeyValue(ctx, broker.BucketName("mesh-config", "values")) + if err != nil { + return err + } + w, err := kv.WatchAll(ctx, jetstream.IgnoreDeletes()) + if err != nil { + return err + } + defer func() { _ = w.Stop() }() + for e := range w.Updates() { + if e == nil { + return nil + } + var v struct { + Value json.RawMessage `json:"value"` + } + if err := json.Unmarshal(e.Value(), &v); err != nil { + return fmt.Errorf("mesh-config holds %q, which is not a value: %w", e.Key(), err) + } + out[e.Key()] = v.Value + } + return errors.New("mesh-config's values stopped answering") + }) + return out, err +} + +func (busConfig) Resolved(ctx context.Context) ([]resolvedEntry, error) { return readResolved(ctx) } + +func (busConfig) Set(ctx context.Context, level configLevel, key string, value any, by, why string) error { + args := map[string]any{"action": "set", "key": key, "value": value, "why": why, "by": by} + if level.Node != "" { + args["node"] = level.Node + } + if level.Module != "" { + args["module"] = level.Module + } + return onTheBus(func(conn *nats.Conn) error { + answer, err := link.AskMeshSeatToolAs(ctx, conn, broker.ControllerName, "mesh-config", "config", args, 30*time.Second) + if err != nil { + return err + } + if answer.Error != "" { + return errors.New(answer.Error) + } + return nil + }) +} + +// moveConfigCommand is `move-config`: the move of config (issue 664). +func moveConfigCommand(ctx context.Context, args []string) error { + set := flag.NewFlagSet("move-config", flag.ContinueOnError) + asJSON := set.Bool("json", false, "answer as JSON") + confirm := set.Bool("confirm", false, "set the values and switch, rather than only say what would move") + f := addHandActFlags(set) + positionals, err := parseAround(set, args) + if err != nil { + return err + } + if len(positionals) != 0 { + return errors.New("move-config [--json] [--confirm --why ]") + } + if !*confirm && f.given() { + return errors.New("move-config takes --why only with --confirm: without it nothing is moved. Nothing was done") + } + if *confirm { + if err := f.require("move-config"); err != nil { + return err + } + } + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + if *confirm { + f.record(ctx, "move-config", nil) + } + why := "" + if *confirm { + why = strings.TrimSpace(*f.why) + } + answer, err := runConfigMove(ctx, open.inventory, busConfig{}, *confirm, "the move of config, run by "+link.Caller(), why) + if *asJSON { + encoder := json.NewEncoder(os.Stdout) + encoder.SetIndent("", " ") + if encErr := encoder.Encode(answer); encErr != nil { + return encErr + } + return err + } + fmt.Print(describeConfigMove(answer)) + return err +} + +// describeConfigMove says the answer as a person reads it. +func describeConfigMove(a configMoveAnswer) string { + var b strings.Builder + verb := "set" + if a.DryRun { + verb = "would set" + } + for _, s := range a.Sets { + state := verb + if s.Already { + state = "holds already" + } + fmt.Fprintf(&b, "%s %s = %s at %s (from %s)\n", state, s.Key, compactJSON(s.Value), s.configLevel, + strings.Join(s.From, " and ")) + } + for _, s := range a.Unread { + fmt.Fprintf(&b, "emptied, set nowhere: %s\n", s) + } + for _, s := range a.Stays { + fmt.Fprintf(&b, "stays: %s\n", s) + } + for _, s := range a.Problems { + fmt.Fprintf(&b, "refused: %s\n", s) + } + switch { + case len(a.Problems) > 0: + b.WriteString("nothing was set: the move refuses until each is put right\n") + case a.DryRun: + fmt.Fprintf(&b, "%d layer(s), %d value(s); nothing was set — with --confirm --why they are\n", + len(a.Moved), len(a.Sets)) + case a.Switched: + fmt.Fprintf(&b, "%d layer(s) moved into mesh-config, each kept whole in its history first; the controller "+ + "reads mesh-config where it read them\n", len(a.Moved)) + } + return b.String() +} diff --git a/cmd/mesh-controller/config_moved_test.go b/cmd/mesh-controller/config_moved_test.go new file mode 100644 index 00000000..75657f8b --- /dev/null +++ b/cmd/mesh-controller/config_moved_test.go @@ -0,0 +1,785 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + snapshot "github.com/novox/mesh-controller/internal/facts" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/overlay" +) + +// configuredMesh is every live settings layer (issue 664) — read with the settings verb on 2026-10-15, on the four +// machines as anchor, laptop, workstation and desktop, the whole mesh's with no node; personal values stood in for, +// and the placement words left out, which move-placement moves first — with each module in the config form shaped +// as its definition on the feature branch: its config, its files (templates inlined) and directories, placeholders +// of kinds this test's mesh does not hold stood in for. A module still in the settings form (the media catalogue's, +// and kernel) stands in as a mergeable file that takes every key. `on` is where each is assigned, as live. +type configuredMesh struct { + Modules []json.RawMessage `json:"modules"` + Layers []configuredLayer `json:"layers"` + On map[string][]string `json:"on"` + // Predefined is the catalogue's list of predefined keys, which a module falls back to. + Predefined json.RawMessage `json:"predefined"` +} + +type configuredLayer struct { + Node string `json:"node"` + Module string `json:"module"` + Values map[string]any `json:"values"` +} + +var fourMachines = []string{"anchor", "laptop", "workstation", "desktop"} + +// aConfiguredMesh is aMesh with two machines more, every module of the configured mesh registered and assigned, and +// its layers held as the store holds them once move-placement has run. +func aConfiguredMesh(t *testing.T) (*stores, configuredMesh, map[string]map[string]any) { + t.Helper() + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + for i, name := range []string{"workstation", "desktop"} { + record, err := inv.AddNode(ctx, name) + if err != nil { + t.Fatal(err) + } + if err := inv.SetPlace(ctx, name, name+".example:51820", "here", false, fmt.Sprintf("10.77.0.%d", i+3)); err != nil { + t.Fatal(err) + } + if err := inv.RecordProfile(ctx, record.ID, map[string]any{"capabilities": []any{ + map[string]any{"name": "container-runtime", "present": true}, + map[string]any{"name": "systemd", "present": true}}}); err != nil { + t.Fatal(err) + } + if err := inv.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil { + t.Fatal(err) + } + if err := inv.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil { + t.Fatal(err) + } + if _, err := inv.Assign(ctx, name, overlay.Name); err != nil { + t.Fatal(err) + } + } + raw, err := os.ReadFile(filepath.Join("testdata", "config-moved", "mesh.json")) + if err != nil { + t.Fatal(err) + } + var w configuredMesh + if err := json.Unmarshal(raw, &w); err != nil { + t.Fatal(err) + } + list, err := catalogue.ParsePredefined(w.Predefined) + if err != nil { + t.Fatal(err) + } + catalogue.UsePredefined(list) + served := map[string]map[string]any{} + for _, text := range w.Modules { + var shape map[string]json.RawMessage + if err := json.Unmarshal(text, &shape); err != nil { + t.Fatal(err) + } + var name string + _ = json.Unmarshal(shape["module"], &name) + if s, ok := shape["serves-checked"]; ok { + var serves map[string]any + if err := json.Unmarshal(s, &serves); err != nil { + t.Fatal(err) + } + served[name] = serves + delete(shape, "serves-checked") + } + text, err := json.Marshal(shape) + if err != nil { + t.Fatal(err) + } + m, err := catalogue.ParseManifest(text) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + register(t, open, m) + } + for _, l := range w.Layers { + values, err := json.Marshal(l.Values) + if err != nil { + t.Fatal(err) + } + if l.Node == "" { + inv.ExecForTest(t, `insert into settings (node, module, values) values (null, $1, $2)`, l.Module, string(values)) + continue + } + inv.ExecForTest(t, `insert into settings (node, module, values) select id, $2, $3 from node where name = $1`, + l.Node, l.Module, string(values)) + } + for _, module := range movedKeys(w.On) { + for _, node := range w.On[module] { + if _, err := assign(ctx, open, node, module); err != nil { + t.Fatalf("assign %s %s: %v", node, module, err) + } + } + } + return open, w, served +} + +// standInMeshConfig is mesh-config as this test holds it: each value at its level, and resolved from the catalogue +// the store holds as mesh-config resolves (novox/hq ADR 0348 §2): the module's own value on the node, else node +// config then mesh config for the predefined key it falls back to, else its default; a required value nothing sets is +// unset. +type standInMeshConfig struct { + open *stores + values map[string]any + sets []string + // asked is each call of its verb: the action and the level's key. + asked []string +} + +func (c *standInMeshConfig) Values(context.Context) (map[string]json.RawMessage, error) { + out := map[string]json.RawMessage{} + for k, v := range c.values { + raw, err := json.Marshal(v) + if err != nil { + return nil, err + } + out[k] = raw + } + return out, nil +} + +func (c *standInMeshConfig) Set(_ context.Context, level configLevel, key string, value any, by, why string) error { + if by == "" || why == "" { + return fmt.Errorf("%s at %s is set without who or why", key, level) + } + c.values[level.valueKey(key)] = normalJSON(value) + c.sets = append(c.sets, level.valueKey(key)) + return nil +} + +func (c *standInMeshConfig) Resolved(ctx context.Context) ([]resolvedEntry, error) { + entries, err := c.open.inventory.Catalogued(ctx) + if err != nil { + return nil, err + } + var out []resolvedEntry + for _, e := range entries { + m := e.Manifest + if !m.DeclaresConfig() { + continue + } + for _, node := range e.On { + for _, key := range movedKeys(m.Settings) { + d := m.Settings[key] + r := resolvedEntry{Node: node, Module: m.Module, Key: key, Source: "default"} + value, has := c.values["module."+node+"."+m.Module+"."+key] + switch { + case has: + r.Source = "module" + case d.Predefined != "" && c.values["node."+node+"."+d.Predefined] != nil: + value, r.Source = c.values["node."+node+"."+d.Predefined], "node" + case d.Predefined != "" && c.values["mesh."+d.Predefined] != nil: + value, r.Source = c.values["mesh."+d.Predefined], "mesh" + case d.Required: + r.Source = "unset" + default: + value = d.Default + } + if r.Source != "unset" { + if r.Value, err = json.Marshal(value); err != nil { + return nil, err + } + } + out = append(out, r) + } + } + } + return out, nil +} + +// laid is the stand-in as composition reads it once the switch is on (inventory.ConfigValues). +func (c *standInMeshConfig) laid(ctx context.Context, node, module string) (map[string]any, error) { + resolved, err := c.Resolved(ctx) + if err != nil { + return nil, err + } + return configLaid(resolved, node, module) +} + +// snapshotFacts is the facts snapshot as gatherFacts gives it. +type snapshotFacts = snapshot.Facts + +// toldEveryMachine is what each of the four machines is told — every resource and the filter's rules — and what +// each module that serves something serves, settled from its layers, as JSON a person can diff. +func toldEveryMachine(t *testing.T, open *stores, served map[string]map[string]any, on map[string][]string) map[string]string { + t.Helper() + ctx := t.Context() + gens, err := generators(ctx, open) + if err != nil { + t.Fatal(err) + } + shelf, err := open.inventory.Catalogue(ctx) + if err != nil { + t.Fatal(err) + } + told := map[string]string{} + for _, node := range fourMachines { + plan, settings, err := planFor(ctx, open, node) + if err != nil { + t.Fatalf("%s: %v", node, err) + } + with, _, err := renderingFor(ctx, open, node, plan, settings, gens, Allocating) + if err != nil { + t.Fatalf("%s: %v", node, err) + } + composed, err := plan.Compose(with) + if err != nil { + t.Fatalf("%s: %v", node, err) + } + rules, err := plan.Rules(with) + if err != nil { + t.Fatalf("%s: %v", node, err) + } + settles := map[string]any{} + for module, serves := range served { + if !containsString(on[module], node) { + continue + } + layers, err := open.inventory.SettingsFor(ctx, node, module) + if err != nil { + t.Fatal(err) + } + for name, s := range serves { + fact, _ := s.(map[string]any) + settled, err := catalogue.Settle(fact, catalogue.WithDefaults(shelf[module], layers)) + if err != nil { + settles[module+" serves "+name] = "error: " + err.Error() + continue + } + settles[module+" serves "+name] = settled + } + } + got, err := json.MarshalIndent(map[string]any{"resources": composed.Resources, "left-out": composed.LeftOut, + "rules": rules, "served": settles}, "", " ") + if err != nil { + t.Fatal(err) + } + told[node] = string(got) + } + return told +} + +// effectiveEverywhere is each module's every value on each machine it is assigned to, as `settings show` and +// `settings preferences` say them (catalogue.Effective): the value, never where it came from, which the move +// changes by design; a key a layer says by its file's name is read as its config key. +func effectiveEverywhere(t *testing.T, open *stores, w configuredMesh) map[string]any { + t.Helper() + ctx := t.Context() + shelf, err := open.inventory.Catalogue(ctx) + if err != nil { + t.Fatal(err) + } + out := map[string]any{} + for _, module := range movedKeys(w.On) { + if !shelf[module].DeclaresConfig() { + continue + } + for _, node := range w.On[module] { + layers, err := open.inventory.SettingsFor(ctx, node, module) + if err != nil { + t.Fatal(err) + } + // A layer's old spelling of a key (nodered's timeZone) is listed beside the key it is the file's name + // for, and reaches the file as that key does (issue 751): read as that key, which the move sets. + as := catalogue.TakenAs(shelf[module]) + for _, v := range catalogue.Effective(shelf[module], layers) { + if key, old := as[v.Key]; old { + out[node+"."+module+"."+key] = normalJSON(v.Value) + continue + } + if _, said := out[node+"."+module+"."+v.Key]; said && v.FromDefault { + continue + } + out[node+"."+module+"."+v.Key] = normalJSON(v.Value) + } + } + } + return out +} + +func setAt(a configMoveAnswer, key string) []configSet { + var out []configSet + for _, s := range a.Sets { + if s.valueKey(s.Key) == key { + out = append(out, s) + } + } + return out +} + +// **The settings layers move into mesh-config and no machine is told anything new** (novox/hq ADR 0348 §9, design +// 57 K5, issue 664). Every live layer is seeded as the store holds it once move-placement has run, on modules shaped +// as their definitions, and every machine composed. The move, asked without confirming, says each value it would set +// and where from, and sets nothing; confirmed, it sets them in mesh-config, reads them back, empties the layers it +// moved and switches. Each machine is then told byte for byte what it was told before, every module serves the same, +// and `settings` lists the same values `config` resolves. Asked again, it sets nothing. +func TestEveryMachineIsToldTheSameAfterTheMoveOfConfig(t *testing.T) { + open, w, served := aConfiguredMesh(t) + ctx := t.Context() + readBackTries = 0 + t.Cleanup(func() { readBackTries = 15 }) + inv := open.inventory + config := &standInMeshConfig{open: open, values: map[string]any{}} + inv.ReadsConfigFrom(config.laid) + before := toldEveryMachine(t, open, served, w.On) + listedBefore := effectiveEverywhere(t, open, w) + factsBefore, err := gatherFacts(ctx, open, "2.11.17") + if err != nil { + t.Fatal(err) + } + + asked, err := runConfigMove(ctx, inv, config, false, "the move of config, run by a test", "") + if err != nil || !asked.DryRun || asked.Switched || len(asked.Problems) > 0 || len(config.sets) > 0 { + t.Fatalf("the move, not confirmed, says %+v (%v) and set %v", asked, err, config.sets) + } + // nodered's live layer says the file's name for its time zone: set under the config key (issue 751). + if s := setAt(asked, "module.anchor.nodered.time-zone"); len(s) != 1 || s[0].Value != "Europe/Brussels" || + !strings.HasSuffix(s[0].From[0], ": timeZone") { + t.Errorf("nodered's time zone moves as %+v", s) + } + for _, s := range asked.Sets { + if s.Key == "timeZone" { + t.Errorf("a key is set under the file's name: %+v", s) + } + } + // claude-code's whole-mesh managed settings become mesh config for agent-managed-settings (ADR 0348's + // consequences), and its whole-mesh layer sets no module value. + if s := setAt(asked, "mesh.agent-managed-settings"); len(s) != 1 || s[0].From[0] != "claude-code on the whole mesh: managed_settings" { + t.Errorf("claude-code's managed settings move as %+v", s) + } + for _, node := range fourMachines { + if s := setAt(asked, "module."+node+".claude-code.managed_settings"); len(s) > 0 { + t.Errorf("claude-code's managed settings are also set on %s: %+v", node, s) + } + } + // A whole-mesh layer of a value no predefined key holds goes to each machine it is assigned to, under that + // machine's layer: power's on the workstation is the workstation's, its own elsewhere. + if s := setAt(asked, "module.workstation.power.handle-power-key"); len(s) != 1 || s[0].Value != "suspend" { + t.Errorf("power's key on the workstation moves as %+v", s) + } + if s := setAt(asked, "module.anchor.power.handle-power-key"); len(s) != 1 || s[0].Value != "poweroff" { + t.Errorf("power's key on the anchor moves as %+v", s) + } + for _, s := range asked.Sets { + if s.Level == "node" { + t.Errorf("a machine's layer became node config: %+v", s) + } + } + var stays []string + for _, s := range asked.Stays { + stays = append(stays, strings.Fields(s)[0]) + } + sort.Strings(stays) + if want := []string{"bazarr", "bazarr", "bookshelf", "jackett", "kernel", "kometa", "lidarr", "nzbget", "ombi", + "plex", "qbittorrent", "radarr", "sonarr", "tautulli"}; len(stays) < 10 || !containsString(stays, "kernel") { + t.Errorf("the layers that stay are %v, want those of %v", stays, want) + } + if after := toldEveryMachine(t, open, served, w.On); !reflect.DeepEqual(after, before) { + t.Fatal("a dry run changed what a machine is told") + } + + moved, err := runConfigMove(ctx, inv, config, true, "the move of config, run by a test", "release 57 is delivered") + if err != nil || moved.DryRun || !moved.Switched { + t.Fatalf("the move says %+v (%v)", moved, err) + } + if len(config.sets) != len(asked.Sets) { + t.Fatalf("the move set %d values, and said it would set %d", len(config.sets), len(asked.Sets)) + } + if held := inv.QueryForTest(t, `select module from settings s where values <> '{}' + and module in (select name from module where manifest ? 'config')`); len(held) != 0 { + t.Fatalf("a layer of a module in the config form still holds values: %v", held) + } + after := toldEveryMachine(t, open, served, w.On) + for _, node := range fourMachines { + if after[node] != before[node] { + t.Errorf("%s is told something else after the move:\n%s\nwas:\n%s", node, after[node], before[node]) + } + } + if listedAfter := effectiveEverywhere(t, open, w); !reflect.DeepEqual(listedAfter, listedBefore) { + t.Errorf("settings lists other values after the move:\n%v\nwas:\n%v", listedAfter, listedBefore) + } + // And config lists what settings listed: each module's resolved value on each machine is the value settings + // said before the move. + resolved, err := config.Resolved(ctx) + if err != nil { + t.Fatal(err) + } + for _, r := range resolved { + said, listed := listedBefore[r.Node+"."+r.Module+"."+r.Key] + if r.Source == "unset" { + if listed && said != nil { + t.Errorf("%s of %s on %s is unset in config, and settings said %v", r.Key, r.Module, r.Node, said) + } + continue + } + var v any + _ = json.Unmarshal(r.Value, &v) + if !listed || !reflect.DeepEqual(v, said) { + t.Errorf("config resolves %s of %s on %s as %v, and settings said %v", r.Key, r.Module, r.Node, v, said) + } + } + + // The facts snapshot carries mesh-config's values for the merge gate, as each machine's layer of each module, and + // each machine's declaration composes to the digest it did before the move. + factsAfter, err := gatherFacts(ctx, open, "2.11.17") + if err != nil { + t.Fatal(err) + } + // Machines are named in a snapshot by stand-ins of its own; each is found by what it composes. + digests := func(f snapshotFacts) ([]string, string) { + var out []string + zone := "" + for _, m := range f.Machines { + out = append(out, m.Declaration.Digest) + for _, l := range m.Settings { + if l.Module == "nodered" { + zone = compactJSON(l.Values["time-zone"]) + } + } + } + sort.Strings(out) + return out, zone + } + was, _ := digests(factsBefore) + now, zone := digests(factsAfter) + if zone != `"Europe/Brussels"` { + t.Errorf("the facts snapshot does not carry nodered's time zone from mesh-config: %s", zone) + } + if len(now) != len(fourMachines) || now[0] == "" || !reflect.DeepEqual(now, was) { + t.Errorf("the declarations in the facts snapshot are %v after the move, %v before", now, was) + } + for _, s := range factsAfter.Settings { + if _, isConfig := map[string]bool{"betterbird": true, "claude-code": true, "power": true, "xdg": true}[s.Module]; isConfig && + len(s.Values) > 0 { + t.Errorf("the facts snapshot still carries the whole mesh's layer of %s: %v", s.Module, s.Values) + } + } + + config.sets = nil + again, err := runConfigMove(ctx, inv, config, true, "the move of config, run by a test", "again") + if err != nil || len(config.sets) != 0 || len(again.Moved) != 0 { + t.Fatalf("asked again, the move set %v and says %+v (%v)", config.sets, again, err) + } + if after := toldEveryMachine(t, open, served, w.On); !reflect.DeepEqual(after, before) { + t.Fatal("the move asked again changed what a machine is told") + } +} + +// **The move refuses what it cannot move as it composes, and sets nothing** (issue 664): a layer that still holds +// placement (move-placement has not run), a key its module does not declare, and a value its declaration refuses — +// each said by name; and a value mesh-config resolves from a level no layer says (it would be laid once the switch +// is on) leaves every layer and the switch as they were. +func TestTheMoveOfConfigRefusesWhatWouldChangeAMachine(t *testing.T) { + open, _, _ := aConfiguredMesh(t) + ctx := t.Context() + readBackTries = 0 + t.Cleanup(func() { readBackTries = 15 }) + inv := open.inventory + config := &standInMeshConfig{open: open, values: map[string]any{}} + inv.ReadsConfigFrom(config.laid) + + inv.ExecForTest(t, `update settings set values = values || '{"endpoints": {"web": {"reach": "public"}}}' + where module = 'nodered'`) + inv.ExecForTest(t, `update settings set values = values || '{"colour": "blue"}' where module = 'unifi'`) + inv.ExecForTest(t, `update settings set values = '{"font-size": 900}' where module = 'dunst'`) + answer, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered") + if err == nil || answer.Switched || len(config.sets) > 0 { + t.Fatalf("the move went ahead: %+v (%v), set %v", answer, err, config.sets) + } + for _, want := range []string{"nodered on anchor still holds placement (endpoints): run move-placement first", + `unifi on anchor says "colour", which unifi does not declare`, "font-size at dunst's own value on workstation"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the refusal does not say %q: %v", want, err) + } + } + inv.ExecForTest(t, `update settings set values = values - 'endpoints' where module = 'nodered'`) + inv.ExecForTest(t, `update settings set values = values - 'colour' where module = 'unifi'`) + inv.ExecForTest(t, `update settings set values = '{"font-size": 13, "width": 500}' where module = 'dunst'`) + + // Set in mesh-config before the move, on a module whose layers say nothing of it: the switch would lay it. + config.values["module.anchor.grafana."+movedKeys(func() map[string]catalogue.SettingDeclaration { + shelf, _ := inv.Catalogue(ctx) + return shelf["grafana"].Settings + }())[0]] = "set before the move" + answer, err = runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered") + if err == nil || !strings.Contains(err.Error(), "grafana on anchor resolves") || answer.Switched { + t.Fatalf("a value no layer says was laid by the switch: %+v (%v)", answer, err) + } + if moved, _ := inv.ConfigMoved(ctx); moved.Moved { + t.Fatal("the switch is on after a refused move") + } + if held := inv.QueryForTest(t, `select count(*) from settings where values <> '{}'`); held[0][0] == "0" { + t.Fatal("a refused move emptied the layers") + } +} + +// The verb's command line: a dry run unless confirmed, and a confirmed move says why, recorded as a person's step of +// the release, never a repair (issue 664). +func TestTheMoveOfConfigIsADryRunUnlessConfirmed(t *testing.T) { + for _, c := range []struct { + args map[string]any + want string + }{ + {map[string]any{}, "move-config --json"}, + {map[string]any{"confirm": "true", "why": "release 57 is delivered"}, "move-config --json --confirm --why release 57 is delivered"}, + } { + argv, err := argvFor("move-config", c.args) + if err != nil || strings.Join(argv, " ") != c.want { + t.Errorf("%v: %q %v, want %q", c.args, argv, err, c.want) + } + } + for _, args := range []map[string]any{{"confirm": "true"}, {"why": "tidy"}, {"node": "anchor"}} { + if argv, err := argvFor("move-config", args); err == nil { + t.Errorf("%v was composed as %q", args, argv) + } + } + if repairingCommand([]string{"move-config", "--json", "--confirm", "--why", "x"}) != "move-config" { + t.Error("a confirmed move through the generic verb would go unrecorded") + } + if !personsDecision(link.HandAct{Verb: "move-config"}) { + t.Error("a move of the release would count toward a healer the mesh lacks") + } +} + +// ask is mesh-config's verb as the stand-in answers it: list, set, clear and propose. +func (c *standInMeshConfig) ask(ctx context.Context, args map[string]any) (json.RawMessage, error) { + str := func(k string) string { s, _ := args[k].(string); return s } + level := configLevel{Level: "mesh"} + switch { + case str("module") != "" && str("node") != "": + level = configLevel{Level: "module", Node: str("node"), Module: str("module")} + case str("node") != "": + level = configLevel{Level: "node", Node: str("node")} + } + c.asked = append(c.asked, str("action")+" "+level.valueKey(str("key"))) + switch str("action") { + case "list": + resolved, err := c.Resolved(ctx) + if err != nil { + return nil, err + } + out := map[string]any{"resolved": []resolvedEntry{}, "set": []any{}} + var listed []resolvedEntry + for _, r := range resolved { + if r.Module == str("module") && (str("node") == "" || r.Node == str("node")) { + listed = append(listed, r) + } + } + out["resolved"] = listed + var set []any + for _, k := range movedKeys(c.values) { + parts := strings.SplitN(k, ".", 4) + if len(parts) == 4 && parts[0] == "module" && parts[2] == str("module") { + set = append(set, map[string]any{"level": configLevel{Level: "module", Node: parts[1], Module: parts[2]}, + "key": parts[3], "value": c.values[k], "by": "a test"}) + } + } + out["set"] = set + return json.Marshal(out) + case "set": + if str("why") == "" || str("by") == "" { + return nil, fmt.Errorf("a set without why or by: %v", args) + } + return nil, c.Set(ctx, level, str("key"), args["value"], str("by"), str("why")) + case "clear": + if _, had := c.values[level.valueKey(str("key"))]; !had { + return nil, fmt.Errorf("nothing is set for %s", str("key")) + } + delete(c.values, level.valueKey(str("key"))) + return json.RawMessage(`{}`), nil + case "propose": + return json.RawMessage(`{"proposal":"p-1"}`), nil + } + return nil, fmt.Errorf("the stand-in does not %s", str("action")) +} + +// **`settings` answers by forwarding to `config`** (novox/hq ADR 0348 §1, design 57 K5, issue 664): once the move +// has run, a module in the config form is shown from mesh-config, its values set and cleared there as its own on the +// machine (placement still beside the assignment), and proposed through it; a trusted value is refused through a +// verb, a module value for the whole mesh is refused, and a module still in the settings form keeps today's verb. +func TestTheSettingsVerbForwardsToConfigOnceTheValuesMoved(t *testing.T) { + open, w, served := aConfiguredMesh(t) + ctx := t.Context() + inv := open.inventory + readBackTries = 0 + t.Cleanup(func() { readBackTries = 15 }) + config := &standInMeshConfig{open: open, values: map[string]any{}} + inv.ReadsConfigFrom(config.laid) + savedRead, savedAsk := readConfigValues, configAsk + readConfigValues, configAsk = config.laid, config.ask + t.Cleanup(func() { readConfigValues, configAsk = savedRead, savedAsk }) + if _, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil { + t.Fatal(err) + } + + shown := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"show", "nodered", "--node", "anchor"}) }) + for _, want := range []string{`"time-zone": "Europe/Brussels"`, `time-zone = "Europe/Brussels" (module)`, + "mesh-config.config", `"mqtt": {`} { + if !strings.Contains(shown, want) { + t.Errorf("settings show does not say %q:\n%s", want, shown) + } + } + if strings.Contains(shown, "timeZone") { + t.Errorf("settings show still says the file's name:\n%s", shown) + } + + config.asked = nil + said := stdoutOf(t, func() error { + return settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12, "width": 500}`, "--node", "workstation"}) + }) + if config.values["module.workstation.dunst.font-size"] != float64(12) || !strings.Contains(said, "~ font-size") || + containsString(config.asked, "set module.workstation.dunst.width") { + t.Fatalf("settings set reached mesh-config as %v (%v):\n%s", config.values, config.asked, said) + } + told := toldEveryMachine(t, open, served, w.On)["workstation"] + if !strings.Contains(told, "12") { + t.Error("the workstation is not told the font size set through settings") + } + if err := settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12}`, "--node", "workstation"}); err == nil || + !strings.Contains(err.Error(), "would no longer set width") { + t.Fatalf("a set dropping a value without --replace: %v", err) + } + if err := settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12}`}); err == nil || + !strings.Contains(err.Error(), "no own value for the whole mesh") { + t.Fatalf("a module value for the whole mesh: %v", err) + } + + t.Setenv(verbVar, "settings") + for _, c := range []struct{ module, node, values, key, held, why string }{ + // A file root reads: refused as before the move (novox/hq issue 339). + {"hostname", "anchor", `{"hostname": "elsewhere"}`, "module.anchor.hostname.hostname", "Ace", "issue 339"}, + {"messenger", "laptop", `{"desktop-machines": ["g14", "shanks"], "time-zone": "UTC"}`, + "module.laptop.messenger.time-zone", "Europe/Brussels", "issue 339"}, + } { + err := settingsCommand(ctx, []string{"set", c.module, c.values, "--node", c.node}) + if err == nil || !strings.Contains(err.Error(), c.why) || config.values[c.key] != c.held { + t.Fatalf("a trusted value of %s through a verb: %v, holds %v", c.module, err, config.values[c.key]) + } + } + // And a trusted value no file makes the terminal's is refused as trusted, since mesh-config takes the controller's + // word for it (ADR 0348 §8). + if err := refuseTrustedThroughAVerb(ctx, inv, "messenger", "laptop", []string{"time-zone"}); err == nil || + !strings.Contains(err.Error(), "ADR 0348 §8") { + t.Fatalf("a trusted value through a verb: %v", err) + } + if err := refuseTrustedThroughAVerb(ctx, inv, "dunst", "workstation", []string{"width"}); err != nil { + t.Fatalf("a value nobody trusts through a verb: %v", err) + } + stdoutOf(t, func() error { + return settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12, "width": 600}`, "--node", "workstation"}) + }) + if config.values["module.workstation.dunst.width"] != float64(600) { + t.Fatalf("a value nobody trusts was not set through a verb: %v", config.values["module.workstation.dunst.width"]) + } + t.Setenv(verbVar, "") + + stdoutOf(t, func() error { return settingsCommand(ctx, []string{"clear", "dunst", "--node", "workstation"}) }) + for _, k := range []string{"font-size", "width"} { + if _, held := config.values["module.workstation.dunst."+k]; held { + t.Errorf("settings clear left dunst's %s on the workstation", k) + } + } + + config.asked = nil + proposed := stdoutOf(t, func() error { + return settingsCommand(ctx, []string{"propose", "hostname", `{"hostname": "elsewhere"}`, "--node", "anchor"}) + }) + if !containsString(config.asked, "propose module.anchor.hostname.hostname") || !strings.Contains(proposed, "p-1") { + t.Fatalf("settings propose reached mesh-config as %v:\n%s", config.asked, proposed) + } + + config.asked = nil + kept := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"show", "kernel", "--node", "laptop"}) }) + if !strings.Contains(kept, "transparent-huge-pages") || len(config.asked) > 0 { + t.Fatalf("a module in the settings form is shown as %q, asking mesh-config %v", kept, config.asked) + } +} + +// **An approval for a module whose config has moved is refused, and writes nothing** (issue 830, novox/hq ADR 0348): +// a proposal raised while grafana read its layer (it had none, so the layer is as the operator was shown it) and +// approved after the move would put its values back into the layer, ReadsConfig would turn false, and the module +// would compose from emptied layers everywhere. +func TestAnApprovalRaisedBeforeTheMoveOfConfigIsRefusedAfterIt(t *testing.T) { + open, _, _ := aConfiguredMesh(t) + ctx := t.Context() + readBackTries = 0 + t.Cleanup(func() { readBackTries = 15 }) + inv := open.inventory + config := &standInMeshConfig{open: open, values: map[string]any{}} + inv.ReadsConfigFrom(config.laid) + shelf, err := inv.Catalogue(ctx) + if err != nil { + t.Fatal(err) + } + key := movedKeys(func() map[string]catalogue.SettingDeclaration { return shelf["grafana"].Settings }())[0] + values := map[string]any{key: "raised before the move"} + p := settingsProposal{Module: "grafana", Node: "anchor", Values: values, From: "g14/claude-code", At: time.Now(), + Digest: layerDigest(values), BeforeDigest: layerDigest(nil)} + if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has { + t.Fatalf("grafana has a layer on anchor before the test: %v %v", has, err) + } + if moved, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil || !moved.Switched { + t.Fatalf("the move says %+v (%v)", moved, err) + } + if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads { + t.Fatalf("grafana does not read config after the move: %v %v", reads, err) + } + _, err = setLayerIn(open)(ctx, p, "s30", "approved by the operator") + if err == nil || !strings.Contains(err.Error(), "moved to mesh-config since the operator was shown the change") || + !strings.Contains(err.Error(), "propose it again") { + t.Fatalf("the stale approval: %v", err) + } + if layer, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has && len(layer) > 0 { + t.Errorf("the refused approval wrote the layer: %v %v %v", layer, has, err) + } + if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads { + t.Errorf("the refused approval turned ReadsConfig off: %v %v", reads, err) + } +} + +// **A proposal of placement only is set after the move, as before** (issue 830): forwardPropose sends it down the +// old path on purpose, so refusing it would loop the operator (refused, proposed again, refused). +func TestAPlacementOnlyApprovalRaisedBeforeTheMoveIsSetAfterIt(t *testing.T) { + open, _, _ := aConfiguredMesh(t) + ctx := t.Context() + readBackTries = 0 + t.Cleanup(func() { readBackTries = 15 }) + inv := open.inventory + config := &standInMeshConfig{open: open, values: map[string]any{}} + inv.ReadsConfigFrom(config.laid) + values := map[string]any{catalogue.PlacesSetting: map[string]any{"data": "/srv/grafana-data"}} + p := settingsProposal{Module: "grafana", Node: "anchor", Values: values, From: "g14/claude-code", At: time.Now(), + Digest: layerDigest(values), BeforeDigest: layerDigest(nil)} + if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has { + t.Fatalf("grafana has a layer on anchor before the test: %v %v", has, err) + } + if moved, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil || !moved.Switched { + t.Fatalf("the move says %+v (%v)", moved, err) + } + if _, err := setLayerIn(open)(ctx, p, "s31", "approved by the operator"); err != nil { + t.Fatalf("the placement-only approval after the move: %v", err) + } + if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || !has { + t.Errorf("the approval did not set the layer: %v %v", has, err) + } + if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads { + t.Errorf("the approval turned ReadsConfig off: %v %v", reads, err) + } +} diff --git a/cmd/mesh-controller/config_read.go b/cmd/mesh-controller/config_read.go new file mode 100644 index 00000000..39dbc725 --- /dev/null +++ b/cmd/mesh-controller/config_read.go @@ -0,0 +1,128 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/inventory" +) + +// **Where the controller reads config values once the move has run** (novox/hq ADR 0348 §9, design 57 K5, issue 664): +// mesh-config_resolved, the bucket mesh-config alone writes, keyed `..`, each value with the +// source it came from. Read only for a module the switch reaches (inventory.ReadsConfig), so a controller before the +// move never asks the bus for config. The exception to ADR 0348 §9 ("the controller holds no config value") that +// ADR 0349 clause 17's old way needs, until design 57's R1 hands every module to the provisioner. + +// resolvedEntry is one key of mesh-config_resolved as the controller reads it: the value and where it came from. +type resolvedEntry struct { + Node string `json:"node"` + Module string `json:"module"` + Key string `json:"key"` + Value json.RawMessage `json:"value,omitempty"` + Source string `json:"source"` +} + +// The sources a resolved value is laid over a module's files from: mesh config, node config and the module's own +// value. Its manifest's default is composition's already, and a taken key's default is never laid over its file; +// a required value nothing sets (`unset`) is refused by composition by name, as it is today. +var laidSources = map[string]bool{"mesh": true, "node": true, "module": true} + +// configLaid is the values of one module on one node that are laid over its files, from the resolved entries. +func configLaid(entries []resolvedEntry, node, module string) (map[string]any, error) { + out := map[string]any{} + for _, e := range entries { + if e.Node != node || e.Module != module || !laidSources[e.Source] { + continue + } + var v any + if err := json.Unmarshal(e.Value, &v); err != nil { + return nil, fmt.Errorf("mesh-config's value %s of %s on %s cannot be read: %w", e.Key, module, node, err) + } + out[e.Key] = v + } + return out, nil +} + +// resolvedRead is what one read of the whole bucket held, and when it was read. +type resolvedRead struct { + mu sync.Mutex + at time.Time + entries []resolvedEntry +} + +// resolvedFresh is how long one read of the bucket answers: a plan reads every module of a machine, and one read +// serves them all; a value set since is read by the next composition. +const resolvedFresh = 5 * time.Second + +var theResolved resolvedRead + +// readConfigValues is where an opened inventory reads config values once the switch is on; a test replaces it. +var readConfigValues inventory.ConfigValues = busConfigValues + +// busConfigValues reads one module's config values on one node from mesh-config_resolved (inventory.ConfigValues). +func busConfigValues(ctx context.Context, node, module string) (map[string]any, error) { + theResolved.mu.Lock() + defer theResolved.mu.Unlock() + if theResolved.entries == nil || time.Since(theResolved.at) > resolvedFresh { + entries, err := readResolved(ctx) + if err != nil { + return nil, err + } + theResolved.entries, theResolved.at = entries, time.Now() + } + return configLaid(theResolved.entries, node, module) +} + +// readResolved is every entry of mesh-config_resolved, read on the bus. +func readResolved(ctx context.Context) ([]resolvedEntry, error) { + entries := []resolvedEntry{} + err := onTheBus(func(conn *nats.Conn) error { + js, err := jetstream.New(conn) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(ctx, 20*time.Second) + defer cancel() + kv, err := js.KeyValue(ctx, broker.ConfigResolvedBucket) + if err != nil { + return fmt.Errorf("%s cannot be opened: %w", broker.ConfigResolvedBucket, err) + } + w, err := kv.WatchAll(ctx, jetstream.IgnoreDeletes()) + if err != nil { + return err + } + defer func() { _ = w.Stop() }() + for { + select { + case <-ctx.Done(): + return fmt.Errorf("%s was not read whole: %w", broker.ConfigResolvedBucket, ctx.Err()) + case e, open := <-w.Updates(): + if !open { + return errors.New(broker.ConfigResolvedBucket + " stopped answering") + } + if e == nil { + return nil + } + var r resolvedEntry + if err := json.Unmarshal(e.Value(), &r); err != nil { + return fmt.Errorf("%s holds %q, which is not a resolved value: %w", broker.ConfigResolvedBucket, + e.Key(), err) + } + if r.Node == "" || r.Module == "" || r.Key == "" || !strings.HasPrefix(e.Key(), r.Node+"."+r.Module+".") { + return fmt.Errorf("%s holds %q, whose value names another key", broker.ConfigResolvedBucket, e.Key()) + } + entries = append(entries, r) + } + } + }) + return entries, err +} diff --git a/cmd/mesh-controller/core_box.go b/cmd/mesh-controller/core_box.go new file mode 100644 index 00000000..54f641e6 --- /dev/null +++ b/cmd/mesh-controller/core_box.go @@ -0,0 +1,297 @@ +package main + +import ( + "context" + "crypto/ecdh" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "sort" + "strings" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// The core's own credentials, sealed by the vault (novox/hq ADR 0349 §16, issue 755; the lead's decision on issue 755, +// notes 4 and 5). +// +// For a machine whose engine opens the vault's box (link.VaultBoxContract), each own secret of a module on the core's +// list (catalogue.CoreCredentialModules) travels as the vault sealed it to that machine's key, which the controller +// asks for on the send path (`get`, answered to the controller alone and sealed only to the key that machine's engine +// stated: mesh-vault's core.go) and carries without being able to open it. A machine below that contract is sent the +// controller's own sealed value, exactly as before. +// +// **The vault seals every answer afresh**, so the box last got is kept (inventory.CoreBox) and carried again while the +// vault's generation and the machine's key are the same: a declaration that changed with every answer would never be +// current. Every question but a send reads the kept box and asks nothing. +// +// **When the vault does not answer, the controller never falls back to its own seal.** A credential with a kept box +// keeps it — the machine keeps the file it has — and one with none leaves its module out of the declaration (the +// machine keeps what it holds of it, and a module not there yet is not placed). Either way the send raises a +// condition naming the vault and the machine, cleared by the first send the vault answers whole. + +// engineOpensVaultBox says whether a machine's node-engine opens the vault's box, by its own newest statement. +func engineOpensVaultBox(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) { + stated, had, err := inv.HealthOf(ctx, node) + if err != nil { + return false, err + } + return had && stated.Contract >= link.VaultBoxContract, nil +} + +// coreAnswer is the vault's `get` answer as the controller reads it: the name, the generation and the box, never +// opened. +type coreAnswer struct { + Name string `json:"name"` + Generation int `json:"generation"` + Sealed json.RawMessage `json:"sealed"` +} + +// coreAsk asks the vault for one secret sealed to the given key. +type coreAsk func(name, publicKey string) (coreAnswer, error) + +// coreBoxes is what the vault's boxes make of one machine's composition. +type coreBoxes struct { + // sealed is each core credential's box, by module and name (catalogue.Rendering.VaultSealed). + sealed map[string]map[string]json.RawMessage + // notPlaced is each core module left out, with why (catalogue.Rendering.NotPlaced). + notPlaced map[string]string + // keep is each box newly answered, to keep. + keep []inventory.CoreBox + // held is each credential the vault did not answer for whose kept box is carried again; silent is why the vault + // did not answer, per credential. + held, silent []string +} + +// coreModulesOf is every module of the plan on the core's list whose own secrets this machine's declaration carries, +// with their plain own secret names, sorted: one handed to the provisioner has its secrets from the vault through the +// client, and one issued outside the mesh is never the vault's to seal here. +func coreModulesOf(modules []catalogue.Manifest, handed func(catalogue.Manifest) bool) map[string][]string { + out := map[string][]string{} + for _, m := range modules { + if !catalogue.CarriesVaultSealed(m.Module) || (handed != nil && handed(m)) { + continue + } + var names []string + for name, own := range m.OwnSecrets.Plain() { + if own.IssuedBy != catalogue.IssuedOutside { + names = append(names, name) + } + } + if len(names) > 0 { + sort.Strings(names) + out[m.Module] = names + } + } + return out +} + +// sealingKeyPEM is a machine's sealing key, recorded raw in base64, as the SPKI PEM the vault seals to. +func sealingKeyPEM(raw string) (string, error) { + b, err := base64.StdEncoding.DecodeString(strings.TrimSpace(raw)) + if err != nil || len(b) != 32 { + return "", errors.New("the machine's sealing key is not a raw X25519 key") + } + pub, err := ecdh.X25519().NewPublicKey(b) + if err != nil { + return "", err + } + der, err := x509.MarshalPKIXPublicKey(pub) + if err != nil { + return "", err + } + return string(pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: der})), nil +} + +// coreBoxesOf decides each core credential of a machine: on a send, the vault's answer (the kept box while its +// generation and key are the same), the kept box where the vault did not answer, or the module left out where nothing +// was kept; on any other question, the kept box, or a stand-in the next send asks for. Pure: what to keep and what to +// say is answered, not done. +func coreBoxesOf(node, sealingKey string, core map[string][]string, kept map[string]map[string]inventory.CoreBox, + choosing Choosing, ask coreAsk) coreBoxes { + out := coreBoxes{sealed: map[string]map[string]json.RawMessage{}, notPlaced: map[string]string{}} + modules := make([]string, 0, len(core)) + for m := range core { + modules = append(modules, m) + } + sort.Strings(modules) + for _, module := range modules { + if sealingKey == "" { + out.notPlaced[module] = fmt.Sprintf("%s has no sealing key, so the vault can seal %s's credentials to "+ + "nothing there (novox/hq ADR 0349 §16)", node, module) + continue + } + key, err := sealingKeyPEM(sealingKey) + if err != nil { + out.notPlaced[module] = fmt.Sprintf("%s's credentials cannot be asked of the vault: %v", module, err) + continue + } + boxes := map[string]json.RawMessage{} + for _, name := range core[module] { + k, has := kept[module][name] + valid := has && k.SealedTo == sealingKey && len(k.Box) > 0 + if choosing != Allocating { + if valid { + boxes[name] = k.Box + } else { + boxes[name] = catalogue.ForeseenVaultBox + } + continue + } + secret := "own/" + node + "/" + module + "/" + name + got, err := ask(secret, key) + if err == nil && (got.Generation <= 0 || canonicalBox(got.Sealed) == nil) { + err = errors.New("it answered no box") + } + if err != nil { + out.silent = append(out.silent, fmt.Sprintf("%s: %v", secret, err)) + if valid { + boxes[name] = k.Box + out.held = append(out.held, module+"/"+name) + continue + } + out.notPlaced[module] = fmt.Sprintf("the vault did not give %s's credential %s for %s (%v), and the "+ + "controller never seals it itself (novox/hq ADR 0349 §16): not placed until the vault answers", + module, name, node, err) + break + } + if valid && k.Generation == got.Generation { + boxes[name] = k.Box + continue + } + box := canonicalBox(got.Sealed) + boxes[name] = box + out.keep = append(out.keep, inventory.CoreBox{Module: module, Name: name, Generation: got.Generation, + SealedTo: sealingKey, Box: box}) + } + if _, left := out.notPlaced[module]; !left { + out.sealed[module] = boxes + } + } + return out +} + +// canonicalBox is the vault's box in the one form the declaration carries — the sealed box's fields, in their order, +// and nothing else — or nil when it is not one. +func canonicalBox(raw json.RawMessage) json.RawMessage { + var b struct { + V int `json:"v"` + Eph string `json:"eph"` + IV string `json:"iv"` + Tag string `json:"tag"` + Ct string `json:"ct"` + } + if err := json.Unmarshal(raw, &b); err != nil || b.V <= 0 || b.Eph == "" || b.Ct == "" { + return nil + } + out, _ := json.Marshal(b) + return out +} + +// coreVault is how a send asks the vault for a core credential: its `get`, in the controller's name. A test stands +// in for it. +var coreVault = func(ctx context.Context) coreAsk { + vault := theVault(ctx) + return func(name, publicKey string) (coreAnswer, error) { + raw, err := vault("get", map[string]any{"name": name, "public_key": publicKey}) + if err != nil { + return coreAnswer{}, err + } + var a coreAnswer + if err := json.Unmarshal(raw, &a); err != nil { + return coreAnswer{}, fmt.Errorf("the vault's answer is not one: %w", err) + } + return a, nil + } +} + +// coreBoxesFor is coreBoxesOf for one machine, read from and kept in the store, and said on a send. +func coreBoxesFor(ctx context.Context, inv *inventory.Inventory, node string, modules []catalogue.Manifest, + handed func(catalogue.Manifest) bool, choosing Choosing) (coreBoxes, error) { + core := coreModulesOf(modules, handed) + if len(core) == 0 { + if choosing == Allocating { + settleVaultSilent(ctx, node) + } + return coreBoxes{}, nil + } + key, err := inv.SealingKeyOf(ctx, node) + if err != nil { + return coreBoxes{}, err + } + kept, err := inv.KeptCoreBoxes(ctx, node) + if err != nil { + return coreBoxes{}, err + } + var ask coreAsk + if choosing == Allocating { + ask = coreVault(ctx) + } + out := coreBoxesOf(node, key, core, kept, choosing, ask) + for _, b := range out.keep { + if err := inv.KeepCoreBox(ctx, node, b); err != nil { + return coreBoxes{}, err + } + } + if choosing == Allocating { + sayVaultSilent(ctx, node, out) + } + return out, nil +} + +// kindVaultSilent is a send that did not have the vault's box for a core credential of a machine. +const kindVaultSilent = "vault-silent" + +// vaultSilentObservation is that, as a condition: `machine..vault-silent`. +func vaultSilentObservation(node string, b coreBoxes) conditions.Observation { + o := conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Kind: kindVaultSilent, Token: kindVaultSilent, + Machine: node, Severity: conditions.Warning, Source: coreDowngradeSource} + var said []string + if len(b.notPlaced) > 0 { + o.Severity = conditions.Urgent + var left []string + for m := range b.notPlaced { + left = append(left, m) + } + sort.Strings(left) + said = append(said, "not placed: "+strings.Join(left, ", ")) + } + if len(b.held) > 0 { + said = append(said, "kept as last sent: "+strings.Join(b.held, ", ")) + } + o.Summary = fmt.Sprintf("the vault did not answer for the core's credentials on %s; %s", node, strings.Join(said, "; ")) + o.Said = o.Summary + " — " + strings.Join(b.silent, "; ") + return o +} + +// sayVaultSilent raises the condition for a send the vault did not answer whole, and clears it for one it did. +func sayVaultSilent(ctx context.Context, node string, b coreBoxes) { + if len(b.silent) == 0 && len(b.notPlaced) == 0 { + settleVaultSilent(ctx, node) + return + } + fmt.Printf("HELD: %s\n", vaultSilentObservation(node, b).Said) + if conditionsFrom == nil { + return + } + if _, err := conditionsFrom.Observe(ctx, vaultSilentObservation(node, b)); err != nil { + fmt.Printf("the condition for the vault's silence on %s could not be raised: %v\n", node, err) + } +} + +// settleVaultSilent clears the condition once a send had every core credential of the machine from the vault. +func settleVaultSilent(ctx context.Context, node string) { + if conditionsFrom == nil { + return + } + key := vaultSilentObservation(node, coreBoxes{}).Key() + if _, err := conditionsFrom.Clear(ctx, key, "the vault answered for every core credential of "+node); err != nil { + fmt.Printf("the condition %s could not be cleared: %v\n", key, err) + } +} diff --git a/cmd/mesh-controller/core_box_test.go b/cmd/mesh-controller/core_box_test.go new file mode 100644 index 00000000..6a53ccac --- /dev/null +++ b/cmd/mesh-controller/core_box_test.go @@ -0,0 +1,220 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// The core's own credentials travel sealed by the vault to their machine (novox/hq ADR 0349 §16, issue 755; the lead's +// decision, issue 755 notes 4 and 5): to an engine that opens the vault's box, the send carries the box the vault +// answered the controller and never the controller's own seal; when the vault does not answer, a kept box is carried +// again and a module with none is not placed, with a condition naming the vault and the machine. An engine below the +// contract composes exactly as before, and the vault is not asked. + +// coreStandIn answers `get` with a box per name and generation, or its silence. +type coreStandIn struct { + asked []string + keys []string + generation int + silent error +} + +func (v *coreStandIn) ask(name, key string) (coreAnswer, error) { + v.asked, v.keys = append(v.asked, name), append(v.keys, key) + if v.silent != nil { + return coreAnswer{}, v.silent + } + box := `{"v":1,"eph":"ZQ==","iv":"aQ==","tag":"dA==","ct":"` + strings.Repeat("c", len(v.asked)) + `"}` + return coreAnswer{Name: name, Generation: v.generation, Sealed: json.RawMessage(box)}, nil +} + +func withVault(t *testing.T, v *coreStandIn) { + t.Helper() + before := coreVault + coreVault = func(context.Context) coreAsk { return v.ask } + t.Cleanup(func() { coreVault = before }) +} + +func aCoreMesh(t *testing.T, contract int) (*stores, *coreStandIn) { + t.Helper() + open := aMesh(t) + ctx := t.Context() + client := aClientModule() + register(t, open, client) + if _, err := assign(ctx, open, "laptop", client.Module); err != nil { + t.Fatal(err) + } + if err := issueProvisionerClient(ctx, open.inventory, client, "laptop", broker.Broker{Fingerprint: "ab"}, + "10.0.0.1:4222"); err != nil { + t.Fatal(err) + } + if _, err := open.inventory.RecordHealth(ctx, inventory.NodeHealth{Node: "laptop", Contract: contract, + SaidAt: time.Now()}); err != nil { + t.Fatal(err) + } + v := &coreStandIn{generation: 1} + withVault(t, v) + return open, v +} + +func clientsBroker(resources []map[string]any) map[string]any { + for _, r := range resources { + if r["id"] == "mesh-provisioner-client.needs-"+broker.ProvisionerSecret { + return r + } + } + return nil +} + +func vaultSilence(t *testing.T, node string) (conditions.Condition, bool) { + t.Helper() + c, found, err := conditionsFrom.Get(t.Context(), vaultSilentObservation(node, coreBoxes{}).Key()) + if err != nil { + t.Fatal(err) + } + return c, found +} + +func TestTheSendCarriesTheVaultsBoxForACoreCredentialAndKeepsIt(t *testing.T) { + open, vault := aCoreMesh(t, link.VaultBoxContract) + key, err := open.inventory.SealingKeyOf(t.Context(), "laptop") + if err != nil { + t.Fatal(err) + } + f := clientsBroker(sentTo(t, open, "laptop")) + if f == nil || f["sealed"] != nil || f[catalogue.VaultSealedField] == nil { + t.Fatalf("the core credential is sent as %v: the vault's box, never the controller's seal", f) + } + if want := []string{"own/laptop/mesh-provisioner-client/" + broker.ProvisionerSecret}; !reflect.DeepEqual(vault.asked, want) { + t.Fatalf("the vault was asked %v", vault.asked) + } + if want, _ := sealingKeyPEM(key); vault.keys[0] != want { + t.Fatalf("the vault was asked to seal to %q, not the machine's key", vault.keys[0]) + } + first, _ := json.Marshal(f[catalogue.VaultSealedField]) + + // The vault answers afresh with the same generation: the kept box is carried again, so the declaration is stable. + again, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField]) + if string(again) != string(first) { + t.Fatalf("a fresh box of the same generation changed the declaration: %s, then %s", first, again) + } + // A rotation in the vault (a newer generation) is carried. + vault.generation = 2 + rotated, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField]) + if string(rotated) == string(first) { + t.Fatal("a newer generation was not carried") + } + if _, found := vaultSilence(t, "laptop"); found { + t.Fatal("a send the vault answered raised its silence") + } +} + +func TestASilentVaultHoldsTheKeptBoxAndSaysSo(t *testing.T) { + open, vault := aCoreMesh(t, link.VaultBoxContract) + first, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField]) + vault.silent = errors.New("no responders") + held := clientsBroker(sentTo(t, open, "laptop")) + got, _ := json.Marshal(held[catalogue.VaultSealedField]) + if held == nil || held["sealed"] != nil || string(got) != string(first) { + t.Fatalf("a silent vault's credential is sent as %v: the box kept, never the controller's seal", held) + } + c, found := vaultSilence(t, "laptop") + if !found || !strings.Contains(c.Summary, "vault") || !strings.Contains(c.Summary, "laptop") { + t.Fatalf("no condition names the vault and the machine: %+v", c) + } + // The vault answers again: the condition clears. + vault.silent = nil + sentTo(t, open, "laptop") + if _, found := vaultSilence(t, "laptop"); found { + t.Fatal("the condition stayed after the vault answered") + } +} + +func TestACoreModuleFirstPlacedWhileTheVaultIsSilentIsNotPlaced(t *testing.T) { + open, vault := aCoreMesh(t, link.VaultBoxContract) + vault.silent = errors.New("no responders") + plan, settings, err := planFor(t.Context(), open, "laptop") + if err != nil { + t.Fatal(err) + } + declared, err := declarationFor(t.Context(), open, "laptop", plan, settings) + if err != nil { + t.Fatal(err) + } + for _, r := range declared.Resources { + if strings.HasPrefix(r["id"].(string), "mesh-provisioner-client.") { + t.Fatalf("a core module was placed though the vault gave nothing for it: %v", r) + } + } + if why := declared.leftOutWhy["mesh-provisioner-client"]; !strings.Contains(why, "never seals it itself") { + t.Fatalf("left out %v", declared.leftOutWhy) + } + c, found := vaultSilence(t, "laptop") + if !found || c.Severity != conditions.Urgent || !strings.Contains(c.Summary, "not placed") { + t.Fatalf("the condition does not say why: %+v", c) + } +} + +// An engine below VaultBoxContract is sent the controller's own seal, as before, and the vault is not asked. +func TestAnEngineBelowTheContractIsSentTheControllersSealAsBefore(t *testing.T) { + open, vault := aCoreMesh(t, link.HandedContract) + f := clientsBroker(sentTo(t, open, "laptop")) + if f == nil || f["sealed"] == nil || f[catalogue.VaultSealedField] != nil { + t.Fatalf("an older engine is sent %v", f) + } + if len(vault.asked) != 0 { + t.Fatalf("the vault was asked %v for an older engine", vault.asked) + } +} + +// Every question but a send reads the kept box and asks nothing; with none kept, a stand-in the next send asks for. +func TestOnlyASendAsksTheVault(t *testing.T) { + core := map[string][]string{"nats": {"broker"}} + key := aPublicKey(t) + kept := map[string]map[string]inventory.CoreBox{"nats": {"broker": {Module: "nats", Name: "broker", + Generation: 1, SealedTo: key, Box: json.RawMessage(`{"v":1}`)}}} + never := func(string, string) (coreAnswer, error) { + t.Fatal("the vault was asked by a question that is not a send") + return coreAnswer{}, nil + } + for _, choosing := range []Choosing{Reading, Foreseeing} { + if got := coreBoxesOf("anchor", key, core, kept, choosing, never); string(got.sealed["nats"]["broker"]) != `{"v":1}` { + t.Errorf("read %s", got.sealed["nats"]["broker"]) + } + if got := coreBoxesOf("anchor", key, core, nil, choosing, never); string(got.sealed["nats"]["broker"]) != + string(catalogue.ForeseenVaultBox) { + t.Errorf("with none kept, read %s", got.sealed["nats"]["broker"]) + } + } + // A box kept for another key of the machine is never carried. + other := coreBoxesOf("anchor", aPublicKey(t), core, kept, Allocating, + func(string, string) (coreAnswer, error) { return coreAnswer{}, errors.New("silent") }) + if _, left := other.notPlaced["nats"]; !left { + t.Fatalf("a box sealed to the machine's old key was carried: %+v", other) + } +} + +// The core's modules on a machine: on the list, not handed, with their own secrets but one issued outside. +func TestTheCoresModulesAreTheListsWithTheirOwnSecrets(t *testing.T) { + modules := []catalogue.Manifest{ + {Module: "nats", OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/a"}, "given": {Path: "/b", + IssuedBy: catalogue.IssuedOutside}}}, + {Module: "postgres", OwnSecrets: catalogue.OwnSecrets{"superuser": {Path: "/c"}}}, + {Module: "mesh-provisioner", OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/d"}}}, + } + got := coreModulesOf(modules, func(m catalogue.Manifest) bool { return m.Module == "mesh-provisioner" }) + if want := map[string][]string{"nats": {"broker"}}; !reflect.DeepEqual(got, want) { + t.Fatalf("got %v, want %v", got, want) + } +} diff --git a/cmd/mesh-controller/delivery.go b/cmd/mesh-controller/delivery.go index 30d78461..8db2bca5 100644 --- a/cmd/mesh-controller/delivery.go +++ b/cmd/mesh-controller/delivery.go @@ -1051,7 +1051,9 @@ func appliedFrom(ctx context.Context, inv *inventory.Inventory) inventory.Applie fmt.Fprintf(os.Stderr, "the report of %s after %s could not be read: %v\n", node, walkSent.Format(time.RFC3339), err) return inventory.AppliedReport{}, false } - return r, ok + // And, for each module the send handed to the provisioner, its client's report on that build (novox/hq ADR + // 0362): applied only once both said it, at the later; failed if the client failed. + return inventory.ProvisionedSend(node, send, r, ok, provLookup(ctx)) } } diff --git a/cmd/mesh-controller/desk_secret.go b/cmd/mesh-controller/desk_secret.go index 930a29da..b8cabc90 100644 --- a/cmd/mesh-controller/desk_secret.go +++ b/cmd/mesh-controller/desk_secret.go @@ -72,6 +72,9 @@ type deskGive struct { // nil keeps no record (a test that does not look). end closes it with how it ended. open func(node, module, name, desk string) (int64, error) end func(id int64, outcome string) error + // vault, when set, is the path of a module whose secrets are filled the new way (vault_secrets.go): the vault + // records the ask, holds its bounds and opens what is typed; open, end and accept are not used. + vault vaultAsker } // askedByName is the caller as the prompt names it: the first clause of what the bus said, in the characters @@ -138,6 +141,9 @@ func (d deskGive) give(node, module, name, desk string) (string, error) { "bus, where an agent may answer first (novox/hq ADR 0259 §10)", module, name, node, module, name) } } + if d.vault != nil { + return d.giveToVault(node, module, name, desk) + } // The bounds, read and kept before anybody is asked to type (novox/hq ADR 0277): one open prompt per secret, // few an hour. How the ask ends is recorded whatever happens below. outcome := "failed" @@ -297,6 +303,15 @@ func askAtDesk(ctx context.Context, node, module, name, desk string) error { return announceSecretGiven(ctx, node, module, name, how, askedBy) }, } + // A module whose secrets are filled the new way is asked for by the vault (ADR 0349 clause 9); every other + // keeps today's path (clause 17). + ours, err := filledTheNewWay(module) + if err != nil { + return fmt.Errorf("nothing was asked: %w", err) + } + if ours { + d.vault = theVault(ctx) + } words, err := d.give(node, module, name, desk) if err != nil { return err diff --git a/cmd/mesh-controller/facts.go b/cmd/mesh-controller/facts.go index 50b891cf..af04f202 100644 --- a/cmd/mesh-controller/facts.go +++ b/cmd/mesh-controller/facts.go @@ -348,11 +348,11 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot if layer.From == catalogue.MeshWideLayer { if !meshWide[module] { meshWide[module] = true - f.Settings = append(f.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Values)}) + f.Settings = append(f.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Whole())}) } continue } - m.Settings = append(m.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Values)}) + m.Settings = append(m.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Whole())}) } } m.Declaration = declarationFacts(ctx, open, n.Name, gens, gensErr, scrub) diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index 8a1e1a1c..45792f00 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -235,6 +235,10 @@ type gateFacts struct { sent map[string]inventory.SentDeclaration sentBuilds map[string]map[string]string commits map[string]string + // provisioned answers each machine's provisioner client's newest report of a module (novox/hq ADR 0349 §15): + // a module the gate's send handed to the provisioner is healthy only once it reported that build placed. Nil + // reads as nothing said. + provisioned inventory.ProvLookup } // heldReading is the provider a module's findings are held under, as " on ", and, when that @@ -291,7 +295,7 @@ func (f gateFacts) supersededOn(module, machine string) (string, bool) { var gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { inv := open.inventory f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, - rolledBack: witnessed.all()} + rolledBack: witnessed.all(), provisioned: provLookup(ctx)} reports, err := inv.LastReports(ctx) if err != nil { return f, err @@ -368,6 +372,12 @@ func judgeHealth(module, component string, m catalogue.Manifest, machine string, case r.Outcome != inventory.OutcomeApplied: return healthNotYet, fmt.Sprintf("%s reported %q", machine, r.Outcome) } + // **And its files placed** (novox/hq ADR 0349 §15): a module the send handed to the provisioner is judged only + // once that machine's client reported the build's templates placed, so a module whose files are not yet written + // is never judged; a failure there is the build's on that machine. + if h, why := judgeProvisioned(module, machine, r, f); h != healthGood { + return h, why + } // **No new condition about it**: about the machine itself, or naming the module on that machine, // raised since the judging began. The gate's own are not evidence about the build. A fault that was // there at the send and reopened since is not new; one that had cleared before the send and came back @@ -449,6 +459,32 @@ func judgeHealth(module, component string, m catalogue.Manifest, machine string, return healthGood, "" } +// judgeProvisioned is whether a module's files are placed on a machine, for a module the gate's send handed to the +// provisioner there: healthy for any other. +func judgeProvisioned(module, machine string, r inventory.Reported, f gateFacts) (health, string) { + sent, kept := f.sent[machine] + digest := sent.Provisioned[module] + if !kept || digest == "" { + return healthGood, "" + } + at := time.Time{} + if r.At != nil { + at = *r.At + } + one := inventory.SentDeclaration{Digest: sent.Digest, At: sent.At, Provisioned: map[string]string{module: digest}} + got, ok := inventory.ProvisionedSend(machine, one, inventory.AppliedReport{At: at, + Outcome: inventory.OutcomeApplied}, true, f.provisioned) + switch { + case !ok: + return healthNotYet, fmt.Sprintf("the provisioner on %s has not reported %s's files placed for the new "+ + "build", machine, module) + case got.Outcome != inventory.OutcomeApplied: + return healthBroken, fmt.Sprintf("the provisioner on %s did not place %s's files: %s", machine, module, + got.Outcome) + } + return healthGood, "" +} + // machineWord is what one judging found wrong with a machine itself, apart from its modules: facts is // the judging's facts without those conditions, on what each names among the modules the send moved, // and whole what holds the machine back as a whole. diff --git a/cmd/mesh-controller/generation.go b/cmd/mesh-controller/generation.go index abf9ff0e..b8d1203b 100644 --- a/cmd/mesh-controller/generation.go +++ b/cmd/mesh-controller/generation.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "fmt" "io" "os" @@ -60,12 +61,30 @@ type sentRecord struct { toldGeneration int64 sender string modules []string + // plans is each module the send handed to the provisioner, its plan, as JSON (design 57 P6); nil for none. + plans []byte } // sentRecordOf is a composed send's record: its sender is the caller this process acts for. func sentRecordOf(ctx context.Context, d sendable) sentRecord { - return sentRecord{sequence: d.Sequence, epoch: d.actingEpoch, generation: d.composedFrom, + r := sentRecord{sequence: d.Sequence, epoch: d.actingEpoch, generation: d.composedFrom, toldGeneration: d.Generation, sender: senderOf(callerOf(ctx)), modules: d.modules} + if len(d.plans) > 0 { + kept := map[string]keptPlan{} + for module, p := range d.plans { + kept[module] = keptPlan{Plan: p, RequiredBy: d.requiredBy[module]} + } + // A map of plain strings and slices: it marshals. + r.plans, _ = json.Marshal(kept) + } + return r +} + +// keptPlan is a handed module's plan as a send keeps it (migration 0100): the plan, and for a module only pulled in +// by another, the module that requires it (issue 765, inventory.SentProvisioningPlans). +type keptPlan struct { + catalogue.Plan + RequiredBy string `json:"required-by,omitempty"` } // callerOf is who asked for what this process does: the seat call's caller when ctx belongs to one, the diff --git a/cmd/mesh-controller/given_test.go b/cmd/mesh-controller/given_test.go index 827ae1b7..8a45a465 100644 --- a/cmd/mesh-controller/given_test.go +++ b/cmd/mesh-controller/given_test.go @@ -8,7 +8,8 @@ import ( ) // A rotation asked through the seat carries why to the command, which records it in the hand-act -// log (novox/hq ADR 0228); why with a provision is refused as passed over, not dropped. +// log (novox/hq ADR 0228); and with a provision too, since the vault asks why of every rotation of a +// credential it holds (issue 767). func TestARotationThroughTheSeatCarriesWhy(t *testing.T) { argv, err := argvFor("rotate", map[string]any{"node": "anchor", "module": "letta", "secret": "server-password", "why": "leaked into logs", "cause": "leaked"}) @@ -19,8 +20,10 @@ func TestARotationThroughTheSeatCarriesWhy(t *testing.T) { if err != nil || strings.Join(argv, " ") != "secret rotate anchor letta server-password" { t.Fatalf("without why: %v %v", argv, err) } - if argv, err := argvFor("rotate", map[string]any{"provision": "postgres-database", "why": "leaked"}); err == nil { - t.Fatalf("why beside a provision was passed over: %v", argv) + argv, err = argvFor("rotate", map[string]any{"provision": "postgres-database", "consumer": "ace", "why": "leaked", + "cause": "leaked"}) + if err != nil || strings.Join(argv, " ") != "rotate postgres-database --consumer ace --why leaked --cause leaked" { + t.Fatalf("why beside a provision: %v %v", argv, err) } } diff --git a/cmd/mesh-controller/handacts.go b/cmd/mesh-controller/handacts.go index 147a6dca..6b451ef7 100644 --- a/cmd/mesh-controller/handacts.go +++ b/cmd/mesh-controller/handacts.go @@ -105,6 +105,12 @@ var handActVerbs = []handActVerb{ // (ADR 0257). {Verb: "mirrors", Decision: "which copies in the store no record names are the mesh's is a person's " + "word (ADR 0257)"}, + // A move of data a release runs once after its delivery (design 57): a person's step of the release, never a + // repair. + {Verb: "move-placement", Decision: "a release's move of data runs once after its delivery, on a person's " + + "word (design 57, issue 736)"}, + {Verb: "move-config", Decision: "a release's move of data runs once after its delivery, on a person's " + + "word (design 57, issue 664)"}, {Verb: "bus upgrade", Decision: "the bus is never rolled by the mesh: replacing it is a planned step a " + "person starts (ADR 0236)"}, {Verb: "upgrade release-backlog", Decision: "after a release plan failed, the next opens only when a " + @@ -118,6 +124,9 @@ var handActVerbs = []handActVerb{ {Verb: "secret accept", Decision: "a value an outside party issued is given by a person, at their desk"}, {Verb: "secret rotate", Decision: "a value a person judged disclosed is replaced on their word", DecidedFor: []string{causeLeakedInLogs}}, + // A provider's credential the vault holds, rotated there on a person's word (issue 767): the same judgement. + {Verb: "rotate", Decision: "a value a person judged disclosed is replaced on their word", + DecidedFor: []string{causeLeakedInLogs}}, } // personsDecision is whether an act in the log is a person's decision by design, by the verb that diff --git a/cmd/mesh-controller/issued.go b/cmd/mesh-controller/issued.go index 1f1b348c..4ae02291 100644 --- a/cmd/mesh-controller/issued.go +++ b/cmd/mesh-controller/issued.go @@ -43,6 +43,32 @@ const issuedAsk = 20 * time.Second // issuedProvider is one assignment of a module whose offer says its software issues the provision's credential. type issuedProvider struct { Node, Module, Tool string + // Holders are the provision's holders from this assignment's machine, named to the vault, whose tool may answer + // a value for each (issue 749). + Holders []issuedHolder +} + +// issuedHolder is one holder of a provision from a provider: a consuming module on one machine (issue 749). +type issuedHolder struct { + Node string `json:"node"` + Module string `json:"module"` +} + +func (h issuedHolder) String() string { return h.Module + " on " + h.Node } + +// issuedHoldersFrom are the provision's holders from provider, each consuming module on each machine once. +func issuedHoldersFrom(holders []inventory.Holder, provider string) []issuedHolder { + seen := map[issuedHolder]bool{} + var out []issuedHolder + for _, h := range holders { + one := issuedHolder{Node: h.Consumer, Module: h.ConsumerModule} + if h.Provider != provider || seen[one] { + continue + } + seen[one] = true + out = append(out, one) + } + return out } // issuedTaker is the line's work, with what it reaches handed in: the store, the bus and the sends. @@ -53,6 +79,10 @@ type issuedTaker struct { keep func(provision, provider, value string) (inventory.IssuedKept, error) record func(act link.HandAct) error send func(machines []string) error + // ofTheVault says whether a provider module's secrets are filled the new way; for one that is, the value goes + // into the vault (vault, vault_secrets.go) and nothing is kept, sealed or sent here. Nil is none. + ofTheVault vaultSwitch + vault vaultAsker } // take asks every provider of the provision for its software's value and keeps each; it answers what it did, in @@ -90,6 +120,15 @@ func (t issuedTaker) take(provision string) (string, error) { // takeFrom is one provider: asked, opened, kept, recorded and sent when it changed. func (t issuedTaker) takeFrom(provision string, p issuedProvider) (string, error) { + if t.ofTheVault != nil { + ours, err := t.ofTheVault(p.Module) + if err != nil { + return "", err + } + if ours { + return t.takeIntoVault(provision, p) + } + } public, private, err := secrets.Keypair() if err != nil { return "", fmt.Errorf("no key could be made to take the value: %w", err) @@ -174,6 +213,16 @@ func issuedProvidersOf(ctx context.Context, inv *inventory.Inventory, provision if err != nil { return nil, true, err } + out, err := issuedProvidersFrom(shelf, holders, func(node string) ([]string, error) { + return inv.Assigned(ctx, node) + }, provision) + return out, true, err +} + +// issuedProvidersFrom are the assignments to ask, from the provision's holders: on every machine providing it, the +// assigned modules whose offer says their software issues it, each with that machine's holders (issue 749). +func issuedProvidersFrom(shelf map[string]catalogue.Manifest, holders []inventory.Holder, + assignedOn func(node string) ([]string, error), provision string) ([]issuedProvider, error) { nodes := map[string]bool{} for _, h := range holders { nodes[h.Provider] = true @@ -185,13 +234,17 @@ func issuedProvidersOf(ctx context.Context, inv *inventory.Inventory, provision sort.Strings(names) var out []issuedProvider for _, node := range names { - assigned, err := inv.Assigned(ctx, node) + assigned, err := assignedOn(node) if err != nil { - return nil, true, err + return nil, err } - out = append(out, issuedOn(shelf, node, assigned, provision)...) + on := issuedOn(shelf, node, assigned, provision) + for i := range on { + on[i].Holders = issuedHoldersFrom(holders, node) + } + out = append(out, on...) } - return out, true, nil + return out, nil } // offersIssued is whether any module offers the provision with a credential its software issues. @@ -262,7 +315,9 @@ func secretIssued(ctx context.Context, args []string) error { return err }) }, - send: func(machines []string) error { return sendTo(ctx, open, machines) }, + send: func(machines []string) error { return sendTo(ctx, open, machines) }, + ofTheVault: filledTheNewWay, + vault: theVault(ctx), } words, err := t.take(provision) if words != "" { diff --git a/cmd/mesh-controller/issued_test.go b/cmd/mesh-controller/issued_test.go index c9b1b6c8..f2b9da84 100644 --- a/cmd/mesh-controller/issued_test.go +++ b/cmd/mesh-controller/issued_test.go @@ -193,7 +193,7 @@ func TestTheProvidersAskedAreTheOffersThatSaySo(t *testing.T) { "jellyfin": {Module: "jellyfin", Provides: []catalogue.Offer{{Name: "plex-api"}}}, } got := issuedOn(shelf, "ace", []string{"jellyfin", "plex", "kometa"}, "plex-api") - if len(got) != 1 || got[0] != (issuedProvider{Node: "ace", Module: "plex", Tool: "plex_issued_credential"}) { + if len(got) != 1 || got[0].Node != "ace" || got[0].Module != "plex" || got[0].Tool != "plex_issued_credential" { t.Fatalf("the providers asked: %+v", got) } if err := issuedRefusesRotation(shelf, "plex-api"); err == nil || !strings.Contains(err.Error(), "secret issued plex-api") { @@ -216,3 +216,84 @@ func TestAnIssuedValueIsADecisionInTheHandActLog(t *testing.T) { } t.Fatal("secret issued is not in the hand-act table") } + +// **A need's credential is the vault's for its holder alone** (issue 757) exactly when its providing module is marked +// moved, so `secret issued` goes to the vault naming each holder, and its software issues the provision's credential. +func TestANeedIsIssuedToItsHolderOnlyWhereTheProviderIsMovedAndItsSoftwareIssuesIt(t *testing.T) { + forge := func(moved bool, issued bool) catalogue.Manifest { + offer := catalogue.Offer{Name: "gitea-read"} + if issued { + offer.Credential = &catalogue.OfferCredential{IssuedBy: catalogue.IssuedBySoftware, Tool: "gitea_agent_token"} + } + return catalogue.Manifest{Module: "gitea", Moved: moved, Provides: []catalogue.Offer{offer}} + } + shared := func(moved bool) catalogue.Manifest { + return catalogue.Manifest{Module: "gitea", Moved: moved, OwnSecrets: catalogue.OwnSecrets{"key": {Path: "/k"}}, + Provides: []catalogue.Offer{{Name: "gitea-read", Credential: &catalogue.OfferCredential{Own: "key"}}}} + } + given := func() catalogue.Manifest { + return catalogue.Manifest{Module: "gitea", Moved: true, Provides: []catalogue.Offer{{Name: "gitea-read", + Credential: &catalogue.OfferCredential{IssuedBy: catalogue.IssuedOutside}}}} + } + need := catalogue.Needed{Name: "gitea-read", From: "novox", Module: "gitea", For: "claude-code"} + for why, c := range map[string]struct { + shelf map[string]catalogue.Manifest + need catalogue.Needed + want bool + }{ + "moved and issued by its software": {map[string]catalogue.Manifest{"gitea": forge(true, true)}, need, true}, + "not moved": {map[string]catalogue.Manifest{"gitea": forge(false, true)}, need, false}, + "a login for each consumer": {map[string]catalogue.Manifest{"gitea": forge(true, false)}, need, true}, + "a login for each, not moved": {map[string]catalogue.Manifest{"gitea": forge(false, false)}, need, false}, + "a credential every holder shares": {map[string]catalogue.Manifest{"gitea": shared(true)}, need, false}, + "a credential a person gives": {map[string]catalogue.Manifest{"gitea": given()}, need, false}, + "a provider not on the shelf": {map[string]catalogue.Manifest{}, need, false}, + "answered by a record": {map[string]catalogue.Manifest{"gitea": forge(true, true)}, + catalogue.Needed{Name: "gitea-read", ByRecord: true, Module: "gitea", For: "claude-code"}, false}, + } { + if got := issuedToHolder(c.shelf, c.need); got != c.want { + t.Errorf("%s: issued to its holder is %v, want %v", why, got, c.want) + } + } +} + +// **The vault is a moved provider for its consumers' credentials** (issue 793): its pair rows are the vault's since +// V3, so a need of `secret` is issued to its holder and answered by a moved provider, whether or not the module is +// marked moved; another module is not. +func TestTheModuleHoldingTheVaultSeatCountsAsMovedForItsConsumers(t *testing.T) { + vault := catalogue.Manifest{Module: "mesh-vault", Claims: []catalogue.Claim{{Name: "mesh-vault"}}, + Provides: []catalogue.Offer{{Name: "secret"}}} + other := catalogue.Manifest{Module: "mesh-vault", Provides: []catalogue.Offer{{Name: "secret"}}} + need := catalogue.Needed{Name: "secret", Local: "admin", From: "novox", Module: "mesh-vault", For: "gitea"} + if !issuedToHolder(map[string]catalogue.Manifest{"mesh-vault": vault}, need) || + !providerMoved(map[string]catalogue.Manifest{"mesh-vault": vault}, need) { + t.Error("the vault's consumer is not answered by a moved provider") + } + if issuedToHolder(map[string]catalogue.Manifest{"mesh-vault": other}, need) || + providerMoved(map[string]catalogue.Manifest{"mesh-vault": other}, need) { + t.Error("a module not holding the vault seat counts as moved") + } +} + +// **A provision answered by the helper makes its provider a moved one for that provision alone** (issue 792): the +// provides entry says `"answered-by": "helper"`, the module need not be marked moved, and no credential is the vault's +// for it. Another provision of the module, or a module without the entry, does not count. +func TestAnOfferAnsweredByTheHelperCountsAsMovedForThatProvisionOnly(t *testing.T) { + proxy := catalogue.Manifest{Module: "route-proxy", Provides: []catalogue.Offer{ + {Name: "route", AnsweredBy: catalogue.AnsweredByHelper}, {Name: "other"}}} + plain := catalogue.Manifest{Module: "route-proxy", Provides: []catalogue.Offer{{Name: "route"}}} + shelf := func(m catalogue.Manifest) map[string]catalogue.Manifest { + return map[string]catalogue.Manifest{m.Module: m} + } + route := catalogue.Needed{Name: "route", From: "novox", Module: "route-proxy", For: "gitea"} + other := catalogue.Needed{Name: "other", From: "novox", Module: "route-proxy", For: "gitea"} + if !providerMoved(shelf(proxy), route) { + t.Error("a provision answered by the helper is not answered by a moved provider") + } + if providerMoved(shelf(proxy), other) || providerMoved(shelf(plain), route) { + t.Error("a provision not answered by the helper counts as moved") + } + if issuedToHolder(shelf(proxy), route) { + t.Error("the flag made a credential the vault's") + } +} diff --git a/cmd/mesh-controller/issued_to_vault.go b/cmd/mesh-controller/issued_to_vault.go new file mode 100644 index 00000000..212a3d16 --- /dev/null +++ b/cmd/mesh-controller/issued_to_vault.go @@ -0,0 +1,67 @@ +package main + +import ( + "context" + "fmt" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +// A covered account is handed to the vault when it is minted (novox/hq ADR 0374, issue 787). +// +// **Covered** are the bus accounts `module issue` mints for a core module (catalogue.CoreCredentialModules) or for a +// module marked moved; no other account is. Until the vault makes bus accounts (ADR 0113 clause 6, issue 774) the +// controller mints such an account and writes its hash to the bus's user list, as before, and then hands the whole +// value at once to the vault's `accept` with origin `issued`: sealed here to the key the vault made for that call, so +// the vault alone opens it (ADR 0349 clause 8). The vault keeps it as the module's own secret on that machine and +// raises `secret-changed`; a re-issue is the same hand with a new value, which replaces the old in place. An account +// not covered is minted and kept as it was and handed to nobody. + +// vaultAskerForIssue is the vault as `module issue` asks it. A test replaces it. +var vaultAskerForIssue = func(ctx context.Context) vaultAsker { return theVault(ctx) } + +// accountIsCovered says whether the bus accounts minted for m are the vault's to hold (ADR 0374 clause 1). +func accountIsCovered(m catalogue.Manifest) bool { + return catalogue.CarriesVaultSealed(m.Module) || m.Moved +} + +// vaultInstalled says whether the vault is assigned to any machine. Until it is, as in genesis (ADR 0113 §15 to §18, +// mesh-host's bootstrap, which issues the bus's and the provisioner's accounts before the vault step), there is no +// vault to hand to: the value is kept in the controller's store as before, and the vault takes it afterwards by the +// move (move-in takes a value of the controller's store with origin made, accepted or issued, as it is). Only a +// vault that is installed and does not answer fails the issue. +func vaultInstalled(ctx context.Context, inv *inventory.Inventory) (bool, error) { + entries, err := inv.Catalogued(ctx) + if err != nil { + return false, err + } + for _, e := range entries { + if e.Manifest.Module == "mesh-vault" && len(e.On) > 0 { + return true, nil + } + } + return false, nil +} + +// handIssuedToVault hands the value just minted for module's own secret name on node to the vault, when the module's +// accounts are covered, and does nothing otherwise. Loud when the vault does not take it: the bus's user list +// already holds the new hash, so a vault that kept the old value would serve a password that no longer logs in. +func handIssuedToVault(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, node, name, held string) error { + if !accountIsCovered(m) { + return nil + } + if installed, err := vaultInstalled(ctx, inv); err != nil { + return fmt.Errorf("whether the vault is installed cannot be read, so %s of %s was not handed to it: %w", name, + m.Module, err) + } else if !installed { + return nil + } + target := ownSecretArgs(node, m.Module, name) + target["origin"] = "issued" + if _, err := acceptIntoVault(vaultAskerForIssue(ctx), target, held, "module issue"); err != nil { + return fmt.Errorf("the credential is minted and kept here, and the bus's user list holds its hash, but the "+ + "vault did not take %s of %s on %s: %w (`module issue` again hands a new one)", name, m.Module, node, err) + } + return nil +} diff --git a/cmd/mesh-controller/issued_to_vault_test.go b/cmd/mesh-controller/issued_to_vault_test.go new file mode 100644 index 00000000..d2525d65 --- /dev/null +++ b/cmd/mesh-controller/issued_to_vault_test.go @@ -0,0 +1,157 @@ +package main + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" +) + +// Every other test mints without a vault to hand to: a stand-in that takes anything, replaced by the tests below. +func init() { + vaultAskerForIssue = func(context.Context) vaultAsker { return newStandInVault(nil).ask } +} + +// A covered account is handed to the vault when it is minted (novox/hq ADR 0374, issue 787): the whole value, sealed +// to the key the vault made for the call, with origin issued. A module that is neither core nor moved hands nothing. +func TestModuleIssueHandsACoveredAccountToTheVaultAndOnlyThat(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + theVaultIsInstalled(t, open) + inv := open.inventory + known := broker.Broker{Fingerprint: "ab"} + + core := aProvisioner() + core.Module = "nats" + moved := aBot("moved-bot", true) + moved.OwnSecrets["broker"] = catalogue.OwnSecret{Path: "/var/lib/moved-bot/broker"} + plain := aBot("kept-bot", false) + plain.OwnSecrets["broker"] = catalogue.OwnSecret{Path: "/var/lib/kept-bot/broker"} + + vault := newStandInVault(t) + vaultAskerForIssue = func(context.Context) vaultAsker { return vault.ask } + t.Cleanup(func() { + vaultAskerForIssue = func(context.Context) vaultAsker { return newStandInVault(nil).ask } + }) + + for _, m := range []catalogue.Manifest{core, moved, plain} { + register(t, open, m) + if _, err := assign(ctx, open, "laptop", m.Module); err != nil { + t.Fatal(err) + } + } + issue := func(m catalogue.Manifest, password string) { + t.Helper() + if err := issueWith(ctx, inv, m, "laptop", "", known, "10.0.0.1:4222", "module."+m.Module+".laptop", password); err != nil { + t.Fatal(err) + } + } + + issue(core, "first-core-password") + issue(moved, "first-moved-password") + issue(plain, "kept-password") + if err := issueProvisionerClient(ctx, inv, core, "laptop", known, "10.0.0.1:4222"); err != nil { + t.Fatal(err) + } + + said := vault.said() + for _, want := range []string{`"module":"nats"`, `"module":"moved-bot"`, `"name":"broker"`, + `"name":"` + broker.ProvisionerSecret + `"`, `"node":"laptop"`} { + if !strings.Contains(said, want) { + t.Errorf("the vault was told %s, without %s", said, want) + } + } + if n := strings.Count(said, `"origin":"issued"`); n != 3 { + t.Errorf("%d accounts were handed with origin issued, want 3 (the core's two and the moved module's): %s", n, said) + } + if strings.Contains(said, "kept-bot") { + t.Errorf("an account that is neither core nor moved was handed to the vault: %s", said) + } + if strings.Contains(said, "first-core-password") || strings.Contains(said, "first-moved-password") { + t.Errorf("a value crossed the bus in the clear: %s", said) + } + + // A re-issue hands the new value, which the vault opens: the whole {url, fingerprint, node, user, password}. + issue(moved, "second-moved-password") + if got := vault.taken["accept"]; !strings.Contains(got, "second-moved-password") || + !strings.Contains(got, `"user":"module.moved-bot.laptop"`) || !strings.Contains(got, `"node":"laptop"`) || + !strings.Contains(got, `"url":"nats://10.0.0.1:4222"`) || !strings.Contains(got, `"fingerprint":"ab"`) { + t.Errorf("the vault was handed %q after a re-issue, not the new account whole", got) + } +} + +// A vault that does not take the value fails the issue loudly: the bus's user list holds the new hash already. +func TestModuleIssueFailsWhenTheVaultDoesNotTakeACoveredAccount(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + moved := aBot("moved-bot", true) + moved.OwnSecrets["broker"] = catalogue.OwnSecret{Path: "/var/lib/moved-bot/broker"} + theVaultIsInstalled(t, open) + register(t, open, moved) + if _, err := assign(ctx, open, "laptop", moved.Module); err != nil { + t.Fatal(err) + } + vault := newStandInVault(t) + vault.refuse = errors.New("the vault is not there") + vaultAskerForIssue = func(context.Context) vaultAsker { return vault.ask } + t.Cleanup(func() { + vaultAskerForIssue = func(context.Context) vaultAsker { return newStandInVault(nil).ask } + }) + err := issueWith(ctx, open.inventory, moved, "laptop", "", broker.Broker{Fingerprint: "ab"}, "10.0.0.1:4222", + "module.moved-bot.laptop", "pw") + if err == nil || !strings.Contains(err.Error(), "vault did not take") { + t.Fatalf("a vault that took nothing was reported as %v", err) + } +} + +func theVaultIsInstalled(t *testing.T, open *stores) { + t.Helper() + vault := catalogue.Manifest{Module: "mesh-vault", Version: "1"} + register(t, open, vault) + if _, err := assign(t.Context(), open, "laptop", vault.Module); err != nil { + t.Fatal(err) + } +} + +// In genesis the bus's and the provisioner's accounts are issued before the vault is installed (ADR 0113 §15 to §18): +// there is no vault to hand to, so nothing is asked and nothing fails, and the value stays in the controller's store +// as before, for the move to carry into the vault. Once the vault is installed, one that does not answer fails. +func TestModuleIssueBeforeTheVaultIsInstalledKeepsTheValueAndHandsNothing(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + known := broker.Broker{Fingerprint: "ab"} + core := aProvisioner() + core.Module = "nats" + register(t, open, core) + if _, err := assign(ctx, open, "laptop", core.Module); err != nil { + t.Fatal(err) + } + vault := newStandInVault(t) + vault.refuse = errors.New("there is no vault yet") + vaultAskerForIssue = func(context.Context) vaultAsker { return vault.ask } + t.Cleanup(func() { + vaultAskerForIssue = func(context.Context) vaultAsker { return newStandInVault(nil).ask } + }) + + if err := issueWith(ctx, inv, core, "laptop", "", known, "10.0.0.1:4222", "module.nats.laptop", "pw"); err != nil { + t.Fatalf("genesis's issue of the bus's account failed with no vault installed: %v", err) + } + if err := issueProvisionerClient(ctx, inv, core, "laptop", known, "10.0.0.1:4222"); err != nil { + t.Fatalf("genesis's issue of the client's account failed with no vault installed: %v", err) + } + if said := vault.said(); said != "" { + t.Errorf("the vault was asked before it was installed: %s", said) + } + if _, held, err := inv.ModuleSecretIfIssued(ctx, "laptop", "nats", "broker"); err != nil || !held { + t.Errorf("the value is not kept in the controller's store (held %v, %v)", held, err) + } + + theVaultIsInstalled(t, open) + if err := issueWith(ctx, inv, core, "laptop", "", known, "10.0.0.1:4222", "module.nats.laptop", "pw2"); err == nil { + t.Error("a vault that is installed and does not answer did not fail the issue") + } +} diff --git a/cmd/mesh-controller/lean_walk_test.go b/cmd/mesh-controller/lean_walk_test.go index 48ed5077..eced96cf 100644 --- a/cmd/mesh-controller/lean_walk_test.go +++ b/cmd/mesh-controller/lean_walk_test.go @@ -3,6 +3,7 @@ package main import ( "context" "slices" + "strings" "testing" "time" @@ -30,13 +31,13 @@ func TestAWalkEndsOnItsMachinesReports(t *testing.T) { {Node: "novox", Outcome: inventory.OutcomeFailed, At: &at, Declared: "d-novox"}, {Node: "shanks", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-before"}, } - waiting, failed, silent := walkReports(p, reports, now) + waiting, failed, silent := walkReports(p, reports, now, nil) if !slices.Equal(waiting, []string{"shanks"}) || !slices.Equal(failed, []string{"novox"}) || !slices.Equal(silent, []string{"laptop"}) { t.Fatalf("waiting %v, failed %v, silent %v", waiting, failed, silent) } reports[2].Declared = "d-shanks" - if waiting, _, _ := walkReports(p, reports, now); len(waiting) != 0 { + if waiting, _, _ := walkReports(p, reports, now, nil); len(waiting) != 0 { t.Fatalf("the walk still waits once every machine reported: %v", waiting) } } @@ -186,3 +187,56 @@ func TestAMachineSilentPastItsBoundIsLeftAndSaidAtOnce(t *testing.T) { t.Fatalf("the silence is not said: %+v", obs) } } + +// Issue 840 (novox/hq ADR 0362 §1-5): a walk's end counts a machine's send of a provisioned module applied only once +// the engine and the machine's provisioner client both said so. +func TestAWalkEndWaitsForTheProvisioner(t *testing.T) { + now := time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC) + at := now.Add(-10 * time.Second) + plan := func(sentAt time.Time) inventory.Plan { + return inventory.Plan{Modules: map[string]*inventory.PlanModule{"app": {Rest: map[string]inventory.SentDeclaration{ + "ace": {Digest: "d-ace", At: sentAt, Provisioned: map[string]string{"app": "t1"}}, + "g14": {Digest: "d-g14", At: sentAt, Provisioned: map[string]string{"app": "t1"}}, + }}}} + } + reports := []inventory.Reported{ + {Node: "ace", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-ace"}, + {Node: "g14", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-g14"}, + } + says := map[string]inventory.ProvReport{} + prov := func(node, module string) (inventory.ProvReport, bool) { r, ok := says[node+"/"+module]; return r, ok } + applied := inventory.ProvReport{Status: inventory.ProvApplied, Templates: "t1", At: at} + + // (a) the engine applied, the client has not said: the end waits on both. + waiting, failed, silent := walkReports(plan(now.Add(-time.Minute)), reports, now, prov) + if !slices.Equal(waiting, []string{"ace", "g14"}) || len(failed)+len(silent) != 0 { + t.Fatalf("a: waiting %v, failed %v, silent %v", waiting, failed, silent) + } + // ace's client applied, g14's failed on that build (b): ace done, g14 a failed apply naming the reason. + says["ace/app"] = applied + says["g14/app"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t1", Reason: "no value", At: at} + waiting, failed, silent = walkReports(plan(now.Add(-time.Minute)), reports, now, prov) + if len(waiting) != 0 || len(silent) != 0 || len(failed) != 1 || !strings.HasPrefix(failed[0], "g14 (") || + !strings.Contains(failed[0], "no value") { + t.Fatalf("b: waiting %v, failed %v, silent %v", waiting, failed, silent) + } + // (c) no client report past ApplySilentAfter: named silent, never applied. + delete(says, "g14/app") + waiting, failed, silent = walkReports(plan(now.Add(-inventory.ApplySilentAfter-time.Minute)), reports, now, prov) + if len(waiting) != 0 || len(failed) != 0 || !slices.Equal(silent, []string{"g14"}) { + t.Fatalf("c: waiting %v, failed %v, silent %v", waiting, failed, silent) + } + // both said it: nothing waits. + says["g14/app"] = applied + if waiting, failed, silent = walkReports(plan(now.Add(-time.Minute)), reports, now, prov); len(waiting)+len(failed)+len(silent) != 0 { + t.Fatalf("both said: waiting %v, failed %v, silent %v", waiting, failed, silent) + } + // (d) a send handing nothing to the provisioner is the engine's report alone, as before. + bare := plan(now.Add(-time.Minute)) + bare.Modules["app"].Rest = map[string]inventory.SentDeclaration{"ace": {Digest: "d-ace", At: now.Add(-time.Minute)}} + if waiting, failed, silent = walkReports(bare, reports, now, func(string, string) (inventory.ProvReport, bool) { + return inventory.ProvReport{}, false + }); len(waiting)+len(failed)+len(silent) != 0 { + t.Fatalf("d: waiting %v, failed %v, silent %v", waiting, failed, silent) + } +} diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index 41d7319d..d3f380b2 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -121,6 +121,10 @@ func run() error { return imagesCommand(ctx, args[1:]) case "mirrors": return mirrorsCommand(ctx, args[1:]) + case "move-placement": + return movePlacementCommand(ctx, args[1:]) + case "move-config": + return moveConfigCommand(ctx, args[1:]) case "plans": return plansCommand(ctx, args[1:]) case "delivery": @@ -275,6 +279,10 @@ func usage() { artifacts [--json] [--repository ] [--collected] every artifact the mesh made: kept and why, eligible, collected (ADR 0251) collect [--json] [--most ] what the store's sweep would let go of, now; nothing is done collect --confirm --why [--most ] run the sweep now: hold every kept archive, let go of the eligible + move-placement [--json] which settings layers still hold placement keys (endpoints, expose, reach, ports); nothing is moved + move-placement --confirm --why move them beside the assignment, once, after the delivery (issue 736) + move-config [--json] every config value the settings layers hold, and where in mesh-config it goes; nothing is set + move-config --confirm --why set them in mesh-config, read them back, and read config from there (issue 664) images [--json] every container image the machine's declaration names, now and as last sent seats [--json] every seat this mesh defines, what it delivers, and who holds it seat rename rename a seat; its former name still resolves (ADR 0122) @@ -331,7 +339,7 @@ func usage() { licence add|list|use|key model access, under the name a person calls it licence manager the node that holds a refreshable licence's refresh token licence refresh mint a new access token and seal it to every holder - rotate [--consumer ] [--module ] a new credential for every holder, both ends at once + rotate [--consumer ] [--module ] [--why ] a new credential for every holder, both ends at once (--why where the vault holds it) ask [json] call one of a module's tools over the broker, and print its answer pin which provider this one gets a provision from: the module, and its node diff --git a/cmd/mesh-controller/merge_gate.go b/cmd/mesh-controller/merge_gate.go index ddcfd609..f4dcdd59 100644 --- a/cmd/mesh-controller/merge_gate.go +++ b/cmd/mesh-controller/merge_gate.go @@ -550,8 +550,17 @@ func shelfWithChange(base map[string]catalogue.Manifest, sources map[string]inve out[k] = m } var failures []string + // The list of predefined keys at the tree's root (novox/hq ADR 0348 §7): what the build of this tree judges each + // manifest against and registration publishes, so what the gate judges each against too. A list that is + // wrong is refused by name, and every manifest is then judged against none. + predefined, err := catalogue.ReadPredefined(tree) + if err != nil { + failures = append(failures, fmt.Sprintf("%s: the controller judging this (%s) refuses it — %s", + catalogue.PredefinedFile, version, oneLine(err.Error()))) + predefined = nil + } found := map[string]string{} // module → its directory in the tree - err := filepath.WalkDir(tree, func(path string, d fs.DirEntry, err error) error { + err = filepath.WalkDir(tree, func(path string, d fs.DirEntry, err error) error { if err != nil { return err } @@ -575,7 +584,19 @@ func shelfWithChange(base map[string]catalogue.Manifest, sources map[string]inve if err != nil { return err } - m, err := catalogue.ParseManifest(raw) + m, err := catalogue.ParseManifestWith(raw, predefined) + // The manifest as written, still naming its templates: what the stand-in build resolves, as a real build + // does, so a moved module is handed its templates (issue 876). + written := m + var texts map[string]string + if err == nil { + // Each template's text from the tree beside the manifest, as the build reads it back from the archive it + // packs (novox/hq ADR 0349 §2): so a change to a template alone is a change to its module. + m, err = catalogue.WithTemplatesFrom(m, filepath.Dir(path)) + } + if err == nil && written.Templated() { + texts, err = catalogue.ReadTemplates(written, filepath.Dir(path)) + } if err != nil { if inScope(dir) { failures = append(failures, fmt.Sprintf("%s: the controller judging this (%s) refuses it — %s", @@ -600,12 +621,12 @@ func shelfWithChange(base map[string]catalogue.Manifest, sources map[string]inve "of one module name (issue 239)", orRoot(dir), m.Module, s.Repository)) return nil } - resolved, err := standInBuild(m) + resolved, err := standInBuild(written, texts) if err == nil { // As registration reads a built manifest: strictly, again (takeIn). var raw []byte if raw, err = json.Marshal(resolved); err == nil { - resolved, err = catalogue.ParseManifest(raw) + resolved, err = catalogue.ParseManifestWith(raw, predefined) } } if err == nil { @@ -723,15 +744,24 @@ func builtAlike(m catalogue.Manifest) string { // standInBuild resolves a manifest as a build would, with artifacts nobody built: a digest made from the // module's and the artifact's names, in the store's own vocabulary. Composition and validation read the // shape of a reference, never its bytes. -func standInBuild(m catalogue.Manifest) (catalogue.Manifest, error) { +func standInBuild(m catalogue.Manifest, texts map[string]string) (catalogue.Manifest, error) { if m.Build == nil { return m.Resolve(nil) } + // The templates archive carries each template's text by its path inside the archive, as a build reads it back. + archive, inside := catalogue.TemplatesArtifact(m), m.TemplatesDir()+"/" + read := map[string]string{} + for template, text := range texts { + read[strings.TrimPrefix(template, inside)] = text + } var built []catalogue.Built for _, a := range m.Build.Artifacts { sum := sha256.Sum256([]byte("gate\x00" + m.Module + "\x00" + a.Name)) digest := fmt.Sprintf("sha256:%x", sum) b := catalogue.Built{Name: a.Name, Kind: a.Kind, Digest: digest} + if a.Name == archive && archive != "" { + b.Texts = read + } switch a.Kind { case catalogue.ArtifactImage, catalogue.ArtifactUpstream: b.Reference = catalogue.ArtifactStoreScheme + m.Module + "/" + a.Name + "@" + digest @@ -1083,6 +1113,19 @@ func raiseFromFacts(ctx context.Context, open *stores, f snapshot.Facts, shelf m continue } } + // And the provisioner client's account, the same way (novox/hq ADR 0349 clause 11, design 57 P3). + if a.Provider == "" && a.Name == broker.ProvisionerSecret { + if pm, held := shelf[a.Module]; !held || !runsTheProvisionerClient(pm) { + continue + } + if _, err := inv.MintBusPassword(ctx, inventory.BusUser{Kind: inventory.BusProvisioner, Node: m.Name, + Module: a.Module, Username: broker.Principal{Kind: broker.KindProvisioner, Node: m.Name, + Module: a.Module}.Username()}); err != nil { + notes = append(notes, fmt.Sprintf("%s's provisioner client account for %s is not kept: %s", + m.Described(), a.Module, oneLine(err.Error()))) + continue + } + } var err error if a.Provider == "" { err = inv.AcceptSecretForModule(ctx, m.Name, a.Module, a.Name, standIn) diff --git a/cmd/mesh-controller/mesh_for_test.go b/cmd/mesh-controller/mesh_for_test.go index b2e76be4..eea81d13 100644 --- a/cmd/mesh-controller/mesh_for_test.go +++ b/cmd/mesh-controller/mesh_for_test.go @@ -33,7 +33,21 @@ import ( func aMesh(t *testing.T) *stores { t.Helper() inventory.ForTest(t) // raises the store, migrates it, and points the environment at it - licences.ForTest(t) // planning reaches this one too, by name and never by connection + return aMeshOnTheStore(t) +} + +// aMeshBefore is aMesh on a store migrated only below migration `number`, with the function that runs the +// rest: for a test of what a migration that moves data does to what the mesh composes (novox/hq issue 651). +func aMeshBefore(t *testing.T, number int) (*stores, func()) { + t.Helper() + _, migrate := inventory.ForTestBefore(t, number) + return aMeshOnTheStore(t), migrate +} + +// aMeshOnTheStore is the mesh aMesh makes, on the store the environment points at. +func aMeshOnTheStore(t *testing.T) *stores { + t.Helper() + licences.ForTest(t) // planning reaches this one too, by name and never by connection open, err := openStores(t.Context()) if err != nil { diff --git a/cmd/mesh-controller/modules.go b/cmd/mesh-controller/modules.go index 66804655..80c58cab 100644 --- a/cmd/mesh-controller/modules.go +++ b/cmd/mesh-controller/modules.go @@ -8,6 +8,7 @@ import ( "fmt" "net" "os" + "path/filepath" "sort" "strings" "time" @@ -127,6 +128,10 @@ func moduleCommand(ctx context.Context, args []string) error { if err != nil { return err } + // A manifest handed over by hand takes each template's text from the tree beside it (novox/hq ADR 0349 §2). + if m, err = catalogue.WithTemplatesFrom(m, filepath.Dir(positionals[0])); err != nil { + return err + } from, err := whereItComesFrom(*repo, *ref, *commit, *path, *self) if err != nil { return err @@ -441,6 +446,10 @@ func settingsCommand(ctx context.Context, args []string) error { if len(positionals) == 2 { values = positionals[1] } + // Once its config values are mesh-config's, they are proposed there (issue 664). + if forwarded, err := forwardPropose(ctx, inv, positionals[0], *node, values, *clear); forwarded || err != nil { + return err + } return proposeCommand(ctx, positionals[0], *node, values, *clear, *replace) case "proposals": if len(positionals) > 1 || *node != "" || *clear || *replace || *history { @@ -461,6 +470,15 @@ func settingsCommand(ctx context.Context, args []string) error { where = *node } + // Once the move of config has run, a module in the config form is read and set through mesh-config (issue 664). + if (args[0] == "set" && len(positionals) == 2) || ((args[0] == "show" && !*history) || args[0] == "clear") && + len(positionals) == 1 { + if forwarded, err := forwardSettings(ctx, inv, args[0], positionals[0], *node, positionals, *replace); forwarded || + err != nil { + return err + } + } + switch args[0] { case "set": if len(positionals) != 2 { @@ -537,6 +555,10 @@ func settingsCommand(ctx context.Context, args []string) error { if p.SetBy != "" { by = "; " + p.SetBy } + if p.Why != "" { + // A layer whose placement moved out of it (migration 0097): still the layer, as it stands. + by += "; " + p.Why + } fmt.Printf("%s on %s, until %s (%s%s):\n %s\n", positionals[0], where, p.ReplacedAt.Local().Format("2006-01-02 15:04:05"), p.ReplacedBy, by, shown) } @@ -940,7 +962,55 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue if err != nil { return err } - return issueWith(ctx, inv, m, node, busAddress, known, reachable, user, password) + if err := issueWith(ctx, inv, m, node, busAddress, known, reachable, user, password); err != nil { + return err + } + if !runsTheProvisionerClient(m) { + return nil + } + return issueProvisionerClient(ctx, inv, m, node, known, reachable) +} + +// runsTheProvisionerClient says a module runs its machine's provisioner client, which reaches the bus on an +// account of its own (novox/hq ADR 0349 clause 11, design 57 P3): it claims the seat and declares the own secret +// that account is delivered in. The same test the bus's user list makes (inventory's declaredFor). +func runsTheProvisionerClient(m catalogue.Manifest) bool { + _, reads := m.OwnSecrets[broker.ProvisionerSecret] + return reads && m.ClaimsSeat(broker.NodeProvisionerSeat) +} + +// issueProvisionerClient gives a machine's provisioner client its account: minted under the client's own name, +// never the module's or the tool runner's, and sealed to the machine as the module's own secret +// broker.ProvisionerSecret, which the client's service reads (novox/hq ADR 0349 clause 11, design 57 P3). No +// claims and no consumer: the client holds no seat on this account and is delivered nothing but its answers. +func issueProvisionerClient(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, node string, + known broker.Broker, reachable string) error { + user := broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: m.Module}.Username() + password, err := inv.MintBusPassword(ctx, inventory.BusUser{ + Username: user, Kind: inventory.BusProvisioner, Node: node, Module: m.Module, + }) + if err != nil { + return err + } + held, err := json.Marshal(struct { + URL string `json:"url"` + Fingerprint string `json:"fingerprint,omitempty"` + Node string `json:"node"` + User string `json:"user"` + Password string `json:"password"` + }{URL: "nats://" + reachable, Fingerprint: known.Fingerprint, Node: node, User: user, Password: password}) + if err != nil { + return err + } + if err := inv.AcceptSecretForModule(ctx, node, m.Module, broker.ProvisionerSecret, string(held)); err != nil { + return err + } + if err := handIssuedToVault(ctx, inv, m, node, broker.ProvisionerSecret, string(held)); err != nil { + return err + } + fmt.Printf("bus user %s minted for %s's provisioner client, scoped to %s's keys, its reports and the vault\n", + user, m.Module, node) + return nil } // busKindOf is what a module's bus user is recorded as: the node's tool runtime where the module is @@ -985,6 +1055,9 @@ func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manife if err := inv.AcceptSecretForModule(ctx, node, m.Module, "broker", string(held)); err != nil { return err } + if err := handIssuedToVault(ctx, inv, m, node, "broker", string(held)); err != nil { + return err + } // And how it hears what it consumes. Derived from its declaration, and only when it declared // something: a module that consumes nothing needs no consumer, and creating one would be a diff --git a/cmd/mesh-controller/placement_move.go b/cmd/mesh-controller/placement_move.go new file mode 100644 index 00000000..1f1c289b --- /dev/null +++ b/cmd/mesh-controller/placement_move.go @@ -0,0 +1,99 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "strings" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// movePlacementAnswer is what the move of placement says it did, or would do (issue 736). +type movePlacementAnswer struct { + // DryRun is true when nothing was moved: asked without --confirm. + DryRun bool `json:"dry-run"` + // Moved is each layer that held a placement key among its settings, and which keys. + Moved []inventory.MovedLayer `json:"moved"` +} + +// movePlacementCommand is `move-placement`: the move of placement out of the settings layers (novox/hq ADR 0348 +// clause 11, issue 736), one of the moves of data a release runs once after its delivery (design 57). +func movePlacementCommand(ctx context.Context, args []string) error { + set := flag.NewFlagSet("move-placement", flag.ContinueOnError) + asJSON := set.Bool("json", false, "answer as JSON") + confirm := set.Bool("confirm", false, "move the keys, rather than only say which layers hold them") + f := addHandActFlags(set) + positionals, err := parseAround(set, args) + if err != nil { + return err + } + if len(positionals) != 0 { + return errors.New("move-placement [--json] [--confirm --why ]") + } + if !*confirm && f.given() { + return errors.New("move-placement takes --why only with --confirm: without it nothing is moved. Nothing was done") + } + if *confirm { + // A move a person asks for says why, before anything is asked of the store. + if err := f.require("move-placement"); err != nil { + return err + } + } + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + if *confirm { + f.record(ctx, "move-placement", nil) + } + answer, err := runMovePlacement(ctx, open.inventory, *confirm) + if err != nil { + return err + } + if *asJSON { + encoder := json.NewEncoder(os.Stdout) + encoder.SetIndent("", " ") + return encoder.Encode(answer) + } + fmt.Print(describeMovePlacement(answer)) + return nil +} + +// runMovePlacement is the move against a store: a dry run says what it would move. +func runMovePlacement(ctx context.Context, inv *inventory.Inventory, apply bool) (movePlacementAnswer, error) { + moved, err := inv.MovePlacement(ctx, apply) + if err != nil { + return movePlacementAnswer{}, err + } + return movePlacementAnswer{DryRun: !apply, Moved: moved}, nil +} + +// describeMovePlacement says the answer as a person reads it. +func describeMovePlacement(a movePlacementAnswer) string { + if len(a.Moved) == 0 { + return "no settings layer holds a placement key: nothing to move\n" + } + var b strings.Builder + verb := "moved" + if a.DryRun { + verb = "would move" + } + for _, m := range a.Moved { + where := m.Node + if where == "" { + where = "the whole mesh" + } + fmt.Fprintf(&b, "%s on %s: %s %s\n", m.Module, where, verb, strings.Join(m.Keys, ", ")) + } + if a.DryRun { + fmt.Fprintf(&b, "%d layer(s); nothing was moved — with --confirm --why they are\n", len(a.Moved)) + } else { + fmt.Fprintf(&b, "%d layer(s) moved beside their assignments, each kept whole in its history first\n", len(a.Moved)) + } + return b.String() +} diff --git a/cmd/mesh-controller/placement_moved_test.go b/cmd/mesh-controller/placement_moved_test.go new file mode 100644 index 00000000..5831cc56 --- /dev/null +++ b/cmd/mesh-controller/placement_moved_test.go @@ -0,0 +1,113 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// **Placement leaves the settings layers and no machine is told anything new** (novox/hq ADR 0348 clause +// 11, issue 651). A mesh is seeded as the store holds it before migration 0097 — `places`, `accesses` and a +// kept network among the settings, at the mesh's level and a machine's, one layer of placement only — then +// migrated, and every machine's declaration is composed. What each machine is told of the seeded modules is +// byte for byte what the controller before this change composed from the same seed: the golden files were +// written by the integration branch's controller at bfc79c6 running this same test (MESH_WRITE_GOLDEN=1), +// where there was no migration 0097 to run. +func TestADeclarationIsTheSameAfterPlacementMoves(t *testing.T) { + open, migrate := aMeshBefore(t, 97) + ctx := t.Context() + inv := open.inventory + image := "registry.invalid:5000/library@sha256:" + strings.Repeat("a", 64) + register(t, open, catalogue.Manifest{Module: "library", Version: "1", + Accesses: []catalogue.Access{{ID: "series"}, {ID: "downloads", Mode: catalogue.AccessRead}}, + Resources: []map[string]any{ + {"id": "config", "type": "directory", "mode": "0750"}, + {"id": "server", "type": "container", "image": image, + "volumes": []any{"${dir:config}:/config", "${access:series}:/series", "${access:downloads}:/downloads:ro"}}, + // Today's form: a mergeable file naming no keys takes every key of the layer (ADR 0326 §3). + {"id": "runtime-config", "type": "file", "path": "${dir:config}/runtime.json", "mode": "0640", + "content": `{"log":"info"}`, "merge": "json"}, + }}) + register(t, open, catalogue.Manifest{Module: "found", Version: "1", + Resources: []map[string]any{{"id": "server", "type": "container", "image": image}}}) + if err := inv.SetAdopted(ctx, "anchor", true); err != nil { + t.Fatal(err) + } + for _, l := range []struct{ node, module, values string }{ + {"", "library", `{"accesses": {"downloads": "/storage/downloads"}, "log": "warn"}`}, + {"anchor", "library", `{"places": {"config": {"path": "/var/lib/library/config", "owner": "1001:2000"}}, + "accesses": {"series": "/storage/media/series"}, "workers": 4}`}, + {"laptop", "library", `{"accesses": {"series": "/home/media/series"}}`}, + {"anchor", "found", `{"networks": {"server": ["predecessor_default"]}}`}, + } { + if l.node == "" { + inv.ExecForTest(t, `insert into settings (node, module, values) values (null, $1, $2)`, l.module, l.values) + continue + } + inv.ExecForTest(t, `insert into settings (node, module, values) select id, $2, $3 from node where name = $1`, + l.node, l.module, l.values) + } + + migrate() + // Assigned once the store is migrated: an assignment reads the layers, as this controller holds them. + for _, a := range [][]string{{"anchor", "library", "found"}, {"laptop", "library"}} { + if _, err := assign(ctx, open, a[0], a[1:]...); err != nil { + t.Fatalf("assign %v: %v", a, err) + } + } + + gens, err := generators(ctx, open) + if err != nil { + t.Fatal(err) + } + for _, node := range []string{"anchor", "laptop"} { + plan, settings, err := planFor(ctx, open, node) + if err != nil { + t.Fatal(err) + } + with, _, err := renderingFor(ctx, open, node, plan, settings, gens, Allocating) + if err != nil { + t.Fatal(err) + } + composed, err := plan.Compose(with) + if err != nil { + t.Fatal(err) + } + if len(composed.LeftOut) > 0 { + t.Fatalf("%s leaves out %v", node, composed.LeftOut) + } + var told []map[string]any + for _, r := range composed.Resources { + if owner := composed.Owner[r["id"].(string)]; owner == "library" || owner == "found" { + told = append(told, r) + } + } + sort.Slice(told, func(i, j int) bool { return told[i]["id"].(string) < told[j]["id"].(string) }) + got, err := json.MarshalIndent(told, "", " ") + if err != nil { + t.Fatal(err) + } + golden := filepath.Join("testdata", "placement-moved", node+".json") + if os.Getenv("MESH_WRITE_GOLDEN") != "" { + if err := os.MkdirAll(filepath.Dir(golden), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(golden, append(got, '\n'), 0o644); err != nil { + t.Fatal(err) + } + continue + } + want, err := os.ReadFile(golden) + if err != nil { + t.Fatal(err) + } + if string(want) != string(got)+"\n" { + t.Errorf("%s is told something else after placement moved:\n%s\nwas:\n%s", node, got, want) + } + } +} diff --git a/cmd/mesh-controller/placement_words_moved_test.go b/cmd/mesh-controller/placement_words_moved_test.go new file mode 100644 index 00000000..63aa3185 --- /dev/null +++ b/cmd/mesh-controller/placement_words_moved_test.go @@ -0,0 +1,243 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/link" +) + +// wordedMesh is every live layer that says `endpoints`, `expose`, `reach` or `ports` (issue 736) — 36, read with +// the settings verb on 2026-10-14, ace's as anchor's and novox's as laptop's, one stand-in for an address — split as +// the store holds them once migration 0097 has run: the four words still among the settings, `places` and +// `accesses` already beside them. With each module shaped as its definition in the catalogue (mesh-catalog's, and +// the media catalogue's) is where the four words reach: its listens, what its containers publish and on which +// network, its route, its directories and accesses, and its mergeable files — every key of the layer, or the keys +// they name. The shapes are the modules'; the files' text and the images are the test's. +type wordedMesh struct { + Modules []json.RawMessage `json:"modules"` + Layers []struct { + Node string `json:"node"` + Module string `json:"module"` + Settings map[string]any `json:"settings"` + Placement map[string]any `json:"placement"` + } `json:"layers"` +} + +func theWordedMesh(t *testing.T) wordedMesh { + t.Helper() + raw, err := os.ReadFile(filepath.Join("testdata", "placement-words-moved", "mesh.json")) + if err != nil { + t.Fatal(err) + } + var w wordedMesh + if err := json.Unmarshal(raw, &w); err != nil { + t.Fatal(err) + } + return w +} + +// aMeshWithTheWordsInItsLayers is aMesh with every module of the worded mesh registered, a route provider on both +// machines, and its layers held as the store holds them before the four words move. +func aMeshWithTheWordsInItsLayers(t *testing.T) (*stores, wordedMesh) { + t.Helper() + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + w := theWordedMesh(t) + image := "registry.invalid:5000/library@sha256:" + strings.Repeat("a", 64) + register(t, open, catalogue.Manifest{Module: "proxy", Version: "1", Provides: catalogue.FromAnywhere("route"), + Receives: map[string]string{"route": "/var/lib/proxy/routes.json"}}) + for _, text := range w.Modules { + m, err := catalogue.ParseManifest([]byte(strings.ReplaceAll(string(text), "IMAGE", image))) + if err != nil { + t.Fatal(err) + } + register(t, open, m) + } + on := map[string][]string{"anchor": {"proxy"}, "laptop": {"proxy"}} + for _, l := range w.Layers { + settings, err := json.Marshal(l.Settings) + if err != nil { + t.Fatal(err) + } + inv.ExecForTest(t, `insert into settings (node, module, values) select id, $2, $3 from node where name = $1`, + l.Node, l.Module, string(settings)) + if l.Placement != nil { + placement, err := json.Marshal(l.Placement) + if err != nil { + t.Fatal(err) + } + inv.ExecForTest(t, `insert into placement (node, module, values) select id, $2, $3 from node where name = $1`, + l.Node, l.Module, string(placement)) + } + on[l.Node] = append(on[l.Node], l.Module) + } + for _, node := range []string{"anchor", "laptop"} { + if err := inv.SetPublicDomain(ctx, node, node+".example.test"); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, node, on[node]...); err != nil { + t.Fatalf("assign %s %v: %v", node, on[node], err) + } + } + return open, w +} + +// toldOfTheWords is what a machine is told of the modules whose layers say the four words, and of the route +// provider that serves their names: those modules' resources and the filter's rules, as JSON a person can diff. +func toldOfTheWords(t *testing.T, open *stores, node string) string { + t.Helper() + ctx := t.Context() + gens, err := generators(ctx, open) + if err != nil { + t.Fatal(err) + } + plan, settings, err := planFor(ctx, open, node) + if err != nil { + t.Fatal(err) + } + with, _, err := renderingFor(ctx, open, node, plan, settings, gens, Allocating) + if err != nil { + t.Fatal(err) + } + composed, err := plan.Compose(with) + if err != nil { + t.Fatal(err) + } + if len(composed.LeftOut) > 0 { + t.Fatalf("%s leaves out %v", node, composed.LeftOut) + } + rules, err := plan.Rules(with) + if err != nil { + t.Fatal(err) + } + ours := map[string]bool{"proxy": true} + for _, l := range theWordedMesh(t).Layers { + ours[l.Module] = true + } + var told []map[string]any + for _, r := range composed.Resources { + if ours[composed.Owner[r["id"].(string)]] { + told = append(told, r) + } + } + sort.Slice(told, func(i, j int) bool { return told[i]["id"].(string) < told[j]["id"].(string) }) + got, err := json.MarshalIndent(map[string]any{"resources": told, "rules": rules}, "", " ") + if err != nil { + t.Fatal(err) + } + return string(got) + "\n" +} + +// sameAsTheGolden holds what a machine is told to the golden file the controller before issue 736 wrote from the +// same seed (MESH_WRITE_GOLDEN=1 writes it instead). +func sameAsTheGolden(t *testing.T, open *stores, when string) { + t.Helper() + for _, node := range []string{"anchor", "laptop"} { + got := toldOfTheWords(t, open, node) + golden := filepath.Join("testdata", "placement-words-moved", node+".json") + if os.Getenv("MESH_WRITE_GOLDEN") != "" { + if err := os.MkdirAll(filepath.Dir(golden), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(golden, []byte(got), 0o644); err != nil { + t.Fatal(err) + } + continue + } + want, err := os.ReadFile(golden) + if err != nil { + t.Fatal(err) + } + if string(want) != got { + t.Errorf("%s: %s is told something else:\n%s\nwas:\n%s", when, node, got, want) + } + } +} + +// **The four words leave the settings layers and no machine is told anything new** (novox/hq ADR 0348 clause 11, +// issue 736). Every live layer that says `endpoints`, `expose`, `reach` or `ports` is seeded as the store holds it +// after delivery, on a module shaped as its definition is, and every machine's declaration is composed. What each +// is told is byte for byte what the controller before this change composed from the same seed. +func TestADeclarationIsTheSameWhileTheWordsAreStillInTheLayers(t *testing.T) { + open, _ := aMeshWithTheWordsInItsLayers(t) + sameAsTheGolden(t, open, "before the move") +} + +// **And after the move, still nothing new** (issue 736). The move of placement, asked without confirming, says +// every layer that holds one of the four words and moves nothing; confirmed, it moves them all, no settings layer +// holds a placement key after, and each machine is told byte for byte what it was told before the move — and what +// the controller before this change told it. Asked again, it moves nothing. +func TestADeclarationIsTheSameAfterTheMoveOfPlacement(t *testing.T) { + open, w := aMeshWithTheWordsInItsLayers(t) + ctx := t.Context() + inv := open.inventory + asked, err := runMovePlacement(ctx, inv, false) + if err != nil || !asked.DryRun || len(asked.Moved) != len(w.Layers) { + t.Fatalf("the move, not confirmed, says %+v (%v); %d layers hold the words", asked, err, len(w.Layers)) + } + if held := inv.QueryForTest(t, `select module from settings where values ?| $1`, catalogue.PlacementKeys()); len(held) != len(w.Layers) { + t.Fatalf("the move, not confirmed, moved something: %d layers still hold the words", len(held)) + } + sameAsTheGolden(t, open, "after the move was asked without confirming") + + moved, err := runMovePlacement(ctx, inv, true) + if err != nil || moved.DryRun || !reflect.DeepEqual(moved.Moved, asked.Moved) { + t.Fatalf("the move says %+v (%v), it said it would move %+v", moved, err, asked.Moved) + } + if held := inv.QueryForTest(t, `select module from settings where values ?| $1`, catalogue.PlacementKeys()); len(held) != 0 { + t.Fatalf("a settings layer still holds a placement key: %v", held) + } + sameAsTheGolden(t, open, "after the move") + + if again, err := runMovePlacement(ctx, inv, true); err != nil || len(again.Moved) != 0 { + t.Fatalf("asked again, the move moved %+v (%v)", again, err) + } + if said := describeMovePlacement(movePlacementAnswer{}); !strings.Contains(said, "nothing to move") { + t.Fatalf("an empty move says %q", said) + } +} + +// The verb's command line: a dry run unless confirmed, and a confirmed move says why, recorded as a person's step +// of the release, never a repair (issue 736). +func TestTheMoveOfPlacementIsADryRunUnlessConfirmed(t *testing.T) { + for _, c := range []struct { + args map[string]any + want string + }{ + {map[string]any{}, "move-placement --json"}, + {map[string]any{"confirm": "false"}, "move-placement --json"}, + {map[string]any{"confirm": "true", "why": "release 57 is delivered"}, "move-placement --json --confirm --why release 57 is delivered"}, + } { + argv, err := argvFor("move-placement", c.args) + if err != nil || strings.Join(argv, " ") != c.want { + t.Errorf("%v: %q %v, want %q", c.args, argv, err, c.want) + } + } + for _, args := range []map[string]any{ + {"confirm": "true"}, // a move without a why + {"why": "tidy"}, // a why for a dry run, recorded nowhere + {"confirm": "yes", "why": "x"}, // a switch is true or false + {"node": "anchor"}, + } { + if argv, err := argvFor("move-placement", args); err == nil { + t.Errorf("%v was composed as %q", args, argv) + } + } + if repairingCommand([]string{"move-placement", "--json", "--confirm", "--why", "x"}) != "move-placement" { + t.Error("a confirmed move through the generic verb would go unrecorded") + } + if repairingCommand([]string{"move-placement", "--json"}) != "" { + t.Error("a dry run was taken for an act by hand") + } + if !personsDecision(link.HandAct{Verb: "move-placement"}) { + t.Error("a move of the release would count toward a healer the mesh lacks") + } +} diff --git a/cmd/mesh-controller/plain_words.go b/cmd/mesh-controller/plain_words.go index a1049696..6b20b4a7 100644 --- a/cmd/mesh-controller/plain_words.go +++ b/cmd/mesh-controller/plain_words.go @@ -186,6 +186,15 @@ var plainWordings = map[string]func(conditions.Observation) words{ }), // The controller or the node-engine kept at the build a machine was last sent, over an older record (novox/hq // issue 201). + kindVaultSilent: worded(func(o conditions.Observation) words { + m := machineOr(o, "a machine") + return words{Headline: "The vault did not give " + m + " the core's credentials", + Needs: "check the vault answers (vault_status); the next send asks it again.", + Explanation: fmt.Sprintf("The bus's and the provisioner's credentials reach %s sealed by the vault, and "+ + "the vault did not answer the last send for them. %s keeps the files it has; a core module not "+ + "there yet is not placed. The mesh never seals them itself instead.", m, m), + Resolved: "Resolved: the vault gave " + m + " the core's credentials"} + }), kindCoreDowngrade: worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") component := idPart(o, 0) diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index acfb086d..f08199c4 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -150,6 +150,10 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso // password a provider is told to create is the one its consumer was given — and sealed to // this node before it was ever written down, so nothing between here and there can read it. for i, n := range resolved.Needs { + // **Issued for this holder alone and kept by the vault** (issue 757): where this consumer is handed to the + // provisioner, its machine's client asks the vault for it, and the declaration carries no value. + resolved.Needs[i].IssuedToHolder = issuedToHolder(shelf, n) + resolved.Needs[i].ProviderMoved = providerMoved(shelf, n) if n.ByRecord { // Answered by something the mesh holds, so there is no pair-wise secret between two // machines. Its key was supplied by a person and sealed to this node then; the mesh @@ -476,9 +480,23 @@ func declarationWith(ctx context.Context, open *stores, node string, out := sendable{Resources: composed.Resources, Adoption: adoption, Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers, LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld, - unbound: with.Unbound, foreseen: composed.Foreseen, unplaced: composed.Unplaced, - modules: namedModules(plan, composed.LeftOut), + unbound: with.Unbound, foreseen: composed.Foreseen, unplaced: composed.Unplaced, handedOnly: composed.HandedOnly, + modules: namedModules(plan, composed.LeftOut), + Filled: composed.Filled, plans: composed.Plans, localNetwork: catalogue.LocalNetworkPorts(rules, with.LocalNetworks)} + if len(composed.Filled) > 0 { + out.Provisioner = with.Provisioner + } + // A handed module only pulled in by another keeps what requires it, for its value of its own in the assignments + // mirror (issue 765). + for module := range composed.Plans { + if by := plan.RequiredBy(module); by != "" { + if out.requiredBy == nil { + out.requiredBy = map[string]string{} + } + out.requiredBy[module] = by + } + } // And which build of each module it carries, for the send to record (novox/hq issue 259, ADR // 0221). Read only on the send path: a question about what would be sent records nothing. if choosing == Allocating { @@ -591,6 +609,10 @@ func reportLeftOut(node string, declared sendable) { for _, u := range declared.unplaced { fmt.Printf("%s: %s\n", node, u) } + // And each file only the provisioner's client fills, left out because the module is not handed here. + for _, u := range declared.handedOnly { + fmt.Printf("%s: %s\n", node, u) + } } // busCredentialIssued refuses an own secret called `broker` whose bus account nobody issued. @@ -601,10 +623,17 @@ func reportLeftOut(node string, declared sendable) { // cannot read and report the machine applied. Refused by name, with the verb — on the send, and on // a question asked ahead of it (Foreseeing), so that one is never told the push will make it. func busCredentialIssued(ctx context.Context, inv *inventory.Inventory, node, module, name string) error { - if name != "broker" { + var user string + switch name { + case "broker": + user = broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username() + case broker.ProvisionerSecret: + // And the machine's provisioner client's account, issued with its module (novox/hq ADR 0349 clause 11, + // design 57 P3): the same refusal, for the same reason. + user = broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: module}.Username() + default: return nil } - user := broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username() if _, minted, err := inv.BusUserHash(ctx, user); err != nil { return err } else if !minted { @@ -728,7 +757,18 @@ func renderingFor(ctx context.Context, open *stores, node string, // per node, so a module running on three machines has three. needed := map[string]map[string]string{} foreseen := map[string]map[string]bool{} + // **A module handed to the provisioner here has its secrets from the vault** (novox/hq ADR 0349 §7, §11, + // design 57 V4): its machine's client asks for each, so the controller makes, reads and seals none of them for + // this machine. Every other module's are made and sealed as before. + hands, err := engineTakesHanded(ctx, inv, node) + if err != nil { + return catalogue.Rendering{}, inventory.Node{}, err + } + handing := catalogue.Rendering{Hands: hands, Provisioner: provisionerAccount(plan)} for _, m := range plan.Modules { + if handing.Handing(m) { + continue + } // **A secret family is never made** (novox/hq ADR 0283): each member a person gave on this machine is // placed, and one not given is nothing — the module says it waits for it. for _, family := range m.OwnSecrets.Families() { @@ -822,6 +862,22 @@ func renderingFor(ctx context.Context, open *stores, node string, needed[m.Module][name] = sealed } } + // **The core's own credentials as the vault sealed them** (novox/hq ADR 0349 §16, issue 755), for an engine that + // opens the vault's box (core_box.go): the declaration carries the box, never the seal made above. Below that + // contract nothing here is asked and the composition is as it was. + var core coreBoxes + opensBox, err := engineOpensVaultBox(ctx, inv, node) + if err != nil { + return catalogue.Rendering{}, inventory.Node{}, err + } + if opensBox { + core, err = coreBoxesFor(ctx, inv, node, plan.Modules, func(m catalogue.Manifest) bool { + return handing.Handing(m) + }, choosing) + if err != nil { + return catalogue.Rendering{}, inventory.Node{}, err + } + } // And a certificate for this machine's name inside the mesh, when anything on it asks. Issued // rather than stored: the node's key does not change, so signing again produces an equally // valid certificate and there is nothing to keep in step. @@ -1016,8 +1072,11 @@ func renderingFor(ctx context.Context, open *stores, node string, ReadsHealth: readsHealth, JudgesRoot: judgesRoot, ReadsBusy: readsBusy, + Hands: hands, + Provisioner: provisionerAccount(plan), BusMembership: memberships[node], Settings: settings, Generators: gens, Grants: grants, Needed: needed, Foreseen: foreseen, Ports: ports, + VaultSealed: core.sealed, NotPlaced: core.notPlaced, Certificate: certificate, Authority: authority, Mesh: private, Names: names, Machines: machines, Zones: zones, Holders: replicas, Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation, @@ -1059,6 +1118,35 @@ func engineReadsBusy(ctx context.Context, inv *inventory.Inventory, node string) return had && stated.Contract >= link.BusyContract, nil } +// engineTakesHanded says whether a machine's node-engine writes a filled file on the provisioner client's word alone +// (link.HandedContract, design 57 P7), by its own newest statement: only there is a moved module handed to the +// provisioner (design 57 P6). Anywhere else a filled file would have two writers, and the module is sent as before. +func engineTakesHanded(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) { + stated, had, err := inv.HealthOf(ctx, node) + if err != nil { + return false, err + } + return had && stated.Contract >= link.HandedContract, nil +} + +// provisionerAccount is the account the provisioner's client runs as on this machine (novox/hq ADR 0349 §4): the user +// the process of the module running it runs as, empty where no such module is on the machine. +func provisionerAccount(plan catalogue.Resolution) string { + for _, m := range plan.Modules { + if !runsTheProvisionerClient(m) { + continue + } + for _, r := range m.Resources { + if r["type"] == "process" { + if user, _ := r["user"].(string); user != "" && user != "root" { + return user + } + } + } + } + return "" +} + // busyNotSent is, for a node-engine that does not read a container's busy, each container whose module declares // one: not sent it, so replaced at once while it may hold work (novox/hq ADR 0335). Nothing for one that reads it. func busyNotSent(modules []catalogue.Manifest, reads bool) []string { @@ -1846,6 +1934,42 @@ func issuedOutsideBy(shelf map[string]catalogue.Manifest, n catalogue.Needed) (s return catalogue.OffersIssuedOutside(shelf, n.Name) } +// issuedToHolder is whether the vault keeps a need's credential for its holder alone (issues 749, 757 and 779): its +// providing module is marked moved, so the vault holds the credential in a row of its own for each holder +// (filledTheNewWay), and its offer says either that the provider's software issues the provision's credential, or +// that the provider gives each consumer a login of its own (form pair, ADR 0158 §11). A credential every holder +// shares, or one a person gives, is neither. +func issuedToHolder(shelf map[string]catalogue.Manifest, n catalogue.Needed) bool { + if n.ByRecord || n.Module == "" { + return false + } + provider, known := shelf[n.Module] + if !known || !movedProvider(provider) { + return false + } + if _, issued := provider.IssuedCredentialOf(n.Name); issued { + return true + } + return provider.CredentialFormOf(n.Name) == catalogue.CredentialFormPair +} + +// providerMoved is whether a need is answered by a module marked moved (issue 788), or by one whose offer of the +// provision says the provider helper answers it (issue 792). +func providerMoved(shelf map[string]catalogue.Manifest, n catalogue.Needed) bool { + if n.ByRecord || n.Module == "" { + return false + } + provider, known := shelf[n.Module] + return known && (movedProvider(provider) || provider.AnsweredByHelperFor(n.Name)) +} + +// movedProvider is whether a provider's credentials are the vault's: the module is marked moved, or it holds the +// mesh-vault seat, whose pair rows have been in the vault since V3 (issue 793), so its consumers' credentials +// (gitea's internal token and admin password) are filled from those very rows with the values unchanged. +func movedProvider(m catalogue.Manifest) bool { + return m.Moved || slices.ContainsFunc(m.Claims, func(c catalogue.Claim) bool { return c.Name == "mesh-vault" }) +} + func providerModuleOf(resolved catalogue.Resolution, open *stores, ctx context.Context, n catalogue.Needed) string { for _, m := range resolved.Modules { if _, shared := m.SharedCredentialOf(n.Name); shared { diff --git a/cmd/mesh-controller/predefined.go b/cmd/mesh-controller/predefined.go new file mode 100644 index 00000000..7b93b748 --- /dev/null +++ b/cmd/mesh-controller/predefined.go @@ -0,0 +1,93 @@ +package main + +import ( + "context" + "fmt" + "path" + "strings" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// The list of predefined keys the mesh publishes (novox/hq ADR 0348 §7, design 57 package K2, issue 661). +// +// **The newest trunk build of the catalogue wins.** A build of the catalogue reads predefined.json at the commit +// it builds and carries it with the commit's generation (link.PredefinedList). Registration keeps it in the +// controller's store when that generation is larger than the one kept (inventory.KeepPredefined), so an older +// build's list heard later never replaces a newer one, and every manifest the controller reads again is judged +// against the list kept (catalogue.UsePredefined). mesh-config publishes it on the bus from there (design 57 +// package K3); the controller keeps no config value, only the keys. + +// fromTheCatalogue is whether a build is of the catalogue's repository, whose list alone the mesh publishes: by +// its name, in the URL form or the seat form. +func fromTheCatalogue(result link.BuildResult) bool { + for _, repository := range []string{result.Repository, sourceRepository(result)} { + name := strings.TrimSuffix(path.Base(strings.TrimRight(repository, "/")), ".git") + if name == catalogue.CatalogueRepository { + return true + } + } + return false +} + +func sourceRepository(result link.BuildResult) string { + if result.Source == nil { + return "" + } + return result.Source.Repository +} + +// carriedList is the list a build of the catalogue carried, read strictly; nil for a build of another +// repository, or one that carried none. +func carriedList(result link.BuildResult) (catalogue.PredefinedList, error) { + if result.Predefined == nil || !fromTheCatalogue(result) { + return nil, nil + } + list, err := catalogue.ParsePredefined(result.Predefined.Keys) + if err != nil { + return nil, fmt.Errorf("%s carried a list of predefined keys that is refused: %w", result.ID, err) + } + return list, nil +} + +// judgedWith is the list a build's manifest is first judged against at registration: the catalogue's own at the +// commit built, which its build judged it against, where it carried one; the list kept otherwise. +func judgedWith(result link.BuildResult) (catalogue.PredefinedList, error) { + list, err := carriedList(result) + if err != nil || list != nil { + return list, err + } + return catalogue.PublishedPredefined(), nil +} + +// loadPredefined makes the list kept in the store the one this controller judges manifests against. A store +// keeping none leaves the list it has. +func loadPredefined(ctx context.Context, inv *inventory.Inventory) error { + kept, has, err := inv.PredefinedKept(ctx) + if err != nil || !has { + return err + } + catalogue.UsePredefined(kept.Keys) + return nil +} + +// publishPredefined keeps a trunk build's list when it is newer than the one kept, says so, and loads what the +// store keeps then — this build's, or a newer one another controller process kept. +func publishPredefined(ctx context.Context, inv *inventory.Inventory, result link.BuildResult, + list catalogue.PredefinedList) error { + kept, err := inv.KeepPredefined(ctx, inventory.PredefinedRecord{Keys: list, Repository: result.Repository, + Commit: result.Commit, Generation: result.Predefined.Generation, Build: result.ID}) + if err != nil { + return err + } + if kept { + fmt.Printf("%s: the list of predefined keys at %s (generation %d) is published: %s\n", result.ID, + short(result.Commit), result.Predefined.Generation, strings.Join(list.Keys(), ", ")) + } else { + fmt.Printf("%s: the list of predefined keys at %s (generation %d) is not newer than the one kept, which "+ + "stands\n", result.ID, short(result.Commit), result.Predefined.Generation) + } + return loadPredefined(ctx, inv) +} diff --git a/cmd/mesh-controller/predefined_test.go b/cmd/mesh-controller/predefined_test.go new file mode 100644 index 00000000..25f09c50 --- /dev/null +++ b/cmd/mesh-controller/predefined_test.go @@ -0,0 +1,197 @@ +package main + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/link" +) + +// The list of predefined keys (novox/hq ADR 0348 §7, design 57 package K2, issue 661): `module check` and the +// merge gate judge each manifest against the list at the root of its tree; registration publishes the newest +// trunk build's list of the catalogue and judges every module against the list the mesh publishes. + +const theFirstKeys = `{ + "font-size": {"type": "number", "description": "the size text is shown in, in points", "minimum": 6, "maximum": 72}, + "agent-managed-settings": {"type": "object", "description": "the managed settings of every agent session, a JSON object", "trusted": true} +}` + +// notifierFallingBackTo is the notifier in the config form, its font size falling back to the key named. +func notifierFallingBackTo(key, kind string) string { + def := "10" + if kind == "string" { + def = `"10"` + } + return `{"module":"notifier","version":"1", + "config":{"font-size":{"type":"` + kind + `","description":"the notification's font size, in points", + "default":` + def + `,"predefined":"` + key + `"}}, + "resources":[{"id":"conf","type":"file","path":"/etc/notifier/notifier.conf","mode":"0644", + "content":"font = ${config:font-size}\n"}]}` +} + +// withoutAPublishedList judges the test against no list the mesh published, and puts the process's back after. +func withoutAPublishedList(t *testing.T) { + t.Helper() + was := catalogue.ReplacePublished(nil) + t.Cleanup(func() { catalogue.ReplacePublished(was) }) +} + +// aRepositoryTree is a repository's checkout holding one module at modules/notifier, and the list when given. +func aRepositoryTree(t *testing.T, manifest string, list *string) string { + t.Helper() + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, ".git"), 0o755); err != nil { + t.Fatal(err) + } + if list != nil { + if err := os.WriteFile(filepath.Join(root, catalogue.PredefinedFile), []byte(*list), 0o644); err != nil { + t.Fatal(err) + } + } + return aTemplatedModule(t, filepath.Join(root, "modules", "notifier"), manifest, nil) +} + +func TestModuleCheckJudgesAManifestAgainstTheListAtItsRepositoryRoot(t *testing.T) { + withoutAPublishedList(t) + list := theFirstKeys + check := func(path string) (string, error) { + var out bytes.Buffer + err := moduleCheck([]string{path}, &out) + return out.String(), err + } + if out, err := check(aRepositoryTree(t, notifierFallingBackTo("font-size", "number"), &list)); err != nil || + !strings.Contains(out, "notifier: ok") { + t.Fatalf("a module naming font-size was refused: %v\n%s", err, out) + } + for _, c := range []struct { + manifest string + list *string + says string + }{ + {notifierFallingBackTo("font-scale", "number"), &list, `"font-scale", which is not on the list`}, + {notifierFallingBackTo("font-size", "string"), &list, "which is a number, and this value is a string"}, + {notifierFallingBackTo("font-size", "number"), nil, "is not present"}, + {notifierFallingBackTo("font-size", "number"), ptr(`{"font-size": {"type": "number"}}`), "no description"}, + } { + if out, err := check(aRepositoryTree(t, c.manifest, c.list)); err == nil || !strings.Contains(out, c.says) { + t.Errorf("wanted the check refused saying %q, got %v:\n%s", c.says, err, out) + } + } +} + +func ptr(s string) *string { return &s } + +// writeTheList puts the list at the root of a tree the gate judges. +func writeTheList(t *testing.T, tree, list string) string { + t.Helper() + if err := os.WriteFile(filepath.Join(tree, catalogue.PredefinedFile), []byte(list), 0o644); err != nil { + t.Fatal(err) + } + return tree +} + +func TestTheMergeGateJudgesEachManifestAgainstTheListOfTheTree(t *testing.T) { + withoutAPublishedList(t) + f, manifests := templatedMesh(t) + + v := gateJudged(t, f, writeTheList(t, aTree(t, withEdit(manifests, "notifier", + notifierFallingBackTo("font-size", "number"))), theFirstKeys), "modules/notifier/module.json") + if v.Verdict == "fail" || v.Modules["notifier"] != "changed" { + t.Fatalf("a module naming font-size on the tree's list failed the gate (%v):\n%s", v.Modules, v.Report()) + } + for _, c := range []struct{ manifest, list, says string }{ + {notifierFallingBackTo("font-scale", "number"), theFirstKeys, `"font-scale", which is not on the list`}, + {notifierFallingBackTo("font-size", "string"), theFirstKeys, "which is a number"}, + {inlineNotifier, `{"font-size": {"type": "number", "description": "points", "unit": "pt"}}`, `unknown field "unit"`}, + } { + v := gateJudged(t, f, writeTheList(t, aTree(t, withEdit(manifests, "notifier", c.manifest)), c.list), + "modules/notifier/module.json", catalogue.PredefinedFile) + if v.Verdict != "fail" || !strings.Contains(v.Report(), c.says) { + t.Errorf("wanted the gate to fail saying %q:\n%s", c.says, v.Report()) + } + } +} + +// catalogueBuild is a trunk build of the catalogue carrying the list at a generation. +func catalogueBuild(id, manifest, list string, generation int) link.BuildResult { + r := link.BuildResult{ID: id, Repository: "http://forge.internal:20000/novox/mesh-catalog.git", + Path: "modules/notifier", Ref: "main", On: "anchor", Commit: strings.Repeat(id[len(id)-1:], 12), + Manifest: json.RawMessage(manifest), Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}} + if list != "" { + r.Predefined = &link.PredefinedList{Keys: json.RawMessage(list), Generation: generation} + } + return r +} + +func TestRegistrationPublishesTheNewestTrunkListAndJudgesEveryModuleAgainstIt(t *testing.T) { + withoutAPublishedList(t) + open := aMesh(t) + ctx := t.Context() + + // A manifest naming font-size registers, and its build's list is published. + if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, + catalogueBuild("b-1", notifierFallingBackTo("font-size", "number"), theFirstKeys, 10))); err != nil { + t.Fatalf("a module naming font-size was not registered: %v", err) + } + kept, has, err := open.inventory.PredefinedKept(ctx) + if err != nil || !has || kept.Generation != 10 || kept.Build != "b-1" { + t.Fatalf("the list was not published: %+v %v %v", kept, has, err) + } + if got := catalogue.PublishedPredefined(); len(got) != 2 { + t.Fatalf("the controller judges against %v, not the list published", got) + } + + // One naming a key not on the list, or of another type, is refused and never enters the catalogue. + for i, c := range []struct{ manifest, says string }{ + {notifierFallingBackTo("font-scale", "number"), "not on the list"}, + {notifierFallingBackTo("font-size", "string"), "which is a number"}, + } { + id := []string{"b-2", "b-3"}[i] + if _, _, err := takeIn(ctx, open.inventory, catalogueBuild(id, c.manifest, theFirstKeys, 10)); err == nil || + !strings.Contains(err.Error(), c.says) { + t.Errorf("wanted the registration refused saying %q, got %v", c.says, err) + } + } + src, err := open.inventory.SourceOf(ctx, "notifier") + if err != nil || src.BuiltFrom != strings.Repeat("1", 12) { + t.Fatalf("a refused build replaced the registered one: %+v %v", src, err) + } + + // A newer build's list, which drops agent-managed-settings, replaces the published one. + smaller := `{"font-size": {"type": "number", "description": "points", "minimum": 6, "maximum": 72}}` + if _, _, err := takeIn(ctx, open.inventory, catalogueBuild("b-4", + notifierFallingBackTo("font-size", "number"), smaller, 12)); err != nil { + t.Fatal(err) + } + // The older build's outcome heard later registers nothing new about the list: the newer stands. + if _, _, err := takeIn(ctx, open.inventory, catalogueBuild("b-5", + notifierFallingBackTo("font-size", "number"), theFirstKeys, 11)); err != nil { + t.Fatal(err) + } + kept, _, _ = open.inventory.PredefinedKept(ctx) + if kept.Build != "b-4" || len(kept.Keys) != 1 || len(catalogue.PublishedPredefined()) != 1 { + t.Fatalf("an older build's list heard later replaced the newer one: %+v", kept) + } + + // A build of another repository publishes nothing, and its manifest is judged against the published list. + other := catalogueBuild("b-6", notifierFallingBackTo("font-size", "number"), theFirstKeys, 99) + other.Repository, other.Source.Repository = "http://forge.internal:20000/novox/notifier.git", "novox/notifier" + other.Manifest = json.RawMessage(strings.Replace(notifierFallingBackTo("font-size", "number"), `"notifier"`, + `"chimes"`, 1)) + if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, other)); err != nil { + t.Fatalf("a module of another repository naming a published key: %v", err) + } + if kept, _, _ = open.inventory.PredefinedKept(ctx); kept.Build != "b-4" { + t.Fatalf("a build of another repository published its list: %+v", kept) + } + other.ID, other.Manifest = "b-7", json.RawMessage(strings.Replace(notifierFallingBackTo("agent-managed-settings", + "number"), `"notifier"`, `"chimes"`, 1)) + if _, _, err := takeIn(ctx, open.inventory, other); err == nil || !strings.Contains(err.Error(), "not on the list") { + t.Fatalf("a key only another repository's list holds was registered: %v", err) + } +} diff --git a/cmd/mesh-controller/proposals.go b/cmd/mesh-controller/proposals.go index 5bec29d7..aae0587e 100644 --- a/cmd/mesh-controller/proposals.go +++ b/cmd/mesh-controller/proposals.go @@ -54,6 +54,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/asks" + "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" @@ -906,6 +907,21 @@ func setLayerIn(open *stores) setOnWarrant { return "", fmt.Errorf("the layer of %s on %s changed since the operator was shown the change: it is not set; "+ "propose it again", p.Module, p.where()) } + // A proposal that carries config values, raised before the module's config moved, is not applied after it: its + // values in the layer would turn ReadsConfig off and undo the move (issue 830, novox/hq ADR 0348). A proposal + // of placement only is set as before: forwardPropose sends it down this path on purpose. + carried := p.Values + if p.Clear { + carried = before + } + if config, _ := catalogue.SplitPlacement(carried); len(config) > 0 { + if reads, err := inv.ReadsConfig(ctx, p.Module); err != nil { + return "", err + } else if reads { + return "", fmt.Errorf("the config of %s moved to mesh-config since the operator was shown the change: it is "+ + "not set; propose it again (it then goes to mesh-config)", p.Module) + } + } var changed string if p.Clear { if err := inv.ClearSettingsBy(ctx, p.Node, p.Module, setBy); err != nil { diff --git a/cmd/mesh-controller/provisioned.go b/cmd/mesh-controller/provisioned.go new file mode 100644 index 00000000..0c888a17 --- /dev/null +++ b/cmd/mesh-controller/provisioned.go @@ -0,0 +1,100 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/inventory" +) + +// What a machine's provisioner client said of a module's files (novox/hq ADR 0362, ADR 0349 §15, design 57 P6, +// issue 672), read where a walk's end and a gate count a machine's send applied: a module the send handed to the +// provisioner is applied there only once the client reported `prov.applied` on that build's templates as well. + +// provAppliedOn is the client's newest report of a module on a machine, from the stream that keeps it +// (broker.ProvAppliedStream); false when it has said nothing of it. A variable so a test can say the client's words. +var provAppliedOn = func(ctx context.Context, node, module string) (inventory.ProvReport, bool, error) { + var raw []byte + var said bool + read := func(js nats.JetStreamContext) error { + var err error + raw, said, err = broker.LastProvApplied(js, node, module) + return err + } + var err error + if d := doctorFrom; d != nil && d.js != nil { + err = read(d.js.Context()) + } else { + err = onTheBus(func(conn *nats.Conn) error { + js, err := conn.JetStream(nats.Context(ctx)) + if err != nil { + return err + } + return read(js) + }) + } + if err != nil || !said { + return inventory.ProvReport{}, false, err + } + var r inventory.ProvReport + if err := json.Unmarshal(raw, &r); err != nil { + return inventory.ProvReport{}, false, fmt.Errorf("what %s's provisioner client said of %s is not its report: %w", + node, module, err) + } + return r, true, nil +} + +// provLookup answers the clients' reports for one reading; a report that cannot be read is not counted, which +// leaves the machine waiting, then silent, never applied — and is said. +func provLookup(ctx context.Context) inventory.ProvLookup { + return func(node, module string) (inventory.ProvReport, bool) { + r, said, err := provAppliedOn(ctx, node, module) + if err != nil { + fmt.Fprintf(os.Stderr, "what %s's provisioner client said of %s could not be read: %v\n", node, module, err) + return inventory.ProvReport{}, false + } + return r, said + } +} + +// provisioningFailed is every module whose files a machine's provisioner client last said it failed to place, from +// the stream that keeps their reports. +func provisioningFailed(js nats.JetStreamContext) ([]inventory.ProvReport, error) { + all, err := broker.ProvAppliedReports(js) + if err != nil { + return nil, err + } + var out []inventory.ProvReport + for subject, raw := range all { + var r inventory.ProvReport + if err := json.Unmarshal(raw, &r); err != nil { + return nil, fmt.Errorf("what was said on %s is not a provisioner client's report: %w", subject, err) + } + if r.Status == inventory.ProvFailed { + out = append(out, r) + } + } + return out, nil +} + +// provisionedOutcome is a machine's report of one send as a delivery's end counts it (novox/hq ADR 0362 §1-5, issue +// 840): the engine's own report, combined with its provisioner client's for a send that handed modules to it. The +// outcome of a client's failure begins inventory.OutcomeProvisioningFailed; a report not yet applied on both sides +// is none (false). A send handing nothing is the engine's report unchanged. onSend says the report is about the send. +func provisionedOutcome(node string, sent inventory.SentDeclaration, r inventory.Reported, onSend bool, + prov inventory.ProvLookup) (string, bool) { + if len(sent.Provisioned) == 0 { + return r.Outcome, onSend + } + engine := inventory.AppliedReport{Outcome: r.Outcome} + if r.At != nil { + engine.At = *r.At + } + got, ok := inventory.ProvisionedSend(node, sent, engine, onSend, prov) + return got.Outcome, ok +} diff --git a/cmd/mesh-controller/provisioned_test.go b/cmd/mesh-controller/provisioned_test.go new file mode 100644 index 00000000..754ffcf8 --- /dev/null +++ b/cmd/mesh-controller/provisioned_test.go @@ -0,0 +1,114 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +// The controller hands a moved module to the provisioner, and counts it applied only once its files are placed +// (design 57 P6, issue 672; novox/hq ADR 0349 §4, §15, ADR 0362). + +// A declaration that hands files says which, and whose client fills them; one that hands none is byte for byte +// what it was. +func TestADeclarationNamesTheFilledFilesOnlyWhenItHandsAny(t *testing.T) { + plain := sendable{Resources: []map[string]any{{"id": "notify.configuration", "type": "file", "path": "/x", + "content": "a"}}, Sequence: 4} + before, err := plain.Body() + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(before), "filled_files") || strings.Contains(string(before), "provisioner_account") { + t.Fatalf("a declaration handing nothing names filled files: %s", before) + } + handing := plain + handing.Filled, handing.Provisioner = []string{"/x"}, "mesh-provisioner-client" + body, err := handing.Body() + if err != nil { + t.Fatal(err) + } + var envelope struct { + Filled []string `json:"filled_files"` + Provisioner string `json:"provisioner_account"` + } + if err := json.Unmarshal(body, &envelope); err != nil || len(envelope.Filled) != 1 || envelope.Filled[0] != "/x" || + envelope.Provisioner != "mesh-provisioner-client" { + t.Fatalf("the handing declaration says %s", body) + } +} + +func TestTheProvisionersAccountIsTheClientsProcessUser(t *testing.T) { + client := catalogue.Manifest{Module: "mesh-provisioner-client", + OwnSecrets: catalogue.OwnSecrets{"provisioner-broker": {Path: "/var/lib/x/broker"}}, + Claims: []catalogue.Claim{{Name: "node-provisioner", Scope: "node"}}, + Resources: []map[string]any{{"id": "account", "type": "user", "name": "mesh-provisioner-client"}, + {"id": "client", "type": "process", "name": "mesh-provisioner-client", "user": "mesh-provisioner-client"}}} + if got := provisionerAccount(catalogue.Resolution{Modules: []catalogue.Manifest{client}}); got != "mesh-provisioner-client" { + t.Fatalf("the client's account is %q", got) + } + if got := provisionerAccount(catalogue.Resolution{}); got != "" { + t.Fatalf("a machine without a client names %q", got) + } +} + +// The first-machine gate counts a handed module healthy only once its node's prov.applied for the new build has +// come (ADR 0349 §15, kept by ADR 0362 for a walk finishing at its gate); a failure there fails it. +func TestTheGateWaitsForTheProvisionersReport(t *testing.T) { + now := time.Date(2026, 10, 14, 23, 0, 0, 0, time.UTC) + since := now.Add(-time.Minute) + reported := now.Add(-50 * time.Second) + digest := "sha256:" + strings.Repeat("d", 64) + facts := func(said map[string]inventory.ProvReport) gateFacts { + return gateFacts{now: now, + reports: map[string]inventory.Reported{"g14": {Node: "g14", Outcome: inventory.OutcomeApplied, + At: &reported, Current: true}}, + sent: map[string]inventory.SentDeclaration{"g14": {Digest: "sent", At: since, + Provisioned: map[string]string{"dunst": digest}}}, + provisioned: func(node, module string) (inventory.ProvReport, bool) { + r, ok := said[node+"."+module] + return r, ok + }} + } + m := catalogue.Manifest{Module: "dunst"} + if h, why := judgeHealth("dunst", "", m, "g14", since, facts(nil)); h != healthNotYet || + !strings.Contains(why, "has not reported dunst's files placed") { + t.Fatalf("judged without the provisioner's report: %v %q", h, why) + } + failed := map[string]inventory.ProvReport{"g14.dunst": {Status: inventory.ProvFailed, Templates: digest, + Reason: "a value is not there", At: now.Add(-10 * time.Second)}} + if h, why := judgeHealth("dunst", "", m, "g14", since, facts(failed)); h != healthBroken || + !strings.Contains(why, "a value is not there") { + t.Fatalf("a failed placing is not the build's failure there: %v %q", h, why) + } + placed := map[string]inventory.ProvReport{"g14.dunst": {Status: inventory.ProvApplied, Templates: digest, + At: now.Add(-10 * time.Second)}} + if h, why := judgeHealth("dunst", "", m, "g14", since, facts(placed)); strings.Contains(why, "provisioner") { + t.Fatalf("placed, the module is still held on the provisioner: %v %q", h, why) + } + // A module the send did not hand is judged as before. + if h, why := judgeHealth("picom", "", catalogue.Manifest{Module: "picom"}, "g14", since, facts(nil)); strings.Contains(why, "provisioner") { + t.Fatalf("a module not handed waits on the provisioner: %v %q", h, why) + } +} + +// A machine whose provisioner client failed to place a module's files has a failed apply, said as apply-failing +// once the failure outlasts an apply interval, with the client's reason (ADR 0362 §3). +func TestAFailedPlacingRaisesApplyFailing(t *testing.T) { + now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC) + f := calm(now) + f.provFailed = []inventory.ProvReport{{Node: "anchor", Module: "dunst", Status: inventory.ProvFailed, + Reason: "the templates cannot be fetched", At: now.Add(-time.Minute)}} + if got := watchApplyFailing(f); len(got) != 0 { + t.Fatalf("raised within an apply interval: %+v", got) + } + f.provFailed[0].At = now.Add(-10 * time.Minute) + got := watchApplyFailing(f) + if len(got) != 1 || got[0].Key() != "machine.anchor.apply-failing" || + !strings.Contains(got[0].Summary, "dunst") || !strings.Contains(got[0].Said, "the templates cannot be fetched") { + t.Fatalf("a failed placing is not raised as anchor's apply-failing: %+v", got) + } +} diff --git a/cmd/mesh-controller/provisioner_account_test.go b/cmd/mesh-controller/provisioner_account_test.go new file mode 100644 index 00000000..bf537bff --- /dev/null +++ b/cmd/mesh-controller/provisioner_account_test.go @@ -0,0 +1,62 @@ +package main + +import ( + "slices" + "testing" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +func aProvisioner() catalogue.Manifest { + return catalogue.Manifest{Module: "mesh-provisioner", Version: "1", + DefinesSeats: []catalogue.SeatDeclaration{{Name: broker.NodeProvisionerSeat, Scope: catalogue.ScopeNode, + Serves: []catalogue.Verb{{Name: "get"}}}}, + Claims: []catalogue.Claim{{Name: broker.NodeProvisionerSeat, Scope: catalogue.ScopeNode}}, + OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/run/mesh-provisioner/broker"}, + broker.ProvisionerSecret: {Path: "/run/mesh-provisioner/client"}}, + } +} + +// A machine's provisioner client is issued an account of its own, under its own name, sealed to the machine as the +// module's own secret its service reads (novox/hq ADR 0349 clause 11, design 57 P3); until it is, a push refuses +// to invent one, as it does for a module's `broker`. +func TestTheProvisionerClientsAccountIsIssuedForItsServiceAndNeverInvented(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + m := aProvisioner() + register(t, open, m) + if _, err := assign(ctx, open, "laptop", m.Module); err != nil { + t.Fatal(err) + } + if err := busCredentialIssued(ctx, inv, "laptop", m.Module, broker.ProvisionerSecret); err == nil { + t.Fatal("a push would seal a placeholder for the client's account before it was issued") + } + + if err := issueProvisionerClient(ctx, inv, m, "laptop", broker.Broker{Fingerprint: "ab"}, "10.0.0.1:4222"); err != nil { + t.Fatal(err) + } + if _, minted, err := inv.BusUserHash(ctx, "provisioner.laptop"); err != nil || !minted { + t.Fatalf("the client's account is not minted: %v %v", minted, err) + } + states, err := inv.SecretsOf(ctx, "laptop", m.Module) + if err != nil { + t.Fatal(err) + } + if !slices.ContainsFunc(states, func(s inventory.SecretState) bool { return s.Name == broker.ProvisionerSecret }) { + t.Fatalf("the client's account is not sealed to the machine for its service: %+v", states) + } + if err := busCredentialIssued(ctx, inv, "laptop", m.Module, broker.ProvisionerSecret); err != nil { + t.Fatalf("an issued client account is refused: %v", err) + } + if !runsTheProvisionerClient(m) { + t.Fatal("the module is not read as running the client") + } + unread := m + unread.OwnSecrets = catalogue.OwnSecrets{"broker": {Path: "/run/broker"}} + if runsTheProvisionerClient(unread) { + t.Fatal("a module with nowhere to read the client's account is read as running it") + } +} diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 970e0d53..bc47d401 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -265,6 +265,12 @@ func serve(ctx context.Context) (err error) { // The facts snapshot a merge check is fed (novox/hq to-be 45 §9): kept current while this // controller holds the lease, read by the build seat from the artifact store. go exportingFacts(ctx, open, bus.Conn.ConnectedServerVersion) + // The record mirrors (novox/hq ADR 0289, design 57 F3): the assignments and each registered module's + // declared config on the bus, written from the store only when a record changed, by the lease holder. + // The provisioner's plan of each handed module, as its machine was last sent it (design 57 P6), in the module's + // assignment. + inv.SetProvisioningPlans(inv.SentProvisioningPlans) + go mirroringRecords(ctx, inv, bus.Conn, bus) // Every call carries the lease's epoch, and its record is written only under the lease (novox/hq // to-be 45 §6). link.Calls.UnderLease(func() (uint64, error) { return theLease.epoch(ctx) }) @@ -1695,7 +1701,7 @@ func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body // send that is away look failed: it is said, loudly, and the machine's own record stands. err = inv.RecordSentWith(kept, record.ID, digest, builds, epoch, sent.toldGeneration, inventory.Send{ Sequence: sent.sequence, Epoch: int64(sent.epoch), Sender: sent.sender, Generation: sent.generation, - Digest: digest, Modules: sent.modules}) + Digest: digest, Modules: sent.modules, Provisioning: sent.plans}) var unrecorded *inventory.SendNotRecordedError switch { case errors.As(err, &unrecorded): diff --git a/cmd/mesh-controller/record_mirrors.go b/cmd/mesh-controller/record_mirrors.go new file mode 100644 index 00000000..6889d79e --- /dev/null +++ b/cmd/mesh-controller/record_mirrors.go @@ -0,0 +1,106 @@ +package main + +import ( + "context" + "fmt" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/recordmirror" +) + +// The controller's record mirrors on the bus (novox/hq ADR 0289, design 54, design 57 F3): the assignments, +// and each registered module's declared config, which mesh-config resolves a module's config values from. +// +// **Built from the store, on a tick of their own, by the lease holder alone.** A record changes through a +// verb that runs as a command of its own, a reconcile, a merge's registration or a pending assignment made, +// so no one call site sees every change; what every change has in common is that it is committed to the +// store. So the serving controller reads the store every mirrorEvery and writes only what differs from what +// the bucket holds (ADR 0289 §1): a quiet mesh writes nothing, and a change rolled back is never read. A +// mirror is therefore at most mirrorEvery behind its store. + +// mirrorEvery is how often the mirrors are compared with the store: two small reads of the store. +const mirrorEvery = 10 * time.Second + +// mirroringRecords keeps the mirrors current for as long as this controller serves: ctx ends when it stops. +func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn, announce recordmirror.Announcer) { + api, err := jetstream.New(conn) + if err != nil { + fmt.Printf("the record mirrors are not kept: %v\n", err) + return + } + holds := func() bool { + epoch, err := theLease.epoch(ctx) + return err == nil && epoch > 0 + } + epoch := func() uint64 { + e, _ := theLease.epoch(ctx) + return e + } + sources := map[string]func(context.Context) (map[string][]byte, error){ + recordmirror.Assignments.Name: inv.AssignmentsMirror, + recordmirror.DeclaredConfig.Name: inv.DeclaredConfigMirror, + recordmirror.Predefined.Name: inv.PredefinedMirror, + } + writers := map[string]*recordmirror.Writer{} + failing := map[string]string{} + tick := time.NewTicker(mirrorEvery) + defer tick.Stop() + for { + if holds() { + for _, m := range recordmirror.Mirrors() { + why := mirrorOnce(ctx, api, m, writers, sources[m.Name], announce, epoch, holds) + if why != failing[m.Name] { + if why != "" { + fmt.Printf("the %s mirror cannot be kept: %s\n", m.Name, why) + } else { + fmt.Printf("the %s mirror is kept again\n", m.Name) + } + failing[m.Name] = why + } + } + } + select { + case <-ctx.Done(): + return + case <-tick.C: + } + } +} + +// mirrorOnce brings one mirror to what the store holds, and answers why it could not; empty when it could. +func mirrorOnce(ctx context.Context, api jetstream.JetStream, m recordmirror.Mirror, writers map[string]*recordmirror.Writer, + source func(context.Context) (map[string][]byte, error), announce recordmirror.Announcer, epoch func() uint64, + holds func() bool) string { + reading, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + w := writers[m.Name] + if w == nil { + kv, err := api.KeyValue(reading, m.Bucket) + if err != nil { + return fmt.Sprintf("the bucket %s cannot be read: %v", m.Bucket, err) + } + w = &recordmirror.Writer{Mirror: m, KV: kv, Announce: announce, Epoch: epoch, Holds: holds} + writers[m.Name] = w + } + want, err := source(reading) + if err != nil { + return fmt.Sprintf("the store cannot be read: %v", err) + } + lost := w.LostEvents + r, err := w.Sync(reading, want) + if r.Written+r.Removed > 0 { + fmt.Printf("the %s mirror moved: %d written, %d removed\n", m.Name, r.Written, r.Removed) + } + if w.LostEvents > lost { + fmt.Printf("the %s mirror lost %d event(s): the bucket holds the change, its notice was refused twice\n", + m.Name, w.LostEvents-lost) + } + if err != nil { + return err.Error() + } + return "" +} diff --git a/cmd/mesh-controller/release.go b/cmd/mesh-controller/release.go index 4e2b3634..5751b1cd 100644 --- a/cmd/mesh-controller/release.go +++ b/cmd/mesh-controller/release.go @@ -1013,7 +1013,16 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time. } // **A machine that says nothing on its send is left silent, not failed** (ADR 0353 §4): it is not judged // until it reports, and past ApplySilentAfter it is listed and left. - if !reportedOnSend(byNode[node], node, g) { + outcome, reported := byNode[node].Outcome, reportedOnSend(byNode[node], node, g) + if sent, kept := g.Sent[node]; kept && !gateFirstMachine { + // A module handed to the provisioner is applied only once its client said so too (ADR 0362, issue 840). + carried := make([]string, 0, len(g.Carried)) + for _, c := range g.Carried { + carried = append(carried, c.Module) + } + outcome, reported = provisionedOutcome(node, sent.Only(carried...), byNode[node], reported, provLookup(ctx)) + } + if !reported { if now.Sub(*g.Since) > inventory.ApplySilentAfter { g.Silent = true r.Silent = append(r.Silent, node) @@ -1028,18 +1037,25 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time. if !gateFirstMachine { // **Judged by its report alone** (novox/hq ADR 0360 §7): applied passes; failed or refused is named and // fails the walk, with nothing put back — apply-failing says it. - switch rep := byNode[node]; rep.Outcome { + switch outcome { case inventory.OutcomeApplied: decide(g, inventory.GatePassed, node+" reported it applied its send (novox/hq ADR 0360)", now) passCarried(ctx, open, p, g, "") r.Done = append(r.Done, node) changed = true case inventory.OutcomeFailed, inventory.OutcomeRefused: - decide(g, inventory.GateFailed, fmt.Sprintf("%s %s its send", node, rep.Outcome), now) + decide(g, inventory.GateFailed, fmt.Sprintf("%s %s its send", node, outcome), now) r.Failed = append(r.Failed, node) changed = true - notes = append(notes, fmt.Sprintf("%s %s its send: nothing put back, apply-failing says it", node, rep.Outcome)) + notes = append(notes, fmt.Sprintf("%s %s its send: nothing put back, apply-failing says it", node, outcome)) default: + if strings.HasPrefix(outcome, inventory.OutcomeProvisioningFailed) { + decide(g, inventory.GateFailed, fmt.Sprintf("%s: %s", node, outcome), now) + r.Failed = append(r.Failed, node) + changed = true + notes = append(notes, fmt.Sprintf("%s: %s: nothing put back, apply-failing says it", node, outcome)) + continue + } judging = append(judging, node+": not yet reported on its send") } continue diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index fa523935..46cf11fe 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -1153,7 +1153,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, if err != nil { return false, err } - waiting, failed, silent := walkReports(*p, reports, time.Now().UTC()) + waiting, failed, silent := walkReports(*p, reports, time.Now().UTC(), provLookup(ctx)) if len(waiting) > 0 { note := "sent; waiting for " + strings.Join(waiting, ", ") + " to report on their send" changed := p.State != inventory.PlanRolling || p.Note != note @@ -1189,17 +1189,29 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, // walkReports is, for every machine a walk sent its rest, whether it reported on that send (novox/hq ADR 0360 §4): // those still waiting, those that reported a failed or refused apply, and those silent past ApplySilentAfter, // each sorted. Pure. -func walkReports(p inventory.Plan, reports []inventory.Reported, now time.Time) (waiting, failed, silent []string) { +func walkReports(p inventory.Plan, reports []inventory.Reported, now time.Time, prov inventory.ProvLookup) (waiting, failed, silent []string) { byNode := map[string]inventory.Reported{} for _, r := range reports { byNode[r.Node] = r } + // The modules of this delivery sent to each machine: a machine is judged on those, not on every module its + // declaration hands to the provisioner (ADR 0362 §1-2, issue 840). + carried := map[string][]string{} + for m, s := range p.Modules { + if s == nil { + continue + } + for node := range s.Rest { + carried[node] = append(carried[node], m) + } + } seen := map[string]bool{} for _, s := range p.Modules { if s == nil { continue } for node, sent := range s.Rest { + sent = sent.Only(carried[node]...) if seen[node] { continue } @@ -1208,11 +1220,16 @@ func walkReports(p inventory.Plan, reports []inventory.Reported, now time.Time) if at.IsZero() && s.SentAt != nil { at = *s.SentAt } + // A module handed to the provisioner is applied only once its client said so too (ADR 0362 §1-5). + outcome, reported := provisionedOutcome(node, sent, r, r.At != nil && sent.ReportsOn(r), prov) switch { - case r.At != nil && sent.ReportsOn(r): + case reported: seen[node] = true - if r.Outcome == inventory.OutcomeFailed || r.Outcome == inventory.OutcomeRefused { + switch { + case outcome == inventory.OutcomeFailed || outcome == inventory.OutcomeRefused: failed = append(failed, node) + case strings.HasPrefix(outcome, inventory.OutcomeProvisioningFailed): + failed = append(failed, node+" ("+outcome+")") } case !at.IsZero() && now.Sub(at) > inventory.ApplySilentAfter: seen[node] = true diff --git a/cmd/mesh-controller/release_provisioned_test.go b/cmd/mesh-controller/release_provisioned_test.go new file mode 100644 index 00000000..52241dcf --- /dev/null +++ b/cmd/mesh-controller/release_provisioned_test.go @@ -0,0 +1,172 @@ +package main + +import ( + "context" + "encoding/json" + "slices" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// handingAnchor makes every send to anchor hand the provisioner the given modules (module -> templates digest), and +// the clients' words the given reports (key `node/module`), as the stream would give them (issue 840, ADR 0362). +func handingAnchor(t *testing.T, b *backlogMesh, handed map[string]string) map[string]inventory.ProvReport { + t.Helper() + says := map[string]inventory.ProvReport{} + wasSays := provAppliedOn + provAppliedOn = func(_ context.Context, node, module string) (inventory.ProvReport, bool, error) { + r, ok := says[node+"/"+module] + return r, ok, nil + } + inner := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + got, err := inner(ctx, open, names) + if err != nil { + return got, err + } + for _, n := range names { + if n != "anchor" { + continue + } + inv := open.inventory + s, _, err := inv.SentTo(ctx, n) + if err != nil { + return nil, err + } + builds, _, err := inv.SentBuilds(ctx, n) + if err != nil { + return nil, err + } + plans := map[string]inventory.Provisioning{} + for m, d := range handed { + plans[m] = inventory.Provisioning{Templates: inventory.TemplatesArchive{Digest: d}} + } + raw, err := json.Marshal(plans) + if err != nil { + return nil, err + } + id := nodeID(t, open, n) + if err := inv.RecordSentWith(ctx, id, s.Digest, builds, 0, 0, inventory.Send{Node: id, Provisioning: raw}); err != nil { + return nil, err + } + } + return got, nil + } + t.Cleanup(func() { sendRollout, provAppliedOn = inner, wasSays }) + return says +} + +func stepRelease(t *testing.T, b *backlogMesh, n int) inventory.Plan { + t.Helper() + for i := 0; i < n; i++ { + advancePlans(t.Context(), b.open) + } + return b.release(t) +} + +// Issue 840 (novox/hq ADR 0362 §1-5): the release's rest gate, which every delivery passes since ADR 0360, counts a +// machine's send of a provisioned module applied only once the engine and the machine's provisioner client both said +// so; a failed client is a failed apply named with its reason; none past ApplySilentAfter is silent, never done. +func TestTheRestGateWaitsForTheProvisioner(t *testing.T) { + judgeLeavesAsTheCore(t) + applied := inventory.ProvReport{Status: inventory.ProvApplied, Templates: "t1", At: time.Now()} + + t.Run("a engine applied, the client has not said: no verdict, the release stays open", func(t *testing.T) { + b := aBacklog(t) + handingAnchor(t, b, map[string]string{"app": "t1"}) + p := stepRelease(t, b, 8) + g := p.Release.Gates["anchor"] + if !p.Open() || g == nil || g.Verdict != "" || g.Silent || slices.Contains(p.Release.Done, "anchor") { + t.Fatalf("anchor is judged before its provisioner reported: %s %s %+v", p.State, p.Note, g) + } + if !slices.Contains(p.Release.Done, "laptop") { + t.Fatalf("laptop hands nothing and passes on the engine's report: %+v", p.Release) + } + }) + t.Run("b the client failed: a failed apply named with the reason", func(t *testing.T) { + b := aBacklog(t) + says := handingAnchor(t, b, map[string]string{"app": "t1"}) + says["anchor/app"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t1", Reason: "no value", At: time.Now()} + p := stepRelease(t, b, 8) + g := p.Release.Gates["anchor"] + if g == nil || g.Verdict != inventory.GateFailed || !slices.Contains(p.Release.Failed, "anchor") || + !strings.Contains(p.Note, "provisioning failed (app: no value)") { + t.Fatalf("the failure is not named: %s %q %+v %+v", p.State, p.Note, g, p.Release) + } + }) + t.Run("c no client report past ApplySilentAfter: silent, never done", func(t *testing.T) { + b := aBacklog(t) + handingAnchor(t, b, map[string]string{"app": "t1"}) + p := stepRelease(t, b, 8) + g := p.Release.Gates["anchor"] + if g == nil { + t.Fatalf("no gate: %+v", p.Release) + } + since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute) + g.Since = &since + if err := b.open.inventory.SavePlan(t.Context(), &p); err != nil { + t.Fatal(err) + } + p = stepRelease(t, b, 1) + if !slices.Equal(p.Release.Silent, []string{"anchor"}) || slices.Contains(p.Release.Done, "anchor") { + t.Fatalf("anchor is not silent: %s %s %+v", p.State, p.Note, p.Release) + } + }) + t.Run("d both applied: the gate passes", func(t *testing.T) { + b := aBacklog(t) + says := handingAnchor(t, b, map[string]string{"app": "t1"}) + says["anchor/app"] = applied + p := stepRelease(t, b, 8) + if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") || len(p.Release.Failed) > 0 { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + }) + t.Run("e a send handing nothing passes on the engine's report", func(t *testing.T) { + b := aBacklog(t) + p := stepRelease(t, b, 8) + if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + }) + t.Run("f a module the delivery does not carry does not touch its end", func(t *testing.T) { + // anchor's declaration also hands `same`, rebuilt with the very bytes and so not carried: its failed report + // is none of this delivery's. + b := aBacklog(t) + says := handingAnchor(t, b, map[string]string{"app": "t1", "same": "t9"}) + says["anchor/app"] = applied + says["anchor/same"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t9", Reason: "stuck", At: time.Now()} + p := stepRelease(t, b, 8) + if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") || len(p.Release.Failed) > 0 { + t.Fatalf("another module's failure ended this delivery: %s %s %+v", p.State, p.Note, p.Release) + } + }) +} + +// Issue 840: a walk's end waits on the modules it sent to a machine, not on every module that machine's declaration +// hands to the provisioner. +func TestAWalkEndWaitsOnlyOnTheModulesItSent(t *testing.T) { + now := time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC) + at := now.Add(-10 * time.Second) + handed := map[string]string{"app": "t1", "other": "t2"} + p := inventory.Plan{Modules: map[string]*inventory.PlanModule{"app": {Rest: map[string]inventory.SentDeclaration{ + "ace": {Digest: "d-ace", At: now.Add(-time.Minute), Provisioned: handed}}}}} + reports := []inventory.Reported{{Node: "ace", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-ace"}} + says := map[string]inventory.ProvReport{"ace/app": {Status: inventory.ProvApplied, Templates: "t1", At: at}} + prov := func(node, module string) (inventory.ProvReport, bool) { r, ok := says[node+"/"+module]; return r, ok } + // other has said nothing: not this walk's to wait for. + if waiting, failed, silent := walkReports(p, reports, now, prov); len(waiting)+len(failed)+len(silent) != 0 { + t.Fatalf("missing: waiting %v, failed %v, silent %v", waiting, failed, silent) + } + says["ace/other"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t2", Reason: "stuck", At: at} + if waiting, failed, silent := walkReports(p, reports, now, prov); len(waiting)+len(failed)+len(silent) != 0 { + t.Fatalf("failed: waiting %v, failed %v, silent %v", waiting, failed, silent) + } + // once the walk carries it too, it does. + p.Modules["other"] = &inventory.PlanModule{Rest: p.Modules["app"].Rest} + if _, failed, _ := walkReports(p, reports, now, prov); len(failed) != 1 || !strings.Contains(failed[0], "other: stuck") { + t.Fatalf("a carried module's failure is not named: %v", failed) + } +} diff --git a/cmd/mesh-controller/rollout.go b/cmd/mesh-controller/rollout.go index 81f0c2a6..d1fb3b47 100644 --- a/cmd/mesh-controller/rollout.go +++ b/cmd/mesh-controller/rollout.go @@ -379,6 +379,19 @@ func rolloutMint(ctx context.Context, again bool) error { } modules++ + case broker.KindProvisioner: + // A machine's provisioner client, on an account of its own (novox/hq ADR 0349 clause 11), delivered + // to the module running it. + m, inShelf := shelf[p.Module] + if !inShelf || !runsTheProvisionerClient(m) { + skipped++ + continue + } + if err := issueProvisionerClient(ctx, inv, m, p.Node, known, busAddress); err != nil { + return err + } + modules++ + default: skipped++ } diff --git a/cmd/mesh-controller/rotate.go b/cmd/mesh-controller/rotate.go index f9168ddf..3ecdc25d 100644 --- a/cmd/mesh-controller/rotate.go +++ b/cmd/mesh-controller/rotate.go @@ -6,6 +6,7 @@ import ( "flag" "fmt" "sort" + "strings" "github.com/novox/mesh-controller/internal/inventory" ) @@ -39,6 +40,9 @@ func rotateCommand(ctx context.Context, args []string) error { // of one provision, each with its own credential; one module that leaked its credential (novox/hq // issue 268) is no reason to restart every other one on the machine. module := set.String("module", "", "only this consuming module's credential") + // Why, which the vault asks of every rotation (issue 767) and the hand-act log records; required only where a + // provider's credential is the vault's, so nothing the operator typed before changes meaning. + why := addHandActFlags(set) positionals, err := parseAround(set, args) if err != nil { return err @@ -89,17 +93,6 @@ func rotateCommand(ctx context.Context, args []string) error { // Every machine at both ends, named before anything changes. A person about to rotate a // production credential is entitled to know the blast radius before it is the past tense. - affected := map[string]bool{} - for _, h := range holders { - affected[h.Consumer] = true - affected[h.Provider] = true - } - machines := make([]string, 0, len(affected)) - for name := range affected { - machines = append(machines, name) - } - sort.Strings(machines) - fmt.Printf("rotating %s for %d holder(s):\n", provision, len(holders)) for _, h := range holders { // The module, because a machine may hold several credentials for one provision and @@ -107,6 +100,35 @@ func rotateCommand(ctx context.Context, args []string) error { fmt.Printf(" %s on %s, from %s%s\n", h.ConsumerModule, h.Consumer, h.Provider, asLocal(h.Local)) } + // **The vault's, where the provider is filled the new way** (issue 767, ADR 0349 clause 8): its credential is + // the vault's, so the vault remakes it and its holders take it from there; the controller writes no value of its + // own and sends nothing for it. Which holders go where is read before anything changes. + toVault, today, err := rotateRoutes(holders, + func(provider, provision string) (string, error) { return offeringModuleOn(ctx, provider, provision) }, + filledTheNewWay) + if err != nil { + return fmt.Errorf("nothing was rotated: %w", err) + } + if len(toVault) > 0 && !why.given() { + return errors.New("not rotated: the vault holds this credential and rotates it only with a reason, " + + "`--why \"…\"`; nothing was asked") + } + if why.given() { + why.record(ctx, "rotate", append([]string{provision}, rotateNarrowArgs(*only, *module)...)) + } + for _, g := range toVault { + words, err := rotateProvisionInVault(theVault(ctx), g, *only, *module, actingAs(), *why.why) + if err != nil { + return err + } + fmt.Println(words) + } + if len(today) == 0 { + return nil + } + holders = today + machines := machinesOf(holders) + for _, h := range holders { if err := inv.RotateSecret(ctx, h.Provision, h.Consumer, h.ConsumerModule, h.Provider, h.Local); err != nil { // Partly rotated, and said so plainly. What is gone is remade on the next push, so @@ -137,6 +159,21 @@ func rotateCommand(ctx context.Context, args []string) error { return nil } +// machinesOf is every machine at both ends of these holders, in order. +func machinesOf(holders []inventory.Holder) []string { + affected := map[string]bool{} + for _, h := range holders { + affected[h.Consumer] = true + affected[h.Provider] = true + } + machines := make([]string, 0, len(affected)) + for name := range affected { + machines = append(machines, name) + } + sort.Strings(machines) + return machines +} + // ofModule is the holders whose consuming module is this one; all of them when none is named. func ofModule(holders []inventory.Holder, module string) []inventory.Holder { if module == "" { @@ -165,3 +202,96 @@ func asLocal(local string) string { } return " (as " + local + ")" } + +// rotateGroup is the holders of one provider's credential for one provision whose provider is filled the new way: +// the vault's to rotate (issue 767). +type rotateGroup struct { + Provider, Provision, Module string + Holders []inventory.Holder +} + +// rotateRoutes says which holders the vault rotates, grouped by provider and provision, and which today's path +// does: the vault's are those whose provider's offering module on its machine is filled the new way (the switch +// `accept --provider` reads too). Anything that cannot be read is an error, never today's path by default: a value +// remade where its holders no longer look would be reported rotated and change nothing. +func rotateRoutes(holders []inventory.Holder, offering func(provider, provision string) (string, error), + ofTheVault vaultSwitch) ([]rotateGroup, []inventory.Holder, error) { + type key struct{ provider, provision string } + decided := map[key]int{} // index into groups, or -1 for today's path + var groups []rotateGroup + var today []inventory.Holder + for _, h := range holders { + k := key{h.Provider, h.Provision} + i, seen := decided[k] + if !seen { + module, err := offering(h.Provider, h.Provision) + if err != nil { + return nil, nil, fmt.Errorf("which module offers %s on %s cannot be read: %w", h.Provision, h.Provider, err) + } + ours := false + if module != "" { + if ours, err = ofTheVault(module); err != nil { + return nil, nil, err + } + } + i = -1 + if ours { + i = len(groups) + groups = append(groups, rotateGroup{Provider: h.Provider, Provision: h.Provision, Module: module}) + } + decided[k] = i + } + if i < 0 { + today = append(today, h) + continue + } + groups[i].Holders = append(groups[i].Holders, h) + } + return groups, today, nil +} + +// rotateNarrowArgs is --consumer and --module as typed, for the hand-act log. +func rotateNarrowArgs(consumer, module string) []string { + var out []string + if consumer != "" { + out = append(out, "--consumer", consumer) + } + if module != "" { + out = append(out, "--module", module) + } + return out +} + +// rotateProvisionInVault asks the vault to rotate one provider's credential for a provision (issue 767): a shared +// value is made anew for every holder, and the vault refuses it for one consumer alone; for a provider whose +// consumers keep a login each (ADR 0158 §11), consumer and module name whose own login is remade, and only theirs. +// The vault says secret-changed and each holder takes the new value from it; nothing is sealed or sent here. +func rotateProvisionInVault(ask vaultAsker, g rotateGroup, consumer, module, by, why string) (string, error) { + if strings.TrimSpace(why) == "" { + return "", errors.New("not rotated: the vault rotates a credential only with a reason, `--why \"…\"`; nothing was asked") + } + args := provisionArgs(g.Provider, g.Provision) + args["by"], args["why"] = by, strings.TrimSpace(why) + if consumer != "" { + args["consumer"] = consumer + } + if module != "" { + args["consumer_module"] = module + } + changed, err := endVaultCall(ask, "rotate", args) + if err != nil { + return "", fmt.Errorf("%s from %s not rotated, and nothing else was: %w", g.Provision, g.Provider, err) + } + if len(changed.Each) > 0 { + each := make([]string, 0, len(changed.Each)) + for _, c := range changed.Each { + each = append(each, fmt.Sprintf("%s (generation %d)", c.Name, c.Generation)) + } + return fmt.Sprintf("rotated in the vault, each consumer's own login of %s from %s, asked by %s: %s; the values "+ + "are sealed and not shown, and each holder takes its own from the vault", g.Provision, g.Provider, + changed.By, strings.Join(each, ", ")), nil + } + return fmt.Sprintf("rotated %s in the vault (generation %d), one value for every holder of %s from %s, asked by "+ + "%s; the value is sealed and not shown, and each holder takes it from the vault", changed.Name, + changed.Generation, g.Provision, g.Provider, changed.By), nil +} diff --git a/cmd/mesh-controller/rotate_vault_test.go b/cmd/mesh-controller/rotate_vault_test.go new file mode 100644 index 00000000..df0fad01 --- /dev/null +++ b/cmd/mesh-controller/rotate_vault_test.go @@ -0,0 +1,124 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// `rotate ` reaches the vault for a provider filled the new way (issue 767), and the controller writes no +// value of its own for it: its holders go to the vault, named by provider and provision, and no other holder does. +func TestARotationOfAProvisionTheVaultOwnsGoesToTheVaultAndNotTheControllersStore(t *testing.T) { + holders := []inventory.Holder{ + {Provision: "postgres-database", Consumer: "g14", ConsumerModule: "gitea", Provider: "novox"}, + {Provision: "postgres-database", Consumer: "ace", ConsumerModule: "umami", Provider: "novox"}, + {Provision: "postgres-database", Consumer: "ace", ConsumerModule: "n8n", Provider: "ace"}, + } + offering := func(provider, provision string) (string, error) { + if provider == "novox" { + return "postgres", nil + } + return "postgres-old", nil + } + var asked []string + moved := func(module string) (bool, error) { + asked = append(asked, module) + return module == "postgres", nil + } + vault, today, err := rotateRoutes(holders, offering, moved) + if err != nil { + t.Fatal(err) + } + if len(vault) != 1 || vault[0].Provider != "novox" || vault[0].Provision != "postgres-database" || + vault[0].Module != "postgres" || len(vault[0].Holders) != 2 { + t.Fatalf("to the vault: %+v", vault) + } + if len(today) != 1 || today[0].Provider != "ace" { + t.Fatalf("today's path rotates a holder the vault owns, or misses one it does not: %+v", today) + } + if fmt.Sprint(asked) != "[postgres postgres-old]" { + t.Fatalf("the switch was read for %v, not the offering module", asked) + } +} + +// A switch that cannot be read, or a provider whose offering module cannot be found, routes nothing: an error +// before anything changes, never today's path by default. +func TestARotationWhoseOwnerCannotBeReadChangesNothing(t *testing.T) { + holders := []inventory.Holder{{Provision: "postgres-database", Consumer: "g14", ConsumerModule: "gitea", + Provider: "novox"}} + broken := func(string) (bool, error) { return false, errors.New("the catalogue is unreadable") } + if _, _, err := rotateRoutes(holders, func(string, string) (string, error) { return "postgres", nil }, broken); err == nil { + t.Fatal("an unreadable switch fell back to today's path") + } + fine := func(string) (bool, error) { return true, nil } + if _, _, err := rotateRoutes(holders, func(string, string) (string, error) { return "", errors.New("no store") }, + fine); err == nil { + t.Fatal("an unreadable provider fell back to today's path") + } +} + +// What the vault is told: the provider and provision, and --consumer and --module as the one consumer whose own +// login is remade; nothing named when the operator named nobody, so a shared value is rotated for every holder. +func TestTheVaultIsToldTheProvisionAndTheConsumerTheOperatorNamed(t *testing.T) { + group := rotateGroup{Provider: "novox", Provision: "postgres-database", Module: "postgres"} + for _, c := range []struct { + consumer, module string + want, absent []string + }{ + {"", "", []string{`"provider":"novox"`, `"provision":"postgres-database"`, `"why":"yearly"`, + `"by":"g14/claude-code"`}, []string{`"consumer"`, `"consumer_module"`}}, + {"g14", "", []string{`"consumer":"g14"`}, []string{`"consumer_module"`}}, + {"", "gitea", []string{`"consumer_module":"gitea"`}, []string{`"consumer":`}}, + {"g14", "gitea", []string{`"consumer":"g14"`, `"consumer_module":"gitea"`}, nil}, + } { + var said string + ask := func(verb string, args map[string]any) (json.RawMessage, error) { + raw, _ := json.Marshal(map[string]any{"verb": verb, "args": args}) + said = string(raw) + return json.Marshal(vaultChanged{Name: "provision/novox/postgres-database", By: "g14/claude-code", + Each: []vaultChanged{{Name: "pair/novox/postgres-database/g14/gitea", Generation: 3}}}) + } + words, err := rotateProvisionInVault(ask, group, c.consumer, c.module, "g14/claude-code", "yearly") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(said, `"verb":"rotate"`) { + t.Fatalf("%+v: the vault was asked %s", c, said) + } + for _, w := range c.want { + if !strings.Contains(said, w) { + t.Errorf("%+v: the vault was not told %s: %s", c, w, said) + } + } + for _, a := range c.absent { + if strings.Contains(said, a) { + t.Errorf("%+v: the vault was told %s: %s", c, a, said) + } + } + if !strings.Contains(words, "pair/novox/postgres-database/g14/gitea (generation 3)") { + t.Errorf("%+v: the operator was not told whose login was remade: %s", c, words) + } + } +} + +// The vault rotates only with a why: asked of the vault without one, nothing is sent; and the vault's refusal (a +// shared value named for one consumer) is the operator's answer, not a rotation in the controller's store. +func TestARotationInTheVaultSaysWhyAndPassesOnItsRefusal(t *testing.T) { + group := rotateGroup{Provider: "anchor", Provision: "stand-in-api", Module: "stand-in"} + sent := 0 + ask := func(string, map[string]any) (json.RawMessage, error) { + sent++ + return nil, errors.New("not rotated: provision/anchor/stand-in-api is one value every holder shares") + } + if _, err := rotateProvisionInVault(ask, group, "", "", "g14/claude-code", " "); err == nil || sent != 0 { + t.Fatalf("a rotation without a why reached the vault: %v", err) + } + if _, err := rotateProvisionInVault(ask, group, "g14", "", "g14/claude-code", "suspected"); err == nil || + !strings.Contains(err.Error(), "every holder shares") { + t.Fatalf("the vault's refusal was not said: %v", err) + } +} diff --git a/cmd/mesh-controller/runtime_health_test.go b/cmd/mesh-controller/runtime_health_test.go new file mode 100644 index 00000000..8bc63491 --- /dev/null +++ b/cmd/mesh-controller/runtime_health_test.go @@ -0,0 +1,55 @@ +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/link" +) + +// novox/hq ADR 0366 §3: a module's own runtime stated unhealthy by its runtime-health raises the module's usual +// condition, module...unhealthy, through ADR 0240's path, saying the tool's reason, and the first +// statement that says it healthy clears it. +func TestAnUnhealthyOwnRuntimeRaisesTheModulesUsualCondition(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv, k := open.inventory, conditionsFrom + const key = "module.mesh-vault.anchor.unhealthy" + said := func(at time.Time, state, reason string) link.Health { + return link.Health{Contract: link.LivenessContract, At: at, Resources: []link.ResourceHealth{{ + Module: "mesh-vault", Resource: "mesh-vault.own-runtime", Kind: "process", Target: "mesh-vault-runtime.service", + State: state, Reason: reason, Since: at, Check: "tool"}}} + } + why := "re-sealing to the operator key failed on 3 passes in a row: the operator key is not readable" + for i := 0; i < 2; i++ { + at := h0.Add(time.Duration(i) * time.Minute) + if err := stateHealth(ctx, inv, k, "anchor", said(at, link.StateUnhealthy, why), at); err != nil { + t.Fatal(err) + } + } + c, had, err := k.Get(ctx, key) + if err != nil || !had { + t.Fatalf("two unhealthy statements of its own runtime raised no %s: %v", key, err) + } + evidence := "" + for _, e := range c.Evidence { + evidence += e.Said + "\n" + } + if !strings.Contains(c.Summary+evidence, "operator key is not readable") { + t.Errorf("the condition does not say the tool's reason: %s / %s", c.Summary, evidence) + } + at := h0.Add(3 * time.Minute) + if err := stateHealth(ctx, inv, k, "anchor", said(at, link.StateHealthy, ""), at); err != nil { + t.Fatal(err) + } + list, err := k.Open(ctx) + if err != nil { + t.Fatal(err) + } + for _, o := range list { + if o.Key == key { + t.Fatalf("a healthy statement did not clear %s", key) + } + } +} diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index c0c222e3..101db41c 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -730,6 +730,38 @@ func (a *verbArguments) commandLine() ([]string, error) { return nil, fmt.Errorf("%w: letting go of artifacts is a hand act, which says why. Nothing was done", err) } return append(argv, "--confirm", "--why", why), nil + // The move of placement out of the settings layers (issue 736), a dry run unless confirmed. + case "move-placement": + argv := []string{"move-placement", "--json"} + why := str("why") + if !on("confirm") { + if why != "" { + return nil, errors.New("move-placement takes why only with confirm: without confirm nothing is " + + "moved, and a reason for nothing would be recorded nowhere. Nothing was done") + } + return argv, nil + } + if err := need("why"); err != nil { + return nil, fmt.Errorf("%w: moving the settings layers' placement is an act by hand, which says why. "+ + "Nothing was done", err) + } + return append(argv, "--confirm", "--why", why), nil + // The move of config into mesh-config (issue 664), a dry run unless confirmed. + case "move-config": + argv := []string{"move-config", "--json"} + why := str("why") + if !on("confirm") { + if why != "" { + return nil, errors.New("move-config takes why only with confirm: without confirm nothing is set, " + + "and a reason for nothing would be recorded nowhere. Nothing was done") + } + return argv, nil + } + if err := need("why"); err != nil { + return nil, fmt.Errorf("%w: moving the settings layers into mesh-config is an act by hand, which says why. "+ + "Nothing was done", err) + } + return append(argv, "--confirm", "--why", why), nil case "images": if err := need("node"); err != nil { return nil, err @@ -837,6 +869,13 @@ func (a *verbArguments) commandLine() ([]string, error) { return nil, err } return []string{"secret", "ask", str("node"), str("module"), str("secret"), "--at", str("at")}, nil + case "secret-move": + // The move of the controller's secrets into the vault (design 57's V3, issue 676): no value in the arguments + // or the answer, a machine's name at most. + if n := str("node"); n != "" { + return []string{"secret", "move", n}, nil + } + return []string{"secret", "move"}, nil case "issued": // The credential a provider's software issued, taken from the provider (novox/hq ADR 0305): no value in // the arguments, the provision alone. @@ -866,6 +905,13 @@ func (a *verbArguments) commandLine() ([]string, error) { if m := str("module"); m != "" { argv = append(argv, "--module", m) } + // Why, which the vault asks of a rotation of a credential it holds (issue 767); a cause only beside it. + if w := str("why"); w != "" { + argv = append(argv, "--why", w) + if c := str("cause"); c != "" { + argv = append(argv, "--cause", c) + } + } return argv, nil } _, node := a.given["node"] @@ -1036,6 +1082,10 @@ var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "co "hand-acts": true, "durations": true, "conditions": true, "doctor": true, "retire": true, "cleanup": true, "data": true, // What the records say the registries may keep (novox/hq ADR 0251). "artifacts": true, "collect": true, "images": true, "mirrors": true, + // The move of placement says each layer it moved (issue 736). + "move-placement": true, + // The move of config says each value it set and where from (issue 664). + "move-config": true, // The delivery's owner's verbs answer JSON where they read (plan, order, check, walks) — novox/hq ADR 0239. "delivery": true, // The build queue, each ask with its position among the waiting (novox/hq issue 455). @@ -1081,6 +1131,10 @@ func repairingCommand(argv []string) string { return "collect" case argv[0] == "mirrors" && slices.Contains(argv, "--confirm"): return "mirrors" + case argv[0] == "move-placement" && slices.Contains(argv, "--confirm"): + return "move-placement" + case argv[0] == "move-config" && slices.Contains(argv, "--confirm"): + return "move-config" } return "" } @@ -1732,6 +1786,16 @@ func takenFromTheProvider(argv []string) bool { !strings.HasPrefix(argv[2], "-") } +// movedByTheirMachines is exactly the line the `secret-move` verb composes: `secret move`, or `secret move ` +// (design 57's V3, issue 676). It carries no value and answers none: each machine opens what was sealed to it and +// hands it to the vault, which never replaces a value it holds. +func movedByTheirMachines(argv []string) bool { + if len(argv) < 2 || argv[0] != "secret" || argv[1] != "move" { + return false + } + return len(argv) == 2 || (len(argv) == 3 && argv[2] != "" && !strings.HasPrefix(argv[2], "-")) +} + // terminalOnly refuses, through any verb, a command that is the operator's at the controller's terminal // alone (novox/hq ADR 0266). **Every `node` subcommand that is not a read**: `node account` and // `node agent-account` above all. Whoever may call a verb includes agents, and an agent that named itself @@ -1748,7 +1812,8 @@ func terminalOnly(argv []string) error { // Of a secret's commands only rotation, which seals the new value to the machine that uses it, and the // `give` verb's own line: an own secret typed by the operator into the desk's hidden prompt, sealed to this // call and then to the module's machine, so no value travels in the verb or its answer (hq ADR 0259 §10). - if argv[0] == "secret" && (len(argv) < 2 || argv[1] != "rotate") && !givenAtTheDesk(argv) && !takenFromTheProvider(argv) { + if argv[0] == "secret" && (len(argv) < 2 || argv[1] != "rotate") && !givenAtTheDesk(argv) && !takenFromTheProvider(argv) && + !movedByTheirMachines(argv) { return terminalRefusal("secret %s is run at the controller's terminal only, never through a verb: accepting, "+ "recovering or exporting a secret hands it to whoever asks, and that includes agents (novox/hq ADR "+ "0266). Nothing was done", strings.Join(argv[1:], " ")) diff --git a/cmd/mesh-controller/seatverbs_schema_test.go b/cmd/mesh-controller/seatverbs_schema_test.go index 61f3bad9..a5c489dd 100644 --- a/cmd/mesh-controller/seatverbs_schema_test.go +++ b/cmd/mesh-controller/seatverbs_schema_test.go @@ -285,6 +285,8 @@ var accountedFlags = map[string]map[string]string{ "collect": {"json": "set by the verb: the answer is data"}, "images": {"json": "set by the verb: the answer is data"}, "mirrors": {"json": "set by the verb: the answer is data"}, + "move-placement": {"json": "set by the verb: the answer is data"}, + "move-config": {"json": "set by the verb: the answer is data"}, "conditions history": {"json": "set by the verb: the answer is data"}, "conditions show": {"json": "set by the verb: the answer is data"}, "healers": {"json": "set by the verb: the answer is data"}, diff --git a/cmd/mesh-controller/secret.go b/cmd/mesh-controller/secret.go index 85b7b20b..4500f756 100644 --- a/cmd/mesh-controller/secret.go +++ b/cmd/mesh-controller/secret.go @@ -53,6 +53,9 @@ func secretCommand(ctx context.Context, args []string) error { return secretRecover(ctx, args[1:]) case "export": return secretExport(ctx, args[1:]) + case "move": + // The controller's secrets moved into the vault, each by the machine it was sealed to (design 57's V3). + return secretMove(ctx, args[1:]) default: return errors.New(secretUsage) } @@ -111,7 +114,12 @@ func secretCommand(ctx context.Context, args []string) error { if value == "" { return errors.New("there is nothing to seal") } - return acceptForProvision(ctx, node, module, name, value) + // The provider's module decides the path: filled the new way, the vault keeps the provision's credential. + return bySwitch(module, filledTheNewWay, + func() error { return acceptForProvision(ctx, node, module, name, value) }, + func() error { + return said(acceptIntoVault(theVault(ctx), provisionArgs(node, name), value, actingAs())) + }) } if *desk != "" { if *from != "" || *provider != "" { @@ -128,23 +136,112 @@ func secretCommand(ctx context.Context, args []string) error { if value == "" { return errors.New("there is nothing to seal") } + // The module whose secrets hold the value decides the path: with --provider, the value is the provider's one + // credential for the provision, so the provider's module decides, as for an outside offer above. + decides := module + if *provider != "" { + if decides, err = offeringModuleOn(ctx, *provider, name); err != nil { + return err + } + } + return bySwitch(decides, filledTheNewWay, + func() error { return acceptKept(ctx, node, module, name, *provider, *local, value) }, + func() error { + // Sealed here to the vault's key for this one call; only the sealed box leaves this process. + target := ownSecretArgs(node, module, name) + if *provider != "" { + if *local != "" { + return errors.New("--local names a credential inside the module; the vault keeps one credential " + + "per provider and provision, so it takes no --local: nothing was sealed") + } + target = provisionArgs(*provider, name) + } + if *provider != "" { + return said(acceptIntoVault(theVault(ctx), target, value, actingAs())) + } + return giveIntoVault(ctx, node, module, name, target, value) + }) +} +// announceTheGiven is the announcement of a secret given at the terminal (announceSecretGiven, the step acceptKept +// runs); a variable so a test can make it fail. +var announceTheGiven = announceSecretGiven + +// giveIntoVault is `secret accept` of a module's own secret for a module filled the new way, under the same +// N1-give rule as acceptKept, through the same steps (keepGiven, announceSecretGiven): a module that runs as an +// account of its own is announced on the operator's channels before the vault is asked, and when the announcement +// fails nothing is sealed and the vault is never called; any other is sealed first and announced after. +func giveIntoVault(ctx context.Context, node, module, name string, target map[string]any, value string) error { + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + trusted, err := open.inventory.RunsAsItsOwnAccount(ctx, module) + if err != nil { + return err + } + var words string + askedBy := link.Caller() + _, unannounced, err := keepGiven(trusted, + func() error { + return announceTheGiven(ctx, node, module, name, "at the controller's terminal", askedBy) + }, + func() (bool, error) { + var err error + words, err = acceptIntoVault(theVault(ctx), target, value, actingAs()) + return false, err + }) + if err != nil { + if trusted && unannounced != nil { + return fmt.Errorf("%s runs as an account of its own, and the change of its %s could not be announced on "+ + "your channels first, so nothing was sealed and the vault was not asked: %w", module, name, err) + } + return err + } + fmt.Println(words) + if unannounced != nil { + fmt.Printf(" this change could NOT be announced on the operator's channels: %v\n", unannounced) + } + // The giving is recorded, never the value (novox/hq ADR 0380 §6), as acceptKept records its own. + act := terminalGiveHandAct(node, module, name, askedBy) + if err := onTheBus(func(conn *nats.Conn) error { + _, err := link.RecordHandAct(ctx, conn, act) + return err + }); err != nil { + fmt.Printf(" this act could NOT be recorded in the hand-act log, and is done anyway: %v\n", err) + } + return nil +} + +// said prints what was done, or answers why not. +func said(words string, err error) error { + if err != nil { + return err + } + fmt.Println(words) + return nil +} + +// acceptKept is today's `secret accept`: the value sealed into the controller's store, for a module whose secrets +// are filled the old way. +func acceptKept(ctx context.Context, node, module, name, provider, local, value string) error { open, err := openStores(ctx) if err != nil { return err } defer open.Close() - if *provider != "" { + if provider != "" { // Into the pair, not into the module's own secrets: what the provider is asked to create // and what the consumer reads are the same value, and neither end can be told a different // one later without the other (novox/hq 04-ISSUES/070). - if err := open.inventory.AcceptSecretForPair(ctx, name, node, module, *provider, *local, value); err != nil { + if err := open.inventory.AcceptSecretForPair(ctx, name, node, module, provider, local, value); err != nil { return err } - fmt.Printf("%s on %s now holds %q from %s%s, sealed to both machines.\n", module, node, name, *provider, asLocal(*local)) + fmt.Printf("%s on %s now holds %q from %s%s, sealed to both machines.\n", module, node, name, provider, asLocal(local)) fmt.Printf(" the mesh cannot read it back, will not replace it with one of its own, and will not rotate it\n") - fmt.Printf(" run `push %s` and `push %s` to send it\n", *provider, node) + fmt.Printf(" run `push %s` and `push %s` to send it\n", provider, node) return nil } // A trusted party's secret is announced before it is kept (the confirmation review of 2026-10-09, N1-give): @@ -221,7 +318,8 @@ const secretUsage = "secret rotate [--why [--cause "secret accept [--from | --at-desk ] [--provider [--local ]]\n" + "secret accept [--from ] (a credential issued outside the mesh, for every consumer)\n" + "secret recover --key [--out ] [--from-export ] [--provider ]\n" + - "secret export [--out ]" + "secret export [--out ]\n" + + "secret move [] (every secret the controller holds, moved into the vault by the machine it was sealed to)" // secretRecover is break-glass: a secret opened with the operator's key, written to a file. // @@ -500,6 +598,26 @@ func secretRotate(ctx context.Context, args []string) error { return errors.New(secretUsage) } node, module, name := rest[0], rest[1], rest[2] + return bySwitch(module, filledTheNewWay, + func() error { return rotateKept(ctx, node, module, name, why) }, + func() error { + reason := "" + if why.why != nil { + reason = strings.TrimSpace(*why.why) + } + words, err := rotateInVault(theVault(ctx), ownSecretArgs(node, module, name), actingAs(), reason) + if err != nil { + return err + } + why.record(ctx, "secret rotate", []string{node, module, name}) + fmt.Println(words) + return nil + }) +} + +// rotateKept is today's `secret rotate`: made anew in the controller's store and sent, for a module whose +// secrets are filled the old way. +func rotateKept(ctx context.Context, node, module, name string, why handActFlags) error { open, err := openStores(ctx) if err != nil { return err diff --git a/cmd/mesh-controller/secret_move.go b/cmd/mesh-controller/secret_move.go new file mode 100644 index 00000000..9c13e817 --- /dev/null +++ b/cmd/mesh-controller/secret_move.go @@ -0,0 +1,336 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + "sync" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// `secret move` — the controller's secrets move into the vault (novox/hq ADR 0349 clause 7; design 57's V3, issue 676; +// the operator's decision of 2026-10-14 on that issue). Run once, after the feature's delivery. +// +// **The controller cannot open what it holds, so each machine moves its own.** For every machine — or the ones named — +// the controller reads every row sealed to it, as the store holds it, and asks that machine's node-engine to move each +// one, signed (link.AskMoveSecret): the engine opens it with its sealing key and hands it to the vault's move-in, sealed +// to the call's key, keeping its origin and generation. The vault takes it from that engine alone, never over a value +// it holds, and reads back what it kept to the engine, which compares; the engine answers only then. What moved is +// recorded here, never a value. Nothing is rotated, and nothing points at the vault because of this: a module's +// secrets are filled from the vault only once it moves (V4, issue 647). +// +// **A machine asleep is moved when it is next heard from**: once the move has started, a machine that reports is moved +// in the background (moveWhenHeard), and what is left of it is said on the next `secret move`. +// +// The controller's record of asks at a desk, which holds no value, is copied as it is, once (`move-asks`). + +const secretMoveUsage = "secret move [] — move every secret the controller holds into the vault, as it is: each " + + "machine's node-engine opens what was sealed to it and hands it to the vault, which reads it back; no value is " + + "rotated, and a machine not answering is moved when it is next heard from" + +// moveAsk is the ask of one row of a machine's engine. +type moveAsk func(row inventory.SecretToMove, ask link.MoveSecretAsk) (link.MoveSecretAnswer, error) + +// nodeMove is what the move did on one machine. +type nodeMove struct { + Node string + Moved int + Already int + // Refused are the rows not moved, each with why; Changed the rows changed in the store after they moved. + Refused []string + Changed []string + // Unreached is why the machine's engine could not be asked: it is moved when next heard from. + Unreached error + // Left is how many rows were not asked because the machine was not reached. + Left int + // Failed is why its rows could not be read. + Failed error +} + +// done says nothing is left to move on the machine. +func (m nodeMove) done() bool { + return m.Failed == nil && m.Unreached == nil && len(m.Refused) == 0 +} + +func (m nodeMove) String() string { + var b strings.Builder + if m.Failed != nil { + fmt.Fprintf(&b, "%s: its rows could not be read, nothing was moved: %v\n", m.Node, m.Failed) + return b.String() + } + fmt.Fprintf(&b, "%s: %d moved into the vault and read back the same, %d there already", m.Node, m.Moved, m.Already) + if n := len(m.Refused); n > 0 { + fmt.Fprintf(&b, ", %d not moved", n) + } + b.WriteString("\n") + for _, r := range m.Refused { + fmt.Fprintf(&b, " not moved: %s\n", r) + } + for _, c := range m.Changed { + fmt.Fprintf(&b, " %s was changed in the controller's store after it moved; the vault keeps what it took, "+ + "and the move never replaces it\n", c) + } + if m.Unreached != nil { + fmt.Fprintf(&b, " %d left: %s was not reached (%v); it is moved when it is next heard from\n", m.Left, m.Node, + m.Unreached) + } + return b.String() +} + +// moveRowAsk is a row as its machine's engine is asked to move it. +func moveRowAsk(r inventory.SecretToMove) link.MoveSecretAsk { + a := link.MoveSecretAsk{Origin: r.Origin, Generation: r.Generation, MadeAt: r.MadeAt.UTC(), Sealed: r.Sealed, + SealedTo: r.SealedTo} + if r.Own() { + a.Module, a.Name = r.Module, r.Name + } else { + a.Provider, a.Provision, a.Consumer, a.ConsumerModule, a.Local = r.Provider, r.Provision, r.Consumer, + r.ConsumerModule, r.Local + } + return a +} + +// moveNode moves every row sealed to one machine that has not moved, one ask each, and records what the vault kept. +// The first ask the engine does not answer ends the machine's turn: the rest wait for it to be heard from again. +func moveNode(node string, rows func(string) ([]inventory.SecretToMove, error), ask moveAsk, + record func(inventory.SecretToMove, int) error) nodeMove { + m := nodeMove{Node: node} + all, err := rows(node) + if err != nil { + m.Failed = err + return m + } + for n, r := range all { + switch { + case r.Moved != nil && r.Moved.Digest == r.Digest: + m.Already++ + continue + case r.Moved != nil: + m.Changed = append(m.Changed, r.Secret) + continue + case !r.Current: + m.Refused = append(m.Refused, fmt.Sprintf("%s is sealed to an earlier sealing key of %s, which no longer "+ + "opens it: nothing can move it; it moves once it is made or accepted again", r.Secret, node)) + continue + } + answer, err := ask(r, moveRowAsk(r)) + if err != nil { + m.Unreached, m.Left = err, len(all)-n + return m + } + if answer.Refused != "" { + m.Refused = append(m.Refused, fmt.Sprintf("%s: %s", r.Secret, answer.Refused)) + continue + } + if answer.Secret != r.Secret { + m.Refused = append(m.Refused, fmt.Sprintf("%s: the vault kept it as %s, not under the name the move "+ + "gives it", r.Secret, answer.Secret)) + continue + } + if err := record(r, answer.Generation); err != nil { + m.Refused = append(m.Refused, fmt.Sprintf("%s is in the vault, but that it moved could not be recorded: %v", + r.Secret, err)) + continue + } + if answer.Unchanged { + m.Already++ + } else { + m.Moved++ + } + } + return m +} + +// movedAsksAtOnce is how many rows of the record of asks one call to the vault carries (mesh-vault MovedAsksAtOnce). +const movedAsksAtOnce = 500 + +// moveAsks copies the controller's record of asks into the vault, in parts, and answers how many it copied. +func moveAsks(asks []inventory.SecretAsk, vault vaultAsker) (int, error) { + copied := 0 + for start := 0; start < len(asks); start += movedAsksAtOnce { + end := min(start+movedAsksAtOnce, len(asks)) + var rows []map[string]any + for _, a := range asks[start:end] { + row := map[string]any{"id": a.ID, "node": a.Node, "module": a.Module, "name": a.Name, "desk": a.Desk, + "asked_by": a.AskedBy, "opened_at": a.OpenedAt.UTC().Format(time.RFC3339Nano)} + if a.EndedAt != nil { + row["ended_at"], row["outcome"] = a.EndedAt.UTC().Format(time.RFC3339Nano), a.Outcome + } + rows = append(rows, row) + } + raw, err := vault("move-asks", map[string]any{"asks": rows}) + if err != nil { + return copied, fmt.Errorf("the vault did not take the record of asks after %d rows: %w", copied, err) + } + var said struct { + Copied int `json:"copied"` + Already int `json:"already"` + } + _ = json.Unmarshal(raw, &said) + copied += said.Copied + said.Already + } + return copied, nil +} + +// moveOnTheBus is the move's ask over the bus, signed with the mesh's key. +func moveOnTheBus(ctx context.Context, open *stores, conn *nats.Conn, by string) (moveAsk, error) { + ident, err := open.Identity(ctx) + if err != nil { + return nil, fmt.Errorf("the mesh's signing key cannot be read, so nothing was asked: %w", err) + } + return func(r inventory.SecretToMove, row link.MoveSecretAsk) (link.MoveSecretAnswer, error) { + return link.AskMoveSecret(ctx, conn, ident, r.Node, by, row, link.MoveSecretWithin) + }, nil +} + +// secretMove is `secret move []`: the record of asks copied once, then every machine's rows — or one's — and +// what moved said per machine. Any row not moved fails the command, said by name. +func secretMove(ctx context.Context, args []string) error { + if len(args) > 1 || (len(args) == 1 && (args[0] == "" || strings.HasPrefix(args[0], "-"))) { + return errors.New(secretMoveUsage) + } + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + inv := open.inventory + by := actingAs() + if by == "" { + by = handOverBy() + } + move, err := inv.StartSecretMove(ctx, by) + if err != nil { + return err + } + var nodes []string + if len(args) == 1 { + if _, err := inv.NodeByName(ctx, args[0]); err != nil { + return fmt.Errorf("nothing was moved: %w", err) + } + nodes = args + } else { + all, err := inv.Nodes(ctx) + if err != nil { + return err + } + for _, n := range all { + nodes = append(nodes, n.Name) + } + } + fmt.Printf("the move of the secrets into the vault, started %s by %s\n", move.StartedAt.Local().Format("2006-01-02 15:04"), + move.StartedBy) + if move.AsksMovedAt == nil { + asks, err := inv.SecretAsks(ctx, time.Time{}) + if err != nil { + return err + } + copied, err := moveAsks(asks, theVault(ctx)) + if err != nil { + return fmt.Errorf("the record of asks was not copied, and no secret was moved: %w", err) + } + if err := inv.RecordAsksMoved(ctx); err != nil { + return err + } + fmt.Printf("the record of asks at a desk: %d rows copied into the vault as they are\n", copied) + } + failed := 0 + err = onTheBus(func(conn *nats.Conn) error { + ask, err := moveOnTheBus(ctx, open, conn, "the move of the secrets, by "+by) + if err != nil { + return err + } + for _, node := range nodes { + m := moveNode(node, func(n string) ([]inventory.SecretToMove, error) { return inv.SecretsToMoveFrom(ctx, n) }, + ask, func(r inventory.SecretToMove, g int) error { return inv.RecordSecretMoved(ctx, r, g) }) + fmt.Print(m.String()) + if m.Failed != nil || len(m.Refused) > 0 { + failed++ + } + } + return nil + }) + if err != nil { + return err + } + if failed > 0 { + return fmt.Errorf("on %d machine(s) a secret was not moved, said above by name", failed) + } + return nil +} + +// moveHeard is the machines being moved in the background now, and when each was last moved, so a machine that reports +// often is not asked again before moveHeardEvery has passed. +var moveHeard = struct { + sync.Mutex + running map[string]bool + last map[string]time.Time +}{running: map[string]bool{}, last: map[string]time.Time{}} + +// moveHeardEvery is how often a machine heard from is moved at most. +const moveHeardEvery = 10 * time.Minute + +// moveWhenHeard moves what is left of a machine that was just heard from, once the move has started: a machine asleep +// when it started is moved when it next reports. What it did is said on the controller's output, and recorded for the +// next `secret move`. +func moveWhenHeard(ctx context.Context, open *stores, conn *nats.Conn, node string) { + if open == nil || conn == nil || node == "" { + return + } + moveHeard.Lock() + if moveHeard.running[node] || time.Since(moveHeard.last[node]) < moveHeardEvery { + moveHeard.Unlock() + return + } + moveHeard.running[node], moveHeard.last[node] = true, time.Now() + moveHeard.Unlock() + defer func() { + moveHeard.Lock() + delete(moveHeard.running, node) + moveHeard.Unlock() + }() + inv := open.inventory + move, started, err := inv.SecretMoveStarted(ctx) + if err != nil || !started { + return + } + rows, err := inv.SecretsToMoveFrom(ctx, node) + if err != nil { + return + } + left := false + for _, r := range rows { + left = left || (r.Moved == nil && r.Current) + } + if !left { + return + } + ask, err := moveOnTheBus(ctx, open, conn, "the move of the secrets, by "+move.StartedBy+", on hearing from "+node) + if err != nil { + fmt.Fprintf(os.Stderr, "mesh-controller: %s was heard from and its secrets not moved: %v\n", node, err) + return + } + m := moveNode(node, func(string) ([]inventory.SecretToMove, error) { return rows, nil }, ask, + func(r inventory.SecretToMove, g int) error { return inv.RecordSecretMoved(ctx, r, g) }) + if m.done() { + fmt.Printf("mesh-controller: %s was heard from; the move of its secrets is done: %s", node, m.String()) + return + } + fmt.Fprintf(os.Stderr, "mesh-controller: %s was heard from; the move of its secrets is not done: %s", node, m.String()) +} + +// moveHeardOn is the connection a machine heard from is moved over: the serving controller's bus. +func moveHeardOn() *nats.Conn { + if js := servingBus.Load(); js != nil { + return js.Conn() + } + return nil +} diff --git a/cmd/mesh-controller/secret_move_test.go b/cmd/mesh-controller/secret_move_test.go new file mode 100644 index 00000000..cbf179d9 --- /dev/null +++ b/cmd/mesh-controller/secret_move_test.go @@ -0,0 +1,158 @@ +package main + +import ( + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func aRowToMove(secret string, current bool) inventory.SecretToMove { + return inventory.SecretToMove{Secret: secret, Node: "anchor", Module: "postgres", Name: strings.TrimPrefix(secret, "own/anchor/postgres/"), + Origin: "made", Generation: "g-1", MadeAt: time.Unix(1, 0), Sealed: "c2VhbGVk", SealedTo: "k", Current: current, + Digest: "sha256:" + secret} +} + +// Each row sealed to a machine is asked of its engine once, as the store holds it; what the vault kept and read back +// the same is recorded; a row the vault held already counts as there; a row moved before is not asked again, and one +// changed in the store since it moved is said; a row sealed to a key the machine no longer has is said, not asked. +func TestAMachinesRowsAreMovedOnceEachAndWhatMovedIsSaid(t *testing.T) { + rows := []inventory.SecretToMove{ + aRowToMove("own/anchor/postgres/superuser", true), + aRowToMove("own/anchor/postgres/replication", true), + aRowToMove("own/anchor/postgres/old-key", false), + aRowToMove("own/anchor/postgres/refused", true), + aRowToMove("own/anchor/postgres/moved-before", true), + aRowToMove("own/anchor/postgres/changed-since", true), + } + rows[4].Moved = &inventory.SecretMoved{Node: "anchor", Generation: 1, Digest: rows[4].Digest} + rows[5].Moved = &inventory.SecretMoved{Node: "anchor", Generation: 1, Digest: "sha256:something before"} + var asked []link.MoveSecretAsk + ask := func(r inventory.SecretToMove, row link.MoveSecretAsk) (link.MoveSecretAnswer, error) { + asked = append(asked, row) + switch r.Secret { + case "own/anchor/postgres/refused": + return link.MoveSecretAnswer{Refused: "the vault already holds another value; nothing was moved"}, nil + case "own/anchor/postgres/replication": + return link.MoveSecretAnswer{Secret: r.Secret, Generation: 3, Unchanged: true}, nil + } + return link.MoveSecretAnswer{Secret: r.Secret, Generation: 1}, nil + } + recorded := map[string]int{} + record := func(r inventory.SecretToMove, generation int) error { recorded[r.Secret] = generation; return nil } + m := moveNode("anchor", func(string) ([]inventory.SecretToMove, error) { return rows, nil }, ask, record) + if m.Moved != 1 || m.Already != 2 || len(m.Refused) != 2 || len(m.Changed) != 1 || m.Unreached != nil { + t.Fatalf("the move said %+v", m) + } + if len(asked) != 3 || asked[0].Module != "postgres" || asked[0].Name != "superuser" || asked[0].Sealed != "c2VhbGVk" || + asked[0].Origin != "made" || asked[0].Generation != "g-1" || asked[0].SealedTo != "k" { + t.Fatalf("asked %+v", asked) + } + if recorded["own/anchor/postgres/superuser"] != 1 || recorded["own/anchor/postgres/replication"] != 3 || len(recorded) != 2 { + t.Errorf("recorded %v", recorded) + } + said := m.String() + for _, want := range []string{"anchor: 1 moved into the vault and read back the same, 2 there already", + "own/anchor/postgres/old-key", "earlier sealing key", "own/anchor/postgres/refused", "already holds another value", + "own/anchor/postgres/changed-since", "changed in the controller's store after it moved"} { + if !strings.Contains(said, want) { + t.Errorf("the move did not say %q:\n%s", want, said) + } + } + if strings.Contains(said, "c2VhbGVk") { + t.Error("the move said a sealed value") + } +} + +// A machine that does not answer is left for when it is next heard from: nothing more is asked of it now, and that +// is said; a pair is asked with both its ends and never as an own secret. +func TestAMachineNotAnsweringIsMovedWhenNextHeardFrom(t *testing.T) { + pair := inventory.SecretToMove{Secret: "pair/anchor/database/shanks/gitea", Node: "shanks", Provider: "anchor", + Provision: "database", Consumer: "shanks", ConsumerModule: "gitea", Origin: "accepted", Sealed: "x", + SealedTo: "k", Current: true} + rows := []inventory.SecretToMove{pair, pair} + calls := 0 + var asked link.MoveSecretAsk + ask := func(_ inventory.SecretToMove, row link.MoveSecretAsk) (link.MoveSecretAnswer, error) { + calls++ + asked = row + return link.MoveSecretAnswer{}, errors.New("nothing on shanks answers a move") + } + m := moveNode("shanks", func(string) ([]inventory.SecretToMove, error) { return rows, nil }, ask, + func(inventory.SecretToMove, int) error { t.Fatal("recorded"); return nil }) + if calls != 1 || m.Unreached == nil || !strings.Contains(m.String(), "moved when it is next heard from") { + t.Fatalf("calls %d, %+v: %s", calls, m, m.String()) + } + if asked.Module != "" || asked.Provider != "anchor" || asked.Consumer != "shanks" || asked.ConsumerModule != "gitea" { + t.Errorf("the pair was asked as %+v", asked) + } + failing := moveNode("x", func(string) ([]inventory.SecretToMove, error) { return nil, errors.New("no store") }, nil, nil) + if failing.Failed == nil { + t.Error("a store that cannot be read was not said") + } +} + +// The record of asks is copied into the vault in parts, as it is: every row's fields, nothing more. +func TestTheRecordOfAsksIsCopiedInParts(t *testing.T) { + ended := time.Unix(100, 0).UTC() + var asks []inventory.SecretAsk + for n := 1; n <= movedAsksAtOnce+3; n++ { + asks = append(asks, inventory.SecretAsk{ID: int64(n), Node: "anchor", Module: "telegram", Name: "token", + AskedBy: "g14/claude-code", Desk: "laptop", OpenedAt: time.Unix(int64(n), 0).UTC(), EndedAt: &ended, + Outcome: "given"}) + } + var parts []int + vault := func(verb string, args map[string]any) (json.RawMessage, error) { + if verb != "move-asks" { + t.Fatalf("asked %s", verb) + } + rows := args["asks"].([]map[string]any) + parts = append(parts, len(rows)) + if rows[0]["desk"] != "laptop" || rows[0]["asked_by"] != "g14/claude-code" || rows[0]["outcome"] != "given" || + rows[0]["id"] == nil || rows[0]["opened_at"] == nil || rows[0]["ended_at"] == nil { + t.Fatalf("a row went as %v", rows[0]) + } + return json.Marshal(map[string]int{"copied": len(rows)}) + } + copied, err := moveAsks(asks, vault) + if err != nil || copied != movedAsksAtOnce+3 || len(parts) != 2 || parts[0] != movedAsksAtOnce { + t.Fatalf("copied %d in %v, %v", copied, parts, err) + } + if _, err := moveAsks(asks, func(string, map[string]any) (json.RawMessage, error) { + return nil, errors.New("the vault is away") + }); err == nil { + t.Error("a copy the vault refused was not said") + } +} + +// The verb runs the move's one line through the seat and nothing else of the secret commands. +func TestTheMoveIsAVerbOfItsOwn(t *testing.T) { + argv, err := argvFor("secret-move", map[string]any{}) + if err != nil || strings.Join(argv, " ") != "secret move" { + t.Fatalf("argv %v, %v", argv, err) + } + argv, err = argvFor("secret-move", map[string]any{"node": "anchor"}) + if err != nil || strings.Join(argv, " ") != "secret move anchor" { + t.Fatalf("argv %v, %v", argv, err) + } + if err := terminalOnly([]string{"secret", "move", "anchor"}); err != nil { + t.Fatalf("the move's line was refused through its verb: %v", err) + } + for _, line := range [][]string{{"secret", "move", "--from", "x"}, {"secret", "move", "a", "b"}} { + if err := terminalOnly(line); err == nil { + t.Errorf("%q ran through a verb", line) + } + } + found := false + for _, v := range catalogue.ControllerVerbs { + found = found || v.Name == "secret-move" + } + if !found { + t.Fatal("the controller's seat does not declare the secret-move verb") + } +} diff --git a/cmd/mesh-controller/sendable.go b/cmd/mesh-controller/sendable.go index 973813b1..264a46c2 100644 --- a/cmd/mesh-controller/sendable.go +++ b/cmd/mesh-controller/sendable.go @@ -77,6 +77,9 @@ type sendable struct { // unplaced is every contribution its holder's template could not render, naming its module and // why (novox/hq ADR 0255): left out of this declaration, for push and plan to say, never on the wire. unplaced []string + // handedOnly is each file left out because only the provisioner's client fills it and the module is not handed + // here (a provider's consumers file, ADR 0373). + handedOnly []string // Builds is the build of each module this declaration carries — module to the commit its build // was made from — recorded with the send and never on the wire (novox/hq issue 259, ADR 0221). // Composed only on the send path; nil records that it is not known. @@ -84,6 +87,16 @@ type sendable struct { // Bindings is where each consumer of a provision that keeps its data is bound in this declaration // (novox/hq ADR 0232), recorded with the send and never on the wire. Composed only on the send path. Bindings []inventory.Binding + // Filled is the path of every file this declaration hands to the provisioner, and Provisioner the account the + // client that fills them runs as (novox/hq ADR 0349 §4, §5, design 57 P6): on the wire as `filled_files` and + // `provisioner_account`, both absent when nothing is handed, so every other declaration is byte for byte what + // it was. plans is each handed module's plan, kept with the send for the assignments mirror; never on the wire. + Filled []string + Provisioner string + plans map[string]catalogue.Plan + // requiredBy is, for each handed module only pulled in by another, the module that requires it (issue 765): kept + // beside its plan, so the assignments mirror gives the plan a value of its own. Never on the wire. + requiredBy map[string]string } // adoptionEnvelope is what an adopted node is told about its mode. Taken is every module taken on @@ -113,6 +126,10 @@ func (s sendable) Body() ([]byte, error) { if s.Generation > 0 { envelope["generation"] = s.Generation } + if len(s.Filled) > 0 { + envelope["filled_files"] = s.Filled + envelope["provisioner_account"] = s.Provisioner + } // An empty declaration is deliberate here — the node owns nothing the mesh put there // (novox/hq issue 127) — and the host refuses an empty body unless it is told the emptiness // is meant, so a truncated or mis-composed body is never mistaken for "own nothing". diff --git a/cmd/mesh-controller/sent_required_by_test.go b/cmd/mesh-controller/sent_required_by_test.go new file mode 100644 index 00000000..68553609 --- /dev/null +++ b/cmd/mesh-controller/sent_required_by_test.go @@ -0,0 +1,125 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// A send keeps, beside each handed module's plan, what requires a module only pulled in (issue 765), so the +// assignments mirror gives that plan a value of its own; an assigned module's plan is kept as the bytes it was. +func TestASendKeepsWhatRequiresAPulledInModule(t *testing.T) { + plan := catalogue.Plan{Templates: catalogue.PlanTemplates{Source: "http://store:5000/v2/notify/files", + Digest: "sha256:" + strings.Repeat("d", 64)}, + Files: []catalogue.PlanFile{{ID: "rc", Path: "/home/op/rc", Template: "rc", OwnerAtRuntime: "owned"}}} + r := sentRecordOf(t.Context(), sendable{plans: map[string]catalogue.Plan{"dunst": plan, "notify": plan}, + requiredBy: map[string]string{"notify": "desk"}}) + kept := string(r.plans) + want := `{"dunst":{"templates":{"source":"http://store:5000/v2/notify/files","digest":"sha256:` + + strings.Repeat("d", 64) + `"},"files":[{"id":"rc","path":"/home/op/rc","template":"rc","owner-at-runtime":"owned"}]},` + + `"notify":{"templates":{"source":"http://store:5000/v2/notify/files","digest":"sha256:` + strings.Repeat("d", 64) + + `"},"files":[{"id":"rc","path":"/home/op/rc","template":"rc","owner-at-runtime":"owned"}],"required-by":"desk"}}` + if kept != want { + t.Fatalf("the send keeps\n%s\nwant\n%s", kept, want) + } +} + +// **A module only pulled in by another is handed on a machine that can take it** (issue 765): on a contract-11 +// node running a client, a moved module that nothing assigned there, required by one that is, has its files and its +// own secret handed like an assigned one's, no secret of it made or sealed by the controller, and its plan mirrored +// under its own key, saying what requires it. A module not moved, pulled in the same way, is sent as before. +func TestAModulePulledInIsHandedOnAMachineThatCanTakeItAndMirrored(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + client := aClientModule() + moved, kept := aBot("moved-bot", true), aBot("kept-bot", false) + moved.Provides = []catalogue.Offer{{Name: "moved-bot-api"}} + kept.Provides = []catalogue.Offer{{Name: "kept-bot-api"}} + desk := catalogue.Manifest{Module: "desk", Version: "1", Requires: []string{"moved-bot-api", "kept-bot-api"}} + for _, m := range []catalogue.Manifest{client, moved, kept, desk} { + register(t, open, m) + } + for _, m := range []string{client.Module, desk.Module} { + if _, err := assign(ctx, open, "laptop", m); err != nil { + t.Fatal(err) + } + } + if err := issueProvisionerClient(ctx, inv, client, "laptop", broker.Broker{Fingerprint: "ab"}, "10.0.0.1:4222"); err != nil { + t.Fatal(err) + } + if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "laptop", Contract: link.HandedContract, + SaidAt: time.Now()}); err != nil { + t.Fatal(err) + } + + plan, settings, err := planFor(ctx, open, "laptop") + if err != nil { + t.Fatal(err) + } + if plan.RequiredBy("moved-bot") != "desk" || plan.RequiredBy("kept-bot") != "desk" { + t.Fatalf("the bots are on laptop because %v", plan.Because) + } + declared, err := declarationFor(ctx, open, "laptop", plan, settings) + if err != nil { + t.Fatal(err) + } + var movedToken, keptToken, movedConf map[string]any + for _, r := range declared.Resources { + switch r["id"] { + case "moved-bot.needs-token": + movedToken = r + case "kept-bot.needs-token": + keptToken = r + case "moved-bot.conf": + movedConf = r + } + } + if movedToken == nil || movedToken["sealed"] != nil || movedToken["owner-at-runtime"] != "owned" { + t.Fatalf("the pulled-in moved module's secret file is sent as %v: placed, with no value", movedToken) + } + if movedConf == nil || movedConf["content"] != nil || movedConf["owner-at-runtime"] != "owned" { + t.Fatalf("the pulled-in moved module's templated file is sent as %v: placed, with no text", movedConf) + } + if keptToken == nil || keptToken["sealed"] == nil { + t.Fatalf("the pulled-in module not moved has its secret sent as %v: sealed, as before", keptToken) + } + if _, held, err := inv.ModuleSecretIfIssued(ctx, "laptop", "moved-bot", "token"); err != nil || held { + t.Fatalf("the send made the pulled-in handed module's secret in the controller's store (held %v, %v)", held, err) + } + if _, has := declared.plans["kept-bot"]; has || len(declared.plans) != 1 { + t.Fatalf("the send hands %v", declared.plans) + } + + record, err := inv.NodeByName(ctx, "laptop") + if err != nil { + t.Fatal(err) + } + if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, inventory.Send{Digest: "first", + Provisioning: sentRecordOf(ctx, declared).plans}); err != nil { + t.Fatal(err) + } + inv.SetProvisioningPlans(inv.SentProvisioningPlans) + mirror, err := inv.AssignmentsMirror(ctx) + if err != nil { + t.Fatal(err) + } + var value inventory.AssignmentMirrored + if err := json.Unmarshal(mirror["laptop.moved-bot"], &value); err != nil { + t.Fatalf("no value for the pulled-in module: %s (%v)", mirror["laptop.moved-bot"], err) + } + if value.RequiredBy != "desk" || !value.Assigned.IsZero() || value.Provisioning == nil || + len(value.Provisioning.Files) != 1 || value.Provisioning.Files[0].Path != "/etc/moved-bot/bot.conf" || + len(value.Provisioning.Secrets) != 1 || value.Provisioning.Secrets[0].Name != "token" { + t.Fatalf("the pulled-in module is mirrored as %s", mirror["laptop.moved-bot"]) + } + if _, has := mirror["laptop.kept-bot"]; has { + t.Fatalf("a pulled-in module not handed is mirrored: %s", mirror["laptop.kept-bot"]) + } +} diff --git a/cmd/mesh-controller/settings_forward.go b/cmd/mesh-controller/settings_forward.go new file mode 100644 index 00000000..c3c423de --- /dev/null +++ b/cmd/mesh-controller/settings_forward.go @@ -0,0 +1,372 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// **`settings` answers by forwarding to `config`** (novox/hq ADR 0348 §1, design 57 K5, issue 664). Once the move of +// config has run, a module in the config form keeps its config values in mesh-config (inventory.ReadsConfig), so the +// settings verb reads and writes them there: `show` lists what `config list` lists, `set` sets each value as the +// module's own on the node and `clear` clears them, and `propose` proposes each through mesh-config. Placement stays +// the controller's and is read and written beside the assignment as before (ADR 0348 §11). A module still in the +// settings form, and every module before the move, keeps today's verb unchanged. + +// configAsk asks mesh-config's verb in the controller's own name, so mesh-config takes a value as the controller +// carries it (ADR 0348 §8); a test replaces it. +var configAsk = func(ctx context.Context, args map[string]any) (json.RawMessage, error) { + var result json.RawMessage + err := onTheBus(func(conn *nats.Conn) error { + answer, err := link.AskMeshSeatToolAs(ctx, conn, broker.ControllerName, "mesh-config", "config", args, 30*time.Second) + if err != nil { + return err + } + if answer.Error != "" { + return errors.New(answer.Error) + } + result = answer.Result + return nil + }) + return result, err +} + +// configListed is what `config list` answers, as the settings verb reads it. +type configListed struct { + Resolved []resolvedEntry `json:"resolved"` + Set []struct { + Level configLevel `json:"level"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` + By string `json:"by"` + } `json:"set"` +} + +func listConfig(ctx context.Context, module, node string) (configListed, error) { + args := map[string]any{"action": "list", "module": module} + if node != "" { + args["node"] = node + } + raw, err := configAsk(ctx, args) + if err != nil { + return configListed{}, fmt.Errorf("mesh-config does not answer %s's config values: %w", module, err) + } + var l configListed + if err := json.Unmarshal(raw, &l); err != nil { + return configListed{}, fmt.Errorf("mesh-config's listing cannot be read: %w", err) + } + return l, nil +} + +// ownValues is the module's own values on the node, as mesh-config holds them; none for the whole mesh, which has no +// module value (ADR 0348 §3). +func (l configListed) ownValues(module, node string) map[string]any { + out := map[string]any{} + if node == "" { + return out + } + for _, s := range l.Set { + if s.Level.Level == "module" && s.Level.Node == node && s.Level.Module == module { + var v any + if json.Unmarshal(s.Value, &v) == nil { + out[s.Key] = v + } + } + } + return out +} + +// forwardSettings answers a settings action for a module whose config values are mesh-config's, and says whether it +// did: false leaves the action to today's verb. +func forwardSettings(ctx context.Context, inv *inventory.Inventory, action, module, node string, positionals []string, + replace bool) (bool, error) { + if module == "" { + return false, nil + } + reads, err := inv.ReadsConfig(ctx, module) + if err != nil || !reads { + return false, err + } + where := whereOf(node) + switch action { + case "show": + listed, err := listConfig(ctx, module, node) + if err != nil { + return true, err + } + layer, _, err := inv.Layer(ctx, node, module) + if err != nil { + return true, err + } + if layer == nil { + layer = map[string]any{} + } + for k, v := range listed.ownValues(module, node) { + layer[k] = v + } + if len(layer) == 0 { + fmt.Printf("%s on %s: no layer — the module's definition says\n", module, where) + } else { + shown, err := json.MarshalIndent(layer, "", " ") + if err != nil { + return true, err + } + fmt.Println(string(shown)) + } + fmt.Printf(" its config values are mesh-config's since the move of config (issue 664): mesh-config.config\n") + fmt.Printf("%s on %s, every value and where it comes from:\n", module, where) + for _, r := range listed.Resolved { + value := string(r.Value) + if value == "" { + value = "unset" + } + on := "" + if node == "" { + on = " on " + r.Node + } + fmt.Printf(" %s%s = %s (%s)\n", r.Key, on, value, r.Source) + } + return true, nil + + case "set": + var raw []byte + if strings.HasPrefix(strings.TrimSpace(positionals[1]), "{") { + raw = []byte(positionals[1]) + } else { + return true, errors.New("the settings of a module whose config values are mesh-config's are given as " + + "{…}, never a file; or set each with mesh-config.config") + } + var values map[string]any + if err := json.Unmarshal(raw, &values); err != nil { + return true, fmt.Errorf("%s is not a settings object: %w", positionals[1], err) + } + return true, forwardSet(ctx, inv, module, node, values, replace) + + case "clear": + listed, err := listConfig(ctx, module, node) + if err != nil { + return true, err + } + own := listed.ownValues(module, node) + layer, _, err := inv.Layer(ctx, node, module) + if err != nil { + return true, err + } + before := map[string]any{} + for k, v := range layer { + before[k] = v + } + for k, v := range own { + before[k] = v + } + if err := refuseTerminalSettingsThroughAVerb(ctx, inv, before, nil, module, where); err != nil { + return true, err + } + if err := refuseTrustedThroughAVerb(ctx, inv, module, where, movedKeys(own)); err != nil { + return true, err + } + for _, k := range movedKeys(own) { + if _, err := configAsk(ctx, map[string]any{"action": "clear", "module": module, "node": node, "key": k, + "why": "settings clear " + module + " on " + where, "by": setByWords()}); err != nil { + return true, fmt.Errorf("%s of %s on %s was not cleared: %w", k, module, where, err) + } + } + if len(layer) > 0 { + if err := inv.ClearSettingsBy(ctx, node, module, setByWords()); err != nil { + return true, err + } + } + fmt.Printf("%s on %s is back to what the module says\n", module, where) + return true, nil + + } + return false, nil +} + +// forwardSet sets a layer of a module whose config values are mesh-config's: its placement beside the assignment +// as before, each config value as the module's own on the node, and each it no longer names cleared. +func forwardSet(ctx context.Context, inv *inventory.Inventory, module, node string, values map[string]any, + replace bool) error { + where := whereOf(node) + config, placement := catalogue.SplitPlacement(values) + listed, err := listConfig(ctx, module, node) + if err != nil { + return err + } + own := listed.ownValues(module, node) + layer, _, err := inv.Layer(ctx, node, module) + if err != nil { + return err + } + before := map[string]any{} + for k, v := range layer { + before[k] = v + } + for k, v := range own { + before[k] = v + } + if err := refuseTerminalSettingsThroughAVerb(ctx, inv, before, values, module, where); err != nil { + return err + } + added, changed, removed := settingsChange(before, values) + if len(removed) > 0 && !replace { + return fmt.Errorf("%s on %s: this layer would no longer set %s. A layer is replaced whole; read it with "+ + "`settings show %s%s` and include what should stay, or add --replace if the removal is meant (novox/hq "+ + "ADR 0217). Nothing was changed", module, where, strings.Join(removed, ", "), module, nodeFlag(node)) + } + var toSet, toClear []string + for _, k := range movedKeys(config) { + if was, had := own[k]; !had || !sameJSON(valueJSON(was), config[k]) { + toSet = append(toSet, k) + } + } + for _, k := range movedKeys(own) { + if _, kept := config[k]; !kept { + toClear = append(toClear, k) + } + } + if node == "" && len(toSet)+len(toClear) > 0 { + return fmt.Errorf("%s has no own value for the whole mesh (novox/hq ADR 0348 §3): name the node, or set mesh "+ + "config for the predefined key a value falls back to with mesh-config.config. Nothing was changed", module) + } + if err := refuseTrustedThroughAVerb(ctx, inv, module, where, append(append([]string{}, toSet...), toClear...)); err != nil { + return err + } + if placement == nil { + placement = map[string]any{} + } + if _, placed := catalogue.SplitPlacement(layer); !sameLayer(placed, placement) { + if err := inv.SetSettingsBy(ctx, node, module, placement, setByWords()); err != nil { + return err + } + } + for _, k := range toSet { + if _, err := configAsk(ctx, map[string]any{"action": "set", "module": module, "node": node, "key": k, + "value": config[k], "why": "settings set " + module + " on " + where, "by": setByWords()}); err != nil { + return fmt.Errorf("%s of %s on %s was not set: %w", k, module, where, err) + } + } + for _, k := range toClear { + if _, err := configAsk(ctx, map[string]any{"action": "clear", "module": module, "node": node, "key": k, + "why": "settings set " + module + " on " + where + " --replace", "by": setByWords()}); err != nil { + return fmt.Errorf("%s of %s on %s was not cleared: %w", k, module, where, err) + } + } + fmt.Printf("%s on %s:\n", module, where) + if len(added)+len(changed)+len(removed) == 0 { + fmt.Println(" nothing changed") + } + for _, p := range added { + fmt.Printf(" + %s\n", p) + } + for _, p := range changed { + fmt.Printf(" ~ %s\n", p) + } + for _, p := range removed { + fmt.Printf(" - %s\n", p) + } + fmt.Println(" its config values are mesh-config's (issue 664): each was set there, with its history; the module " + + "takes them when its files are filled again") + return nil +} + +func valueJSON(v any) json.RawMessage { + raw, _ := json.Marshal(v) + return raw +} + +// refuseTrustedThroughAVerb refuses, through a verb, a trusted config value (ADR 0348 §8): mesh-config takes the +// controller's word for one as given at its terminal, and whoever may call a verb includes agents. +func refuseTrustedThroughAVerb(ctx context.Context, inv *inventory.Inventory, module, where string, keys []string) error { + verb, through := throughAVerb() + if !through || len(keys) == 0 { + return nil + } + shelf, err := inv.Catalogue(ctx) + if err != nil { + return fmt.Errorf("which config values of %s are trusted cannot be read, so nothing was changed: %w", module, err) + } + for _, k := range keys { + if shelf[module].Settings[k].Trusted { + return fmt.Errorf("%s of %s on %s is trusted (novox/hq ADR 0348 §8): it is set at the controller's terminal "+ + "or on the operator's Approve, never through a verb alone (this line came through %q). Propose it with "+ + "mesh-config.config, action propose. Nothing was changed", k, module, where, verb) + } + } + return nil +} + +// forwardPropose proposes, through mesh-config, each config value of a layer proposed for a module whose config +// values are mesh-config's (ADR 0348 §8): mesh-config asks the operator under its own name, one ask per value. A +// proposal naming placement too is refused, so the operator is never asked half of it; placement alone is proposed +// as before. Says whether it answered. +func forwardPropose(ctx context.Context, inv *inventory.Inventory, module, node, valuesArg string, clear bool) (bool, error) { + reads, err := inv.ReadsConfig(ctx, module) + if err != nil || !reads { + return false, err + } + where := whereOf(node) + if node == "" { + return true, fmt.Errorf("%s has no own value for the whole mesh (novox/hq ADR 0348 §3): name the node, or "+ + "propose mesh config for the predefined key a value falls back to with mesh-config.config. Nothing was "+ + "proposed", module) + } + proposals := map[string]any{} + var clears []string + if clear { + listed, err := listConfig(ctx, module, node) + if err != nil { + return true, err + } + layer, _, err := inv.Layer(ctx, node, module) + if err != nil { + return true, err + } + if len(layer) > 0 { + return true, fmt.Errorf("%s on %s keeps placement beside its assignment (%s): propose clearing it apart "+ + "from its config values. Nothing was proposed", module, where, strings.Join(movedKeys(layer), ", ")) + } + clears = movedKeys(listed.ownValues(module, node)) + } else { + var values map[string]any + if err := json.Unmarshal([]byte(valuesArg), &values); err != nil { + return true, fmt.Errorf("%s is not a settings object: %w", valuesArg, err) + } + config, placement := catalogue.SplitPlacement(values) + if len(placement) > 0 { + if len(config) > 0 { + return true, fmt.Errorf("%s names placement (%s) and config values: propose them apart, the config "+ + "values through mesh-config. Nothing was proposed", module, strings.Join(movedKeys(placement), ", ")) + } + return false, nil + } + proposals = config + } + for _, k := range movedKeys(proposals) { + answer, err := configAsk(ctx, map[string]any{"action": "propose", "module": module, "node": node, "key": k, + "value": proposals[k], "why": "settings propose " + module + " on " + where}) + if err != nil { + return true, fmt.Errorf("%s of %s on %s was not proposed: %w", k, module, where, err) + } + fmt.Printf("%s of %s on %s, proposed through mesh-config: %s\n", k, module, where, answer) + } + for _, k := range clears { + answer, err := configAsk(ctx, map[string]any{"action": "propose", "module": module, "node": node, "key": k, + "clear": true, "why": "settings propose " + module + " on " + where + " --clear"}) + if err != nil { + return true, fmt.Errorf("clearing %s of %s on %s was not proposed: %w", k, module, where, err) + } + fmt.Printf("clearing %s of %s on %s, proposed through mesh-config: %s\n", k, module, where, answer) + } + return true, nil +} diff --git a/cmd/mesh-controller/status_summary.go b/cmd/mesh-controller/status_summary.go index 52d29591..c52f3c41 100644 --- a/cmd/mesh-controller/status_summary.go +++ b/cmd/mesh-controller/status_summary.go @@ -224,5 +224,10 @@ func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, e // nothing it caused (novox/hq ADR 0228). go replaceGiven(context.WithoutCancel(ctx), l.open, report) } + // And, once the move of the secrets into the vault has started, what is left of this machine's (design 57's V3, + // issue 676): a machine asleep when it started is moved when it is next heard from. Off the report's path. + if err == nil && l.open != nil && report.Superseded == "" && report.Rekey == nil && report.Node != "" { + go moveWhenHeard(context.WithoutCancel(ctx), l.open, moveHeardOn(), report.Node) + } return news, err } diff --git a/cmd/mesh-controller/stores.go b/cmd/mesh-controller/stores.go index aeea12c1..5fef54d7 100644 --- a/cmd/mesh-controller/stores.go +++ b/cmd/mesh-controller/stores.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "os" "time" "github.com/novox/mesh-controller/internal/catalogue" @@ -124,6 +125,13 @@ func openInventory(ctx context.Context) (*inventory.Inventory, error) { if aliases, err := inv.Aliases(ctx); err == nil { catalogue.UseAliases(aliases) } + // And the list of predefined keys the mesh publishes (novox/hq ADR 0348 §7), which every manifest read again + // is judged against. A store a migration has not reached yet keeps none, and nothing names a key then. + if err := loadPredefined(ctx, inv); err != nil { + fmt.Fprintf(os.Stderr, "the list of predefined keys kept cannot be read: %v\n", err) + } + // And where config values are read once the move of config has run (issue 664): mesh-config_resolved. + inv.ReadsConfigFrom(readConfigValues) return inv, nil } diff --git a/cmd/mesh-controller/template_test.go b/cmd/mesh-controller/template_test.go new file mode 100644 index 00000000..2a14e3e4 --- /dev/null +++ b/cmd/mesh-controller/template_test.go @@ -0,0 +1,272 @@ +package main + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + snapshot "github.com/novox/mesh-controller/internal/facts" + "github.com/novox/mesh-controller/internal/inventory" +) + +// A file's text kept as a template file in its module's repository (novox/hq ADR 0349 §2, design 57 package T1, +// issue 654): `module check` and the merge gate read it from the tree beside the manifest, so a change to a +// template alone is judged as a change to its module. + +const templatedNotifier = `{"module":"notifier","version":"1", + "build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]}, + "resources":[{"id":"conf","type":"file","path":"/etc/notifier/notifier.conf","mode":"0644","template":"files/notifier.conf"}]}` + +// inlineNotifier is the same module as the mesh holds it: built, its text inline. +const inlineNotifier = `{"module":"notifier","version":"1", + "resources":[{"id":"conf","type":"file","path":"/etc/notifier/notifier.conf","mode":"0644","content":"volume = 3\n"}]}` + +// aTemplatedModule writes a module's directory: its manifest and its files/. +func aTemplatedModule(t *testing.T, dir, manifest string, files map[string]string) string { + t.Helper() + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "module.json") + if err := os.WriteFile(path, []byte(manifest), 0o644); err != nil { + t.Fatal(err) + } + for name, text := range files { + full := filepath.Join(dir, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(text), 0o644); err != nil { + t.Fatal(err) + } + } + return path +} + +func TestModuleCheckReadsTemplatesFromTheTreeBesideTheManifest(t *testing.T) { + good := aTemplatedModule(t, t.TempDir(), templatedNotifier, map[string]string{"files/notifier.conf": "volume = 3\n"}) + var out bytes.Buffer + if err := moduleCheck([]string{good}, &out); err != nil || !strings.Contains(out.String(), "notifier: ok") { + t.Fatalf("a module with its text in files/ was refused: %v\n%s", err, out.String()) + } + + // A change to the template alone, to text the controller refuses inline, is refused the same. + bad := aTemplatedModule(t, t.TempDir(), templatedNotifier, + map[string]string{"files/notifier.conf": "volume = 3\nstore = ${dir:nowhere}\n"}) + out.Reset() + if err := moduleCheck([]string{bad}, &out); err == nil || !strings.Contains(out.String(), "nowhere") { + t.Fatalf("a template whose text is refused inline passed:\n%s", out.String()) + } + + // A symlink under files/ that no template names: the build's packing refuses it, so the check does too. + linked := t.TempDir() + aTemplatedModule(t, linked, templatedNotifier, map[string]string{"files/notifier.conf": "volume = 3\n"}) + if err := os.Symlink("/etc/hostname", filepath.Join(linked, "files", "elsewhere")); err != nil { + t.Fatal(err) + } + out.Reset() + if err := moduleCheck([]string{filepath.Join(linked, "module.json")}, &out); err == nil || + !strings.Contains(out.String(), "files/elsewhere is a symlink") { + t.Fatalf("a symlink under files/ passed the check:\n%s", out.String()) + } + + // And a template the tree lacks, by name. + missing := aTemplatedModule(t, t.TempDir(), templatedNotifier, nil) + out.Reset() + if err := moduleCheck([]string{missing}, &out); err == nil || !strings.Contains(out.String(), "files/notifier.conf") { + t.Fatalf("a missing template was not refused by name:\n%s", out.String()) + } +} + +// templatedMesh is catalogueMesh with the notifier on the laptop, registered as built: its text inline. +func templatedMesh(t *testing.T) (snapshot.Facts, map[string]string) { + t.Helper() + open := aMesh(t) + ctx := t.Context() + manifests := map[string]string{ + "lemurs": `{"module":"lemurs","version":"1","capabilities":["systemd"], + "resources":[{"id":"service","type":"service","unit":"lemurs.service","state":"running","boot":"enabled"}]}`, + "notifier": inlineNotifier, + } + for name, raw := range manifests { + m, err := catalogue.ParseManifest([]byte(raw)) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", + Path: "modules/" + name, BuiltFrom: "c0ffee"}); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, "laptop", name); err != nil { + t.Fatalf("assign %s: %v", name, err) + } + } + f, err := gatherFacts(ctx, open, "2.11.17") + if err != nil { + t.Fatal(err) + } + return f, manifests +} + +// aTemplatedTree is the catalogue's checkout with the notifier's text in files/. +func aTemplatedTree(t *testing.T, manifests map[string]string, text string, has bool) string { + t.Helper() + tree := aTree(t, withEdit(manifests, "notifier", "")) + files := map[string]string{} + if has { + files["files/notifier.conf"] = text + } + aTemplatedModule(t, filepath.Join(tree, "modules", "notifier"), templatedNotifier, files) + return tree +} + +const notifierTemplate = "modules/notifier/files/notifier.conf" + +func TestTheMergeGateJudgesAChangeToATemplateAlone(t *testing.T) { + f, manifests := templatedMesh(t) + + // The module moved to its template, the same text: nothing changes on any machine. + v := gateJudged(t, f, aTemplatedTree(t, manifests, "volume = 3\n", true), "modules/notifier/module.json", + notifierTemplate) + if v.Verdict == "fail" || v.Modules["notifier"] != "" { + t.Fatalf("the module moved to its template with the same text reads as a change (%v):\n%s", v.Modules, v.Report()) + } + + // The template alone changed: its module changed, and composes. + v = gateJudged(t, f, aTemplatedTree(t, manifests, "volume = 4\n", true), notifierTemplate) + if v.Verdict == "fail" || v.Modules["notifier"] != "changed" { + t.Fatalf("a change to the template alone is not a change to its module (%v):\n%s", v.Modules, v.Report()) + } + if v.Width == nil || strings.Join(v.Width.Modules, ",") != "notifier" { + t.Fatalf("a change to the template rebuilds %+v, not its module", v.Width) + } + + // The template alone changed to text the controller refuses: the pull request fails, naming the module. + v = gateJudged(t, f, aTemplatedTree(t, manifests, "store = ${dir:nowhere}\n", true), notifierTemplate) + if v.Verdict != "fail" || !strings.Contains(v.Report(), "notifier") { + t.Fatalf("a template the controller refuses passed the gate:\n%s", v.Report()) + } + + // A symlinked files/ fails: nothing is read from outside the tree. + tree := aTree(t, withEdit(manifests, "notifier", "")) + elsewhere := t.TempDir() + if err := os.WriteFile(filepath.Join(elsewhere, "notifier.conf"), []byte("volume = 3\n"), 0o644); err != nil { + t.Fatal(err) + } + aTemplatedModule(t, filepath.Join(tree, "modules", "notifier"), templatedNotifier, nil) + if err := os.Symlink(elsewhere, filepath.Join(tree, "modules", "notifier", "files")); err != nil { + t.Fatal(err) + } + v = gateJudged(t, f, tree, notifierTemplate) + if v.Verdict != "fail" || !strings.Contains(v.Report(), "files/ is a symlink") { + t.Fatalf("a symlinked files/ passed the gate:\n%s", v.Report()) + } + + // And the template removed from the tree fails, naming it. + v = gateJudged(t, f, aTemplatedTree(t, manifests, "", false), notifierTemplate) + if v.Verdict != "fail" || !strings.Contains(v.Report(), "files/notifier.conf") { + t.Fatalf("a missing template passed the gate:\n%s", v.Report()) + } +} + +// What the gate composes for the module in its template form is what the mesh composes for it inline. +func TestTheMergeGateResolvesATemplatedModuleToItsInlineTwin(t *testing.T) { + dir := t.TempDir() + path := aTemplatedModule(t, dir, templatedNotifier, map[string]string{"files/notifier.conf": "volume = 3\n"}) + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + m, err := catalogue.ParseManifest(raw) + if err != nil { + t.Fatal(err) + } + texts, err := catalogue.ReadTemplates(m, dir) + if err != nil { + t.Fatal(err) + } + resolved, err := standInBuild(m, texts) + if err != nil { + t.Fatal(err) + } + inline, err := catalogue.ParseManifest([]byte(inlineNotifier)) + if err != nil { + t.Fatal(err) + } + if builtAlike(resolved) != builtAlike(inline) { + got, _ := json.Marshal(resolved) + t.Fatalf("the templated module resolves to %s, not its inline twin", got) + } +} + +// A module whose files/ is unpacked onto the machine names its own templates directory (issue 731): `module check` +// reads its templates from there and says of it exactly what it says of the same module with its text inline. +func TestModuleCheckReadsTemplatesFromTheTemplatesDirectoryTheManifestNames(t *testing.T) { + const inline = `{"module":"notifier","version":"1", + "build":{"artifacts":[{"name":"scripts","kind":"archive","from":"files"}]}, + "resources":[{"id":"scripts","type":"archive","path":"/usr/local/lib/notifier","artifact":"scripts"}, + {"id":"conf","type":"file","path":"/etc/notifier/notifier.conf","mode":"0644","content":"volume = 3\n"}]}` + const templated = `{"module":"notifier","version":"1","templates":"templates", + "build":{"artifacts":[{"name":"scripts","kind":"archive","from":"files"}, + {"name":"templates","kind":"archive","from":"templates"}]}, + "resources":[{"id":"scripts","type":"archive","path":"/usr/local/lib/notifier","artifact":"scripts"}, + {"id":"conf","type":"file","path":"/etc/notifier/notifier.conf","mode":"0644","template":"templates/notifier.conf"}]}` + scripts := map[string]string{"files/bin/notify": "#!/bin/sh\n"} + var want bytes.Buffer + if err := moduleCheck([]string{aTemplatedModule(t, t.TempDir(), inline, scripts)}, &want); err != nil { + t.Fatalf("the inline module was refused: %v\n%s", err, want.String()) + } + with := map[string]string{"files/bin/notify": "#!/bin/sh\n", "templates/notifier.conf": "volume = 3\n"} + var got bytes.Buffer + if err := moduleCheck([]string{aTemplatedModule(t, t.TempDir(), templated, with)}, &got); err != nil { + t.Fatalf("a module with its text in templates/ was refused: %v\n%s", err, got.String()) + } + if got.String() != want.String() { + t.Fatalf("module check says otherwise of the templated module:\ninline: %s\ntemplated: %s", want.String(), + got.String()) + } + + // Its text in files/ instead, where the machine unpacks it, is not where its manifest says its templates are. + misplaced := strings.Replace(templated, `"templates/notifier.conf"`, `"files/notifier.conf"`, 1) + got.Reset() + if err := moduleCheck([]string{aTemplatedModule(t, t.TempDir(), misplaced, + map[string]string{"files/notifier.conf": "volume = 3\n"})}, &got); err == nil || + !strings.Contains(got.String(), "files/notifier.conf") { + t.Fatalf("a template outside the templates directory the manifest names passed:\n%s", got.String()) + } +} + +// Issue 876: the gate judges a moved module that names its templates as registration reads it after a real build, +// handed its templates; it refused every such module as "marked moved and names no template". This calls the gate's +// own shelf (shelfWithChange), which needs no database. +func TestReplay876(t *testing.T) { + moved := strings.Replace(templatedNotifier, `"version":"1",`, `"version":"1","moved":true,`, 1) + judge := func(manifest string, files map[string]string) (map[string]catalogue.Manifest, []string) { + tree := t.TempDir() + aTemplatedModule(t, filepath.Join(tree, "modules", "notifier"), manifest, files) + out, failures, err := shelfWithChange(map[string]catalogue.Manifest{}, nil, snapshot.Facts{}, + "novox/mesh-catalog", tree, map[string]string{}, nil) + if err != nil { + t.Fatal(err) + } + return out, failures + } + out, failures := judge(moved, map[string]string{"files/notifier.conf": "volume = 3\n"}) + if len(failures) > 0 { + t.Fatalf("a moved module naming its templates was refused: %v", failures) + } + if h := out["notifier"].Handed; h == nil || h.Templates["conf"] != "notifier.conf" { + t.Fatalf("the gate did not hand the moved module its templates: %+v", h) + } + + // A moved module naming no template is still refused. + none := strings.Replace(strings.Replace(moved, `"template":"files/notifier.conf"`, + `"content":"volume = 3\n"`, 1), `"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},`, "", 1) + if _, failures := judge(none, nil); len(failures) != 1 || !strings.Contains(failures[0], "names no template") { + t.Fatalf("a moved module naming no template was not refused: %v", failures) + } +} diff --git a/cmd/mesh-controller/testdata/config-moved/mesh.json b/cmd/mesh-controller/testdata/config-moved/mesh.json new file mode 100644 index 00000000..d65635ae --- /dev/null +++ b/cmd/mesh-controller/testdata/config-moved/mesh.json @@ -0,0 +1,3661 @@ +{ + "layers": [ + { + "module": "baserow", + "node": "anchor", + "values": { + "email": "operator@example.test" + } + }, + { + "module": "bazarr", + "node": "anchor", + "values": { + "pgid": 2000, + "puid": 1001 + } + }, + { + "module": "bookshelf", + "node": "anchor", + "values": { + "downloads": { + "jackett-api": { + "adopt-hosts": [ + "jackett" + ] + }, + "nzbget-api": { + "name": "NZBGet" + }, + "qbittorrent-api": { + "name": "qBitTorrent" + } + }, + "pgid": 1000, + "puid": 1000 + } + }, + { + "module": "claude-code", + "node": "anchor", + "values": { + "role": "the home server \u2014 media (Plex and the *arr apps), home automation, IoT and app databases" + } + }, + { + "module": "hostname", + "node": "anchor", + "values": { + "hostname": "Ace" + } + }, + { + "module": "influxdb", + "node": "anchor", + "values": { + "bucket": "zurag", + "org": "zurag" + } + }, + { + "module": "kometa", + "node": "anchor", + "values": { + "libraries": { + "Formula 1": { + "metadata_files": [ + { + "url": "https://f1-circuits.zurag.be/kometa/formula-1-metadata.yml" + } + ] + } + }, + "pgid": 1000, + "puid": 1000 + } + }, + { + "module": "lidarr", + "node": "anchor", + "values": { + "downloads": { + "jackett-api": { + "adopt-hosts": [ + "indexers.zurag.be", + "jackett" + ] + }, + "nzbget-api": { + "name": "NZBGet" + }, + "qbittorrent-api": { + "name": "qBitTorrent" + } + }, + "pgid": 2000, + "puid": 1001 + } + }, + { + "module": "nfs-server", + "node": "anchor", + "values": { + "grants": "media=shanks:rw", + "shares": "media=/storage/media" + } + }, + { + "module": "nodered", + "node": "anchor", + "values": { + "mqtt": { + "brokers": [ + "2b0aece9c5f3b307" + ] + }, + "timeZone": "Europe/Brussels" + } + }, + { + "module": "nzbget", + "node": "anchor", + "values": { + "max-download-speed": 10, + "options": { + "UMask": "0000" + }, + "pgid": 2000, + "puid": 1001, + "username": "luffy" + } + }, + { + "module": "ombi", + "node": "anchor", + "values": { + "pgid": 2000, + "puid": 1001 + } + }, + { + "module": "plex", + "node": "anchor", + "values": { + "pgid": 1000, + "puid": 1000 + } + }, + { + "module": "qbittorrent", + "node": "anchor", + "values": { + "max-download-speed": 10, + "pgid": 2000, + "puid": 1001, + "username": "luffy" + } + }, + { + "module": "radarr", + "node": "anchor", + "values": { + "downloads": { + "jackett-api": { + "adopt-hosts": [ + "jackett" + ] + }, + "nzbget-api": { + "name": "NZBGet" + }, + "qbittorrent-api": { + "name": "qBitTorrent" + } + }, + "pgid": 2000, + "puid": 1001 + } + }, + { + "module": "searxng", + "node": "anchor", + "values": { + "search": { + "autocomplete": "", + "suspended_times": { + "SearxEngineAccessDenied": 86400, + "SearxEngineCaptcha": 86400, + "SearxEngineTooManyRequests": 3600 + } + }, + "server": { + "base_url": "https://searxng.zurag.be/", + "method": "POST" + } + } + }, + { + "module": "sonarr", + "node": "anchor", + "values": { + "downloads": { + "jackett-api": { + "adopt-hosts": [ + "jackett" + ] + }, + "nzbget-api": { + "name": "NZBGet" + }, + "qbittorrent-api": { + "name": "qBitTorrent" + } + }, + "pgid": 2000, + "puid": 1001 + } + }, + { + "module": "supabase", + "node": "anchor", + "values": { + "pooler": { + "max-client-connections": 100, + "pool-size": 20, + "tenant": "zurag" + } + } + }, + { + "module": "tautulli", + "node": "anchor", + "values": { + "pgid": 2000, + "puid": 1001 + } + }, + { + "module": "unifi", + "node": "anchor", + "values": { + "username": "luffy" + } + }, + { + "module": "claude-code", + "node": "workstation", + "values": { + "role": "the operator's laptop \u2014 SaaS integrations and mesh control; travels, often behind a corporate VPN client" + } + }, + { + "module": "dunst", + "node": "workstation", + "values": { + "font-size": 13, + "width": 500 + } + }, + { + "module": "hostname", + "node": "workstation", + "values": { + "hostname": "g14" + } + }, + { + "module": "power", + "node": "workstation", + "values": { + "handle-lid-switch": "suspend", + "handle-lid-switch-docked": "suspend", + "handle-lid-switch-external-power": "suspend", + "handle-power-key": "suspend" + } + }, + { + "module": "artifact-store-tools", + "node": "", + "values": { + "older-store-names": "registry-api.novox.be" + } + }, + { + "module": "bazarr", + "node": "", + "values": { + "pgid": 1000, + "puid": 1000 + } + }, + { + "module": "betterbird", + "node": "", + "values": { + "accounts": { + "gmail": { + "address": "operator@example.test", + "from-name": "The Operator", + "incoming": { + "host": "imap.gmail.com", + "kind": "imap", + "port": 993, + "security": "tls", + "sign-in": "oauth", + "user": "operator@example.test" + }, + "outgoing": { + "host": "smtp.gmail.com", + "port": 465, + "security": "tls", + "sign-in": "oauth", + "user": "operator@example.test" + } + }, + "novox": { + "address": "operator@example.test", + "from-name": "The Operator", + "incoming": { + "host": "mail.novox.be", + "kind": "imap", + "port": 143, + "security": "starttls", + "sign-in": "password", + "user": "operator@example.test" + }, + "label": "operator@mail.novox.be", + "outgoing": { + "host": "mail.novox.be", + "port": 587, + "security": "starttls", + "sign-in": "password", + "user": "operator@example.test" + } + }, + "novox2": { + "address": "operator@example.test", + "from-name": "The Operator - MH", + "incoming": { + "host": "mail.novox.be", + "kind": "imap", + "port": 993, + "security": "tls", + "sign-in": "password", + "user": "operator@example.test" + }, + "outgoing": { + "host": "mail.novox.be", + "port": 465, + "security": "tls", + "sign-in": "password", + "user": "operator@example.test" + } + } + }, + "prefs": { + "calendar.alarms.show": true, + "calendar.alarms.showmissed": false, + "calendar.item.promptDelete": false, + "calendar.timezone.useSystemTimezone": true, + "calendar.view.dayendhour": 20, + "calendar.view.daystarthour": 9, + "mail.biff.play_sound": false, + "mail.biff.show_alert": false, + "mail.biff.show_tray_icon": false, + "mail.biff.use_system_alert": true, + "mail.chat.enabled": false, + "mail.dark-reader.enabled": false, + "mail.pane_config.dynamic": 2, + "mail.pane_config.multiline_all": true, + "mail.threadpane.listview": 0, + "mail.uidensity": 0, + "mail.uifontsize": 14, + "mail.warn_on_shift_delete": false, + "mailnews.default_view_flags": 1, + "mailnews.message_display.disable_remote_image": true, + "privacy.donottrackheader.enabled": true + } + } + }, + { + "module": "bookshelf", + "node": "", + "values": { + "pgid": 1000, + "puid": 1000 + } + }, + { + "module": "claude-code", + "node": "", + "values": { + "managed_settings": { + "spinnerTipsEnabled": false + } + } + }, + { + "module": "jackett", + "node": "", + "values": { + "pgid": 1000, + "puid": 1000 + } + }, + { + "module": "lab", + "node": "", + "values": { + "forge": "http://novox.internal:20000" + } + }, + { + "module": "lidarr", + "node": "", + "values": { + "pgid": 1000, + "puid": 1000 + } + }, + { + "module": "mailu", + "node": "", + "values": { + "domain": "novox.be", + "proxy-address": "192.0.2.10", + "sitename": "Novox", + "website": "https://novox.be" + } + }, + { + "module": "messenger", + "node": "", + "values": { + "time-zone": "Europe/Brussels" + } + }, + { + "module": "nzbget", + "node": "", + "values": { + "pgid": 1000, + "puid": 1000 + } + }, + { + "module": "ombi", + "node": "", + "values": { + "pgid": 1000, + "puid": 1000 + } + }, + { + "module": "plex", + "node": "", + "values": { + "pgid": 1000, + "puid": 1000 + } + }, + { + "module": "power", + "node": "", + "values": { + "handle-lid-switch": "suspend", + "handle-lid-switch-docked": "ignore", + "handle-lid-switch-external-power": "suspend", + "handle-power-key": "poweroff" + } + }, + { + "module": "qbittorrent", + "node": "", + "values": { + "pgid": 1000, + "puid": 1000 + } + }, + { + "module": "radarr", + "node": "", + "values": { + "pgid": 1000, + "puid": 1000 + } + }, + { + "module": "records", + "node": "", + "values": { + "repository": "novox/hq" + } + }, + { + "module": "sonarr", + "node": "", + "values": { + "pgid": 1000, + "puid": 1000 + } + }, + { + "module": "tautulli", + "node": "", + "values": { + "pgid": 1000, + "puid": 1000 + } + }, + { + "module": "xdg", + "node": "", + "values": { + "desktop-dir": "$HOME/Desktop", + "publicshare-dir": "$HOME/Public", + "videos-dir": "$HOME/Videos" + } + }, + { + "module": "claude-code", + "node": "laptop", + "values": { + "role": "the control node \u2014 the controller, the bus, the store, git hosting, CI/CD and public web apps" + } + }, + { + "module": "desk-channel", + "node": "laptop", + "values": { + "desktop-machines": [ + "g14", + "shanks" + ] + } + }, + { + "module": "hostname", + "node": "laptop", + "values": { + "hostname": "novox-be" + } + }, + { + "module": "kernel", + "node": "laptop", + "values": { + "transparent-huge-pages": "madvise" + } + }, + { + "module": "keycloak", + "node": "laptop", + "values": { + "issuer": "https://keycloak.novox.be/realms/Novox" + } + }, + { + "module": "messenger", + "node": "laptop", + "values": { + "desktop-machines": [ + "g14", + "shanks" + ] + } + }, + { + "module": "zsh", + "node": "laptop", + "values": { + "execute": "withhold" + } + }, + { + "module": "claude-code", + "node": "desktop", + "values": { + "role": "the operator's interactive desktop \u2014 full mesh control" + } + }, + { + "module": "hostname", + "node": "desktop", + "values": { + "hostname": "Shanks" + } + }, + { + "module": "mounts", + "node": "desktop", + "values": { + "shares": "media=/storage/media", + "sources": "recalbox=smb://root@192.0.2.10/Share@/mnt/recalbox" + } + }, + { + "module": "xdg", + "node": "desktop", + "values": { + "desktop-dir": "$HOME/", + "publicshare-dir": "$HOME/", + "videos-dir": "$HOME/" + } + } + ], + "modules": [ + { + "config": { + "email": { + "default": "", + "description": "The email address of the Baserow account the module's tools sign in as; unset, the code falls back to its environment.", + "trusted": true, + "type": "string" + }, + "host": { + "default": "${bound:route:name}", + "description": "The Host header the module's tools send to Baserow, a host name; the file's default is the name the module's route was given.", + "trusted": true, + "type": "string" + }, + "url": { + "default": "", + "description": "The address the module's tools reach Baserow at, as a URL; unset, the code falls back to its environment, else this machine's Baserow on port 80.", + "trusted": true, + "type": "string" + } + }, + "module": "baserow", + "resources": [ + { + "id": "mesh-state", + "mode": "0700", + "place": "mesh", + "type": "directory" + }, + { + "id": "state", + "mode": "0700", + "place": ".", + "type": "directory" + }, + { + "id": "data", + "mode": "0755", + "type": "directory" + }, + { + "content": "DATABASE_HOST=STAND-IN\nDATABASE_PORT=STAND-IN\nDATABASE_NAME=STAND-IN\nDATABASE_USER=STAND-IN\nDATABASE_PASSWORD_FILE=/run/secrets/database\nDISABLE_EMBEDDED_PSQL=true\nBASEROW_PUBLIC_URL=https://STAND-IN\n", + "id": "server-env", + "mode": "0600", + "path": "${dir:state}/server.env", + "type": "file" + }, + { + "content": "{\n \"password\": \"STAND-IN\",\n \"host\": \"STAND-IN\"\n}\n", + "id": "runtime-config", + "merge": "json", + "mode": "0600", + "path": "${dir:mesh-state}/config.json", + "protected": [ + "password" + ], + "takes-config": [ + "host", + "email", + "url" + ], + "type": "file" + } + ], + "version": "1" + }, + { + "config": { + "agents": { + "default": {}, + "description": "The agents offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.", + "trusted": true, + "type": "object" + }, + "commands": { + "default": {}, + "description": "The slash commands offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.", + "trusted": true, + "type": "object" + }, + "instructions": { + "default": {}, + "description": "The instructions every session on this machine reads, by name, each a text as one line or a list of its lines; empty adds none.", + "trusted": true, + "type": "object" + }, + "managed_settings": { + "default": {}, + "description": "Keys of Claude Code's managed settings every session on this machine obeys, such as spinnerTipsEnabled, laid over the mesh's own; empty adds none. Falls back to agent-managed-settings, the mesh's or this node's.", + "predefined": "agent-managed-settings", + "trusted": true, + "type": "object" + }, + "mcp_servers": { + "default": {}, + "description": "The tool servers every session on this machine is given beside the mesh's own, by name, each as Claude Code's managed MCP configuration takes it; empty adds none.", + "trusted": true, + "type": "object" + }, + "output_styles": { + "default": {}, + "description": "The output styles offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.", + "trusted": true, + "type": "object" + }, + "role": { + "default": "", + "description": "What this machine is for, in a sentence, written into the instructions every Claude Code session on it reads; empty says nothing.", + "trusted": true, + "type": "string" + }, + "skills": { + "default": {}, + "description": "The skills offered to every session on this machine, by name, each its files by path, a text as one line or a list of its lines; a skill without its SKILL.md is left out; empty adds none.", + "trusted": true, + "type": "object" + } + }, + "module": "claude-code", + "resources": [ + { + "id": "managed", + "mode": "0755", + "path": "/etc/claude-code", + "type": "directory" + }, + { + "content": "#!/bin/sh\n# The mesh's (module claude-code, novox/hq ADR 0266): start the agent as the account agents run as on this\n# machine. Written whole at every push: an edit here is overwritten.\nagent='STAND-IN'\nif [ \"$(id -un)\" = \"$agent\" ]; then\n\texec claude \"$@\"\nfi\n# Another account, the operator's, becomes the agent's through its own sudo: the person is the authority. The\n# operator's override of the guard travels only when it is set in this shell, as it would reach claude.\nif [ -n \"$MESH_GUARD_OVERRIDE\" ]; then\n\texec sudo -iu \"$agent\" env MESH_GUARD_OVERRIDE=\"$MESH_GUARD_OVERRIDE\" claude \"$@\"\nfi\nexec sudo -iu \"$agent\" claude \"$@\"\n", + "id": "start", + "mode": "0755", + "path": "/usr/local/bin/claude-agent", + "type": "file" + }, + { + "content": "STAND-IN\n", + "id": "agent-account-name", + "mode": "0644", + "path": "/etc/claude-code/agent-account", + "type": "file" + }, + { + "id": "agent-home", + "mode": "0700", + "path": "STAND-IN/.claude", + "type": "directory" + }, + { + "id": "state", + "mode": "0700", + "place": ".", + "type": "directory" + }, + { + "content": "{\n \"node\": \"STAND-IN\",\n \"console\": \"http://127.0.0.1:STAND-IN/mcp\"\n}\n", + "id": "facts", + "mode": "0600", + "path": "${dir:state}/facts.json", + "type": "file" + }, + { + "content": "{\n \"role\": \"\",\n \"mcp_servers\": {},\n \"managed_settings\": {},\n \"instructions\": {},\n \"output_styles\": {},\n \"skills\": {},\n \"commands\": {},\n \"agents\": {}\n}\n", + "id": "settings", + "merge": "json", + "mode": "0600", + "path": "${dir:state}/settings.json", + "takes-config": [ + "role", + "mcp_servers", + "managed_settings", + "instructions", + "output_styles", + "skills", + "commands", + "agents" + ], + "type": "file" + } + ], + "version": "1" + }, + { + "config": { + "url": { + "default": "", + "description": "The address the module's tools reach Grafana at, as a URL; unset, the code falls back to its environment, else this machine's Grafana on its port.", + "trusted": true, + "type": "string" + }, + "user": { + "default": "admin", + "description": "The Grafana account the module's tools sign in as with the vault's password; the file's default is admin.", + "trusted": true, + "type": "string" + } + }, + "module": "grafana", + "resources": [ + { + "id": "mesh-state", + "mode": "0700", + "place": "mesh", + "type": "directory" + }, + { + "id": "state", + "mode": "0700", + "place": ".", + "type": "directory" + }, + { + "id": "data", + "mode": "0700", + "type": "directory" + }, + { + "content": "STAND-IN", + "id": "admin-secret", + "mode": "0400", + "path": "${dir:state}/admin.secret", + "type": "file" + }, + { + "content": "STAND-IN", + "id": "oidc-secret", + "mode": "0400", + "path": "${dir:state}/oidc-client.secret", + "type": "file" + }, + { + "content": "GF_SERVER_ROOT_URL=https://STAND-IN\nGF_AUTH_GENERIC_OAUTH_ENABLED=true\nGF_AUTH_GENERIC_OAUTH_NAME=Keycloak\nGF_AUTH_GENERIC_OAUTH_CLIENT_ID=STAND-IN\nGF_AUTH_GENERIC_OAUTH_CLIENT_SECRET__FILE=/run/secrets/oidc-client\nGF_AUTH_GENERIC_OAUTH_SCOPES=openid email profile roles\nGF_AUTH_GENERIC_OAUTH_AUTH_URL=STAND-INSTAND-IN\nGF_AUTH_GENERIC_OAUTH_TOKEN_URL=STAND-INSTAND-IN\nGF_AUTH_GENERIC_OAUTH_API_URL=STAND-INSTAND-IN\nGF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH=contains(roles[*], 'admin') && 'Admin' || contains(realm_access.roles[*], 'admin') && 'Admin' || 'Viewer'\nGF_AUTH_GENERIC_OAUTH_USE_PKCE=true\nGF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP=true\nGF_AUTH_GENERIC_OAUTH_ALLOW_ASSIGN_GRAFANA_ADMIN=true\n", + "id": "oidc-env", + "mode": "0644", + "path": "${dir:state}/oidc.env", + "type": "file" + }, + { + "id": "influxdb-datasources", + "mode": "0755", + "path": "${dir:state}/datasources", + "type": "directory" + }, + { + "id": "influxdb-credential", + "mode": "0700", + "path": "${dir:state}/influxdb", + "type": "directory" + }, + { + "content": "STAND-IN", + "id": "influxdb-secret", + "mode": "0400", + "path": "${dir:influxdb-credential}/influxdb-api", + "type": "file" + }, + { + "content": "apiVersion: 1\n# Written by the mesh from grafana's influxdb-api binding, only while it is bound (novox/hq ADR 0330):\n# without influxdb, grafana runs with no mesh data source. grafana reads it at start. Its own name and\n# uid, so a data source somebody made in the UI is never overwritten, and read-only in the UI because\n# the mesh resets it. The password is read from the file the mesh delivers, never written here.\ndatasources:\n - name: InfluxDB (mesh)\n uid: mesh-influxdb-api\n type: influxdb\n access: proxy\n url: STAND-IN://STAND-IN:STAND-IN\n user: STAND-IN\n isDefault: false\n editable: false\n jsonData:\n dbName: STAND-IN\n httpMode: POST\n secureJsonData:\n password: $__file{/run/secrets/influxdb/influxdb-api}\n", + "id": "influxdb-datasource", + "mode": "0644", + "path": "${dir:influxdb-datasources}/mesh-influxdb.yaml", + "type": "file" + }, + { + "content": "{\n \"user\": \"admin\",\n \"password\": \"STAND-IN\"\n}\n", + "id": "runtime-config", + "merge": "json", + "mode": "0600", + "path": "${dir:mesh-state}/config.json", + "protected": [ + "password", + "token" + ], + "takes-config": [ + "user", + "url" + ], + "type": "file" + } + ], + "version": "1" + }, + { + "config": { + "url": { + "default": "", + "description": "The address the module's tools reach Home Assistant at, as a URL; unset, the code falls back to its environment, else this machine's Home Assistant on its port.", + "trusted": true, + "type": "string" + } + }, + "module": "home-assistant", + "resources": [ + { + "id": "mesh-state", + "mode": "0700", + "place": "mesh", + "type": "directory" + }, + { + "id": "state", + "mode": "0700", + "place": ".", + "type": "directory" + }, + { + "id": "config", + "mode": "0700", + "type": "directory" + }, + { + "id": "written", + "mode": "0700", + "type": "directory" + }, + { + "content": "{}\n", + "id": "runtime-config", + "merge": "json", + "mode": "0600", + "path": "${dir:mesh-state}/config.json", + "takes-config": [ + "url" + ], + "type": "file" + }, + { + "content": "MESH_HOMEASSISTANT_URL=http://127.0.0.1:STAND-IN\nMESH_HOMEASSISTANT_TOKEN_FILE=${dir:mesh-state}/token\nMESH_PROVISIONS_DIR=${dir:state}\nMESH_WRITTEN_DIR=${dir:written}\n", + "id": "provisions-env", + "mode": "0600", + "path": "${dir:state}/provisions.env", + "type": "file" + } + ], + "version": "1" + }, + { + "claims": [ + { + "name": "node-hostname", + "scope": "node", + "serves": [ + "entries", + "add", + "remove" + ] + } + ], + "config": { + "hostname": { + "description": "The machine's host name, written to /etc/hostname: the operator's name for this machine, with no default (novox/hq ADR 0112).", + "required": true, + "trusted": true, + "type": "string" + } + }, + "module": "hostname", + "resources": [ + { + "content": "# The machine's own names (module hostname, novox/hq ADR 0199, ADR 0223). Every line outside this\n# block is the operator's: kept across every push, changed through the node-hostname verbs add and\n# remove, and given back when this module goes. The mesh's names are not here: the mesh's resolver\n# answers them.\n127.0.0.1\tlocalhost\n::1\tlocalhost\n127.0.1.1\tSTAND-IN\n", + "id": "own", + "mode": "0644", + "path": "/etc/hosts", + "type": "file" + }, + { + "content": "${config:hostname}\n", + "id": "name", + "mode": "0644", + "path": "/etc/hostname", + "type": "file" + } + ], + "version": "1" + }, + { + "config": { + "url": { + "default": "", + "description": "The address the module's tools reach Icecast at, as a URL; unset, the code falls back to its environment, else this machine's Icecast on its port.", + "trusted": true, + "type": "string" + }, + "user": { + "default": "", + "description": "The Icecast admin account the module's tools sign in as; unset, the code falls back to its environment, else admin.", + "trusted": true, + "type": "string" + } + }, + "module": "icecast", + "resources": [ + { + "id": "mesh-state", + "mode": "0700", + "place": "mesh", + "type": "directory" + }, + { + "id": "state", + "mode": "0700", + "place": ".", + "type": "directory" + }, + { + "id": "logs", + "mode": "0700", + "type": "directory" + }, + { + "content": "\n \n Earth\n icemaster@localhost\n \n 100\n 2\n 524288\n 30\n 15\n 10\n 1\n 65535\n \n \n STAND-IN\n STAND-IN\n admin\n STAND-IN\n \n \n localhost\n \n 8000\n \n \n
\n \n 1\n \n /usr/share/icecast\n /var/log/icecast\n /usr/share/icecast/web\n /usr/share/icecast/admin\n \n \n \n access.log\n error.log\n 3\n 10000\n \n \n 0\n \n \n icecast\n icecast\n \n \n\n", + "id": "server-conf", + "mode": "0600", + "path": "${dir:state}/icecast.xml", + "type": "file" + }, + { + "content": "{}\n", + "id": "runtime-config", + "merge": "json", + "mode": "0600", + "path": "${dir:mesh-state}/config.json", + "takes-config": [ + "url", + "user" + ], + "type": "file" + } + ], + "version": "1" + }, + { + "config": { + "bucket": { + "default": "default", + "description": "The bucket influxdb-api names to its consumers and the module's tools write to; default is the one influxdb makes at its first start (server.env), and a machine whose server was set up with another sets its own (novox/hq issue 696, ADR 0348 clause 11).", + "trusted": true, + "type": "string" + }, + "org": { + "default": "mesh", + "description": "The organisation influxdb-api names to its consumers and the module's tools write to; mesh is the one influxdb makes at its first start (server.env), and a machine whose server was set up with another sets its own (novox/hq issue 696, ADR 0348 clause 11).", + "trusted": true, + "type": "string" + }, + "url": { + "default": "", + "description": "The address the module's tools reach influxdb at, as a URL; empty is this machine's influxdb on its port.", + "trusted": true, + "type": "string" + } + }, + "module": "influxdb", + "resources": [ + { + "id": "mesh-state", + "mode": "0700", + "place": "mesh", + "type": "directory" + }, + { + "id": "state", + "mode": "0700", + "place": ".", + "type": "directory" + }, + { + "id": "data", + "mode": "0700", + "type": "directory" + }, + { + "id": "config", + "mode": "0700", + "type": "directory" + }, + { + "id": "grants", + "mode": "0700", + "type": "directory" + }, + { + "content": "DOCKER_INFLUXDB_INIT_MODE=setup\nDOCKER_INFLUXDB_INIT_USERNAME=admin\nDOCKER_INFLUXDB_INIT_PASSWORD_FILE=/run/secrets/admin\nDOCKER_INFLUXDB_INIT_ADMIN_TOKEN_FILE=/run/secrets/admin-token\nDOCKER_INFLUXDB_INIT_ORG=mesh\nDOCKER_INFLUXDB_INIT_BUCKET=default\n", + "id": "server-env", + "mode": "0600", + "path": "${dir:state}/server.env", + "type": "file" + }, + { + "content": "{}\n", + "id": "runtime-config", + "merge": "json", + "mode": "0600", + "path": "${dir:mesh-state}/config.json", + "takes-config": [ + "bucket", + "org", + "url" + ], + "trusted": true, + "type": "file" + } + ], + "serves-checked": { + "influxdb-api": { + "bucket": "${config:bucket}", + "org": "${config:org}", + "port": 8086, + "scheme": "http" + } + }, + "version": "1" + }, + { + "config": { + "url": { + "default": "", + "description": "The address the module's tools reach Letta at, as a URL; unset, the code falls back to its environment, else this machine's Letta on port 8283.", + "trusted": true, + "type": "string" + } + }, + "module": "letta", + "resources": [ + { + "id": "mesh-state", + "mode": "0700", + "place": "mesh", + "type": "directory" + }, + { + "id": "state", + "mode": "0700", + "place": ".", + "type": "directory" + }, + { + "content": "LETTA_PG_URI=postgresql://STAND-IN@STAND-IN:STAND-IN/STAND-IN\nPGPASSFILE=/run/secrets/pgpass\nLETTA_SERVER_PASSWORD=STAND-IN\nOPENAI_API_KEY=STAND-IN\nSECURE=true\nTZ=Europe/Brussels\n", + "id": "server-env", + "mode": "0600", + "path": "${dir:state}/server.env", + "type": "file" + }, + { + "content": "*:*:*:STAND-IN:STAND-IN\n", + "id": "pgpass", + "mode": "0600", + "path": "${dir:state}/pgpass", + "type": "file" + }, + { + "content": "#!/bin/sh\n# Generated by the mesh. Do not edit: module letta writes this file and replaces it at every push.\n#\n# letta 0.6.8 prints secrets it is given to its log (novox/hq issue 268):\n# letta/server/rest_api/app.py prints its server password when it starts in secure mode;\n# startup.sh, alembic/env.py and letta/server/server.py print LETTA_PG_URI whole.\n# The URI carries no password (libpq reads it from PGPASSFILE), and the one print of the server\n# password is rewritten before the server starts. Either one failing refuses the start: a letta\n# that does not start says why here, and one that leaks says nothing.\nset -e\napp=/app/letta/server/rest_api/app.py\nsed -i 's/Using secure mode with password: {random_password}/Using secure mode (the password is not printed)/' \"$app\"\nif grep -q 'print(.*random_password' \"$app\"; then\n echo \"letta: $app still prints the server password; not starting (novox/hq issue 268)\" >&2\n exit 1\nfi\ncase \"$LETTA_PG_URI\" in\n *://*:*@*)\n echo \"letta: LETTA_PG_URI carries a password, and letta prints that URI; not starting (novox/hq issue 268)\" >&2\n exit 1\n ;;\nesac\nexec ./letta/server/startup.sh\n", + "id": "start", + "mode": "0644", + "path": "${dir:state}/start.sh", + "type": "file" + }, + { + "content": "{\n \"password\": \"STAND-IN\"\n}\n", + "id": "runtime-config", + "merge": "json", + "mode": "0600", + "path": "${dir:mesh-state}/config.json", + "protected": [ + "password" + ], + "takes-config": [ + "url" + ], + "type": "file" + } + ], + "version": "1" + }, + { + "config": { + "brand": { + "default": "Element", + "description": "Element's brand setting of config.json: the name Element shows itself as.", + "trusted": true, + "type": "string" + }, + "bug_report_endpoint_url": { + "default": "", + "description": "Element's bug_report_endpoint_url setting of config.json: where Element sends a bug report, a URL; its default here is empty, and the file's own value (Element's, naming outside servers) stays in the file and is never replaced unless this is set.", + "trusted": true, + "type": "string" + }, + "default_server_config": { + "default": { + "m.homeserver": { + "base_url": "https://${bound:route:name-homeserver}" + }, + "m.identity_server": { + "base_url": "https://vector.im" + } + }, + "description": "Element's default_server_config setting of config.json: the homeserver and identity server Element uses by default, by URL.", + "trusted": true, + "type": "object" + }, + "default_server_name": { + "default": "${bound:route:name-homeserver}", + "description": "Element's default_server_name setting of config.json: the homeserver Element signs in to by default, by name; the file's default is the module's own route.", + "trusted": true, + "type": "string" + }, + "element_call": { + "default": { + "url": "https://call.element.dev" + }, + "description": "Element's element_call setting of config.json: where Element starts a call with Element Call, a URL.", + "trusted": true, + "type": "object" + }, + "enable_presence_by_hs_url": { + "default": { + "https://matrix-client.matrix.org": false, + "https://matrix.org": false + }, + "description": "Element's enable_presence_by_hs_url setting of config.json: whether Element shows presence, per homeserver URL.", + "trusted": true, + "type": "object" + }, + "features": { + "default": { + "feature_rust_crypto": true, + "feature_video_rooms": true + }, + "description": "Element's features setting of config.json: Element's feature switches, each on or off.", + "trusted": true, + "type": "object" + }, + "integrations_rest_url": { + "default": "https://scalar.vector.im/api", + "description": "Element's integrations_rest_url setting of config.json: the integration manager's API, a URL.", + "trusted": true, + "type": "string" + }, + "integrations_ui_url": { + "default": "https://scalar.vector.im/", + "description": "Element's integrations_ui_url setting of config.json: the integration manager's page, a URL.", + "trusted": true, + "type": "string" + }, + "integrations_widgets_urls": { + "default": [ + "https://scalar.vector.im/_matrix/integrations/v1", + "https://scalar.vector.im/api", + "https://scalar-staging.vector.im/_matrix/integrations/v1", + "https://scalar-staging.vector.im/api", + "https://scalar-staging.riot.im/scalar/api" + ], + "description": "Element's integrations_widgets_urls setting of config.json: the URLs whose widgets Element trusts as the integration manager's.", + "trusted": true, + "type": "list" + }, + "room_directory": { + "default": {}, + "description": "Element's room_directory setting of config.json: the servers whose room directories Element offers; its default here is empty, and the file's own value (Element's, naming outside servers) stays in the file and is never replaced unless this is set.", + "trusted": true, + "type": "object" + }, + "show_labs_settings": { + "default": true, + "description": "Element's show_labs_settings setting of config.json: whether Element shows its Labs settings.", + "trusted": true, + "type": "boolean" + }, + "terms_and_conditions_links": { + "default": [], + "description": "Element's terms_and_conditions_links setting of config.json: the links Element shows as its terms, each a URL and its text; its default here is empty, and the file's own value (Element's, naming outside servers) stays in the file and is never replaced unless this is set.", + "trusted": true, + "type": "list" + }, + "uisi_autorageshake_app": { + "default": "element-auto-uisi", + "description": "Element's uisi_autorageshake_app setting of config.json: the app name Element files an automatic report of an undecryptable message under.", + "trusted": true, + "type": "string" + } + }, + "module": "matrix", + "resources": [ + { + "id": "state", + "mode": "0700", + "place": ".", + "type": "directory" + }, + { + "id": "db", + "mode": "0700", + "type": "directory" + }, + { + "content": "# Written by the mesh (modules/matrix). Conduit reads this file (CONDUIT_CONFIG); nothing comes\n# from the environment. server_name is the homeserver's permanent identity: every user id, room id\n# and signature in the database carries it, so it is the name this module is served under\n# (STAND-IN) and never changes once a database exists.\n[global]\nserver_name = \"STAND-IN\"\ndatabase_backend = \"rocksdb\"\ndatabase_path = \"/var/lib/matrix-conduit/\"\naddress = \"0.0.0.0\"\nport = 6167\nmax_request_size = 20000000\nallow_registration = false\nallow_federation = true\nallow_check_for_updates = true\ntrusted_servers = [\"matrix.org\"]\n", + "id": "conduit-conf", + "mode": "0644", + "path": "${dir:state}/conduit.toml", + "type": "file" + }, + { + "content": "{\n \"default_server_name\": \"STAND-IN\",\n \"default_server_config\": {\n \"m.homeserver\": {\n \"base_url\": \"https://STAND-IN\"\n },\n \"m.identity_server\": {\n \"base_url\": \"https://vector.im\"\n }\n },\n \"brand\": \"Element\",\n \"integrations_ui_url\": \"https://scalar.vector.im/\",\n \"integrations_rest_url\": \"https://scalar.vector.im/api\",\n \"integrations_widgets_urls\": [\n \"https://scalar.vector.im/_matrix/integrations/v1\",\n \"https://scalar.vector.im/api\",\n \"https://scalar-staging.vector.im/_matrix/integrations/v1\",\n \"https://scalar-staging.vector.im/api\",\n \"https://scalar-staging.riot.im/scalar/api\"\n ],\n \"bug_report_endpoint_url\": \"https://element.io/bugreports/submit\",\n \"uisi_autorageshake_app\": \"element-auto-uisi\",\n \"show_labs_settings\": true,\n \"room_directory\": {\n \"servers\": [\n \"STAND-IN\",\n \"matrix.org\",\n \"gitter.im\",\n \"libera.chat\"\n ]\n },\n \"enable_presence_by_hs_url\": {\n \"https://matrix.org\": false,\n \"https://matrix-client.matrix.org\": false\n },\n \"terms_and_conditions_links\": [\n {\n \"url\": \"https://element.io/privacy\",\n \"text\": \"Privacy Policy\"\n },\n {\n \"url\": \"https://element.io/cookie-policy\",\n \"text\": \"Cookie Policy\"\n }\n ],\n \"features\": {\n \"feature_video_rooms\": true,\n \"feature_rust_crypto\": true\n },\n \"element_call\": {\n \"url\": \"https://call.element.dev\"\n }\n}\n", + "id": "element-conf", + "merge": "json", + "mode": "0644", + "path": "${dir:state}/element.json", + "takes-config": [ + "default_server_name", + "default_server_config", + "brand", + "integrations_ui_url", + "integrations_rest_url", + "integrations_widgets_urls", + "bug_report_endpoint_url", + "uisi_autorageshake_app", + "show_labs_settings", + "room_directory", + "enable_presence_by_hs_url", + "terms_and_conditions_links", + "features", + "element_call" + ], + "type": "file" + } + ], + "version": "1" + }, + { + "claims": [ + { + "name": "node-nfs-server", + "scope": "node", + "serves": [ + "exports", + "clients", + "test", + "reload", + "adopt" + ] + } + ], + "config": { + "grants": { + "default": "none", + "description": "Which nodes may mount which share, and how: share=node:rw|ro[,node:rw|ro], entries separated by spaces, or none to export to no node whatever a node wants (novox/hq ADR 0263).", + "trusted": true, + "type": "string" + }, + "shares": { + "description": "The folders this machine exports, as space-separated name=path; the operator's, with no default (novox/hq ADR 0263).", + "required": true, + "trusted": true, + "type": "string" + } + }, + "module": "nfs-server", + "resources": [ + { + "content": "# Written by the mesh (module nfs-server, novox/hq ADR 0263). Replaced on every push; a drop-in of\n# the operator's that sorts after this one overrides it, and is theirs.\n#\n# NFS version 4 only: a client needs port 2049 and nothing else, so the module opens nothing more\n# than that, to the private network. Version 3 needs rpcbind and mountd, on ports the mesh does not open.\n[nfsd]\nvers2=n\nvers3=n\nvers4=y\nvers4.0=n\nvers4.1=y\nvers4.2=y\n", + "id": "nfs-conf", + "mode": "0644", + "path": "/etc/nfs.conf.d/50-mesh.conf", + "type": "file" + }, + { + "id": "run-dir", + "mode": "0755", + "path": "/run/nfs-server", + "type": "directory" + }, + { + "id": "wants-dir-parent", + "mode": "0755", + "path": "/var/lib/nfs-server", + "type": "directory" + }, + { + "id": "wants-dir", + "mode": "0755", + "path": "/var/lib/nfs-server/from-bus", + "type": "directory" + }, + { + "id": "config-dir", + "mode": "0755", + "path": "/etc/nfs-server", + "type": "directory" + }, + { + "content": "# Written by the mesh (module nfs-server, novox/hq ADR 0263) from this machine's assignment.\n# Replaced on every push; change the `shares` setting, never this file.\n#\n# The shares: name=folder, or name=folder:ro, one share per folder. The grants: share=node:rw|ro[,\u2026],\n# which nodes may mount each and how. The module's process exports a share to a node's private address\n# only when it is granted here and that node's mounts module wants it, every client mapped to the\n# folder's owner, and only to addresses inside the range below.\nshares=${config:shares}\ngrants=${config:grants}\nrange=STAND-IN\n", + "id": "config", + "mode": "0644", + "path": "/etc/nfs-server/shares.conf", + "type": "file" + } + ], + "version": "1" + }, + { + "config": { + "debug-max-length": { + "default": 1000, + "description": "Node-RED's debugMaxLength setting: the most characters a debug message shows, a count.", + "trusted": true, + "type": "number" + }, + "diagnostics": { + "default": { + "enabled": true, + "ui": true + }, + "description": "Node-RED's diagnostics setting: whether its diagnostics report is on, and shown in the editor.", + "trusted": true, + "type": "object" + }, + "editor-theme": { + "default": { + "projects": { + "enabled": false + } + }, + "description": "Node-RED's editorTheme setting: the editor's look and features, projects among them.", + "trusted": true, + "type": "object" + }, + "export-global-context-keys": { + "default": false, + "description": "Node-RED's exportGlobalContextKeys setting: whether a function node may list the global context's keys.", + "trusted": true, + "type": "boolean" + }, + "external-modules": { + "default": {}, + "description": "Node-RED's externalModules setting: which modules the editor may install, and from where.", + "trusted": true, + "type": "object" + }, + "flow-file": { + "default": "flows.json", + "description": "Node-RED's flowFile setting: the file in the user directory the flows are kept in.", + "trusted": true, + "type": "string" + }, + "flow-file-pretty": { + "default": true, + "description": "Node-RED's flowFilePretty setting: whether the flows file is written indented.", + "trusted": true, + "type": "boolean" + }, + "function-external-modules": { + "default": true, + "description": "Node-RED's functionExternalModules setting: whether a function node may load modules of its own.", + "trusted": true, + "type": "boolean" + }, + "logging": { + "default": { + "console": { + "audit": false, + "level": "info", + "metrics": false + } + }, + "description": "Node-RED's logging setting: the console log's level, and whether metrics and audit events are logged.", + "trusted": true, + "type": "object" + }, + "mqtt": { + "default": {}, + "description": "The broker nodes the module's MQTT step keeps pointed at the mesh's broker, as {\"brokers\": [their ids]}; empty, none.", + "trusted": true, + "type": "object" + }, + "mqtt-reconnect-time": { + "default": 15000, + "description": "Node-RED's mqttReconnectTime setting: how long an MQTT node waits before reconnecting, in milliseconds.", + "trusted": true, + "type": "number" + }, + "runtime-state": { + "default": { + "enabled": false, + "ui": false + }, + "description": "Node-RED's runtimeState setting: whether flows may be started and stopped from the editor.", + "trusted": true, + "type": "object" + }, + "serial-reconnect-time": { + "default": 15000, + "description": "Node-RED's serialReconnectTime setting: how long a serial node waits before reconnecting, in milliseconds.", + "trusted": true, + "type": "number" + }, + "time-zone": { + "default": "", + "description": "The time zone the flows schedule and format in, an IANA name such as Europe/Brussels; unset, the container's UTC.", + "trusted": true, + "type": "string" + } + }, + "module": "nodered", + "resources": [ + { + "id": "mesh-state", + "mode": "0700", + "place": "mesh", + "type": "directory" + }, + { + "id": "state", + "mode": "0700", + "place": ".", + "type": "directory" + }, + { + "id": "data", + "mode": "0700", + "type": "directory" + }, + { + "id": "written", + "mode": "0700", + "type": "directory" + }, + { + "content": "// Node-RED's settings, written by the mesh from the nodered module. What an assignment may change\n// is settings.json beside this file (merged key by key); the credentials are the mesh's secrets and\n// reach Node-RED only through this file. The flows' own credentials stay encrypted in the user\n// directory under the key Node-RED keeps there (.config.runtime.json), which is data, not this.\nconst fs = require(\"fs\");\nconst path = require(\"path\");\nconst crypto = require(\"crypto\");\n\nconst ADMIN_PASSWORD = \"STAND-IN\";\nconst API_TOKEN = \"STAND-IN\";\nconst ADMIN = { username: \"admin\", permissions: \"*\" };\n\nconst settings = JSON.parse(fs.readFileSync(path.join(__dirname, \"settings.json\"), \"utf8\"));\n// The mesh's keys, not Node-RED's: endpoints lands in every merged file; timeZone is the\n// assignment's way to set the zone flows schedule and format in; mqtt names the broker nodes the\n// module's MQTT step keeps pointed at the mesh's broker; topics is what nodered asks the broker for.\nif (settings.timeZone) process.env.TZ = settings.timeZone;\ndelete settings.timeZone;\ndelete settings.endpoints;\ndelete settings.mqtt;\ndelete settings.topics;\n\nfunction same(a, b) {\n const x = crypto.createHash(\"sha256\").update(String(a)).digest();\n const y = crypto.createHash(\"sha256\").update(String(b)).digest();\n return crypto.timingSafeEqual(x, y);\n}\n\n// The admin secret is a password, or, accepted from an existing install, the bcrypt hash its\n// settings held, so the password people already use keeps working.\nfunction passwordMatches(given) {\n if (/^\\$2[aby]\\$\\d\\d\\$/.test(ADMIN_PASSWORD)) return require(\"bcryptjs\").compare(String(given), ADMIN_PASSWORD);\n return Promise.resolve(same(given, ADMIN_PASSWORD));\n}\n\nmodule.exports = Object.assign(settings, {\n uiPort: 1880,\n adminAuth: {\n type: \"credentials\",\n users: (username) => Promise.resolve(username === ADMIN.username ? ADMIN : null),\n authenticate: (username, password) =>\n username === ADMIN.username\n ? passwordMatches(password).then((ok) => (ok ? ADMIN : null))\n : Promise.resolve(null),\n // The module's own tools call the admin API with this bearer token.\n tokens: (token) => Promise.resolve(same(token, API_TOKEN) ? { username: \"mesh\", permissions: \"*\" } : null),\n },\n});\n", + "id": "settings-code", + "mode": "0600", + "path": "${dir:state}/settings.js", + "type": "file" + }, + { + "content": "{\n \"flowFile\": \"flows.json\",\n \"flowFilePretty\": true,\n \"diagnostics\": { \"enabled\": true, \"ui\": true },\n \"runtimeState\": { \"enabled\": false, \"ui\": false },\n \"logging\": { \"console\": { \"level\": \"info\", \"metrics\": false, \"audit\": false } },\n \"exportGlobalContextKeys\": false,\n \"externalModules\": {},\n \"editorTheme\": { \"projects\": { \"enabled\": false } },\n \"functionExternalModules\": true,\n \"debugMaxLength\": 1000,\n \"mqttReconnectTime\": 15000,\n \"serialReconnectTime\": 15000\n}\n", + "id": "settings", + "merge": "json", + "mode": "0600", + "path": "${dir:state}/settings.json", + "takes-config": [ + { + "as": "flowFile", + "key": "flow-file" + }, + { + "as": "flowFilePretty", + "key": "flow-file-pretty" + }, + "diagnostics", + { + "as": "runtimeState", + "key": "runtime-state" + }, + "logging", + { + "as": "exportGlobalContextKeys", + "key": "export-global-context-keys" + }, + { + "as": "externalModules", + "key": "external-modules" + }, + { + "as": "editorTheme", + "key": "editor-theme" + }, + { + "as": "functionExternalModules", + "key": "function-external-modules" + }, + { + "as": "debugMaxLength", + "key": "debug-max-length" + }, + { + "as": "mqttReconnectTime", + "key": "mqtt-reconnect-time" + }, + { + "as": "serialReconnectTime", + "key": "serial-reconnect-time" + }, + "mqtt", + { + "as": "timeZone", + "key": "time-zone" + } + ], + "type": "file" + }, + { + "content": "{\n \"token\": \"STAND-IN\"\n}\n", + "id": "runtime-config", + "mode": "0600", + "path": "${dir:mesh-state}/config.json", + "type": "file" + }, + { + "content": "MESH_NODERED_URL=http://127.0.0.1:STAND-IN\nMESH_NODERED_CONFIG_FILE=${dir:mesh-state}/config.json\nMESH_PROVISIONS_DIR=${dir:state}\nMESH_WRITTEN_DIR=${dir:written}\n", + "id": "mqtt-env", + "mode": "0600", + "path": "${dir:state}/mqtt.env", + "type": "file" + } + ], + "version": "1" + }, + { + "claims": [ + { + "name": "node-backup", + "scope": "node", + "serves": [ + "backed-up", + "now", + "restore" + ] + } + ], + "config": { + "keep-daily": { + "default": 14, + "description": "How many nightly restore points are kept, a count from 1 to 9999: two weeks undo a mistake noticed within a fortnight. Read by the night, which thins nothing while it does not read (novox/hq issue 416); lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", + "maximum": 9999, + "minimum": 1, + "trusted": true, + "type": "number" + }, + "keep-hand-taken-days": { + "default": 7, + "description": "How many days a restore point taken by hand (node-backup.now, or the mesh's before a module moves) is kept at least, a count from 1 to 9999; it is kept until a later good night of its module exists, and this many days in any case, so the next thinning never takes the one taken against a risky change (novox/hq issue 453). Lowering it lets go of the older ones taken by hand at the next thinning, and that cannot be undone.", + "maximum": 9999, + "minimum": 1, + "trusted": true, + "type": "number" + }, + "keep-monthly": { + "default": 6, + "description": "How many monthly restore points are kept behind the weeklies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", + "maximum": 9999, + "minimum": 1, + "trusted": true, + "type": "number" + }, + "keep-weekly": { + "default": 8, + "description": "How many weekly restore points are kept behind the dailies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", + "maximum": 9999, + "minimum": 1, + "trusted": true, + "type": "number" + } + }, + "module": "restic", + "resources": [ + { + "id": "state", + "mode": "0700", + "place": ".", + "type": "directory" + }, + { + "id": "repository", + "mode": "0700", + "type": "directory" + }, + { + "content": "# What the modules on this machine back up, composed by the mesh (novox/hq to-be 43). Do not edit.\nSTAND-IN", + "id": "declared", + "mode": "0600", + "path": "${dir:state}/backups.conf", + "type": "file" + }, + { + "content": "# The data the modules on this machine declare, composed by the mesh (novox/hq ADR 0233). Do not edit.\nSTAND-IN", + "id": "data-declared", + "mode": "0600", + "path": "${dir:state}/data.conf", + "type": "file" + }, + { + "content": "# How many restore points the night keeps, written by the mesh from this machine's settings keep-daily,\n# keep-weekly, keep-monthly (novox/hq issue 416) and keep-hand-taken-days (novox/hq issue 453). Replaced on every push; change the settings, never this file.\ndaily=${config:keep-daily}\nweekly=${config:keep-weekly}\nmonthly=${config:keep-monthly}\nhand=${config:keep-hand-taken-days}\n", + "id": "retention", + "mode": "0644", + "path": "${dir:state}/retention.conf", + "trusted": true, + "type": "file" + } + ], + "version": "1" + }, + { + "config": { + "brand": { + "default": {}, + "description": "SearXNG's brand section of settings.yml, laid over the file's own and SearXNG's defaults: the links the pages show for the instance's source, issues and documentation.", + "trusted": true, + "type": "object" + }, + "categories_as_tabs": { + "default": {}, + "description": "SearXNG's categories_as_tabs section of settings.yml, laid over the file's own and SearXNG's defaults: the categories shown as tabs above the results.", + "trusted": true, + "type": "object" + }, + "engines": { + "default": [], + "description": "SearXNG's engines section of settings.yml, laid over the file's own and SearXNG's defaults: the engines to add or change, each by its name.", + "trusted": true, + "type": "list" + }, + "general": { + "default": {}, + "description": "SearXNG's general section of settings.yml, laid over the file's own and SearXNG's defaults: the instance's name, contact and debug switches.", + "trusted": true, + "type": "object" + }, + "outgoing": { + "default": {}, + "description": "SearXNG's outgoing section of settings.yml, laid over the file's own and SearXNG's defaults: how SearXNG reaches the engines: timeouts, connection pool and proxies.", + "trusted": true, + "type": "object" + }, + "plugins": { + "default": {}, + "description": "SearXNG's plugins section of settings.yml, laid over the file's own and SearXNG's defaults: the plugins to switch on or off, by their id.", + "trusted": true, + "type": "object" + }, + "preferences": { + "default": {}, + "description": "SearXNG's preferences section of settings.yml, laid over the file's own and SearXNG's defaults: the preferences a user cannot change (lock).", + "trusted": true, + "type": "object" + }, + "search": { + "default": { + "formats": [ + "html", + "json" + ] + }, + "description": "SearXNG's search section of settings.yml, laid over the file's own and SearXNG's defaults: safe search, autocomplete, the result formats offered and how long an engine is suspended after an error.", + "trusted": true, + "type": "object" + }, + "server": { + "default": { + "base_url": false, + "image_proxy": false, + "limiter": false, + "public_instance": false, + "secret_key": "${secret:secret}" + }, + "description": "SearXNG's server section of settings.yml, laid over the file's own and SearXNG's defaults: the public base URL, the request method, the limiter and the image proxy; its secret_key is the vault's.", + "trusted": true, + "type": "object" + }, + "ui": { + "default": {}, + "description": "SearXNG's ui section of settings.yml, laid over the file's own and SearXNG's defaults: the theme, the default locale and how results are shown.", + "trusted": true, + "type": "object" + } + }, + "module": "searxng", + "resources": [ + { + "id": "mesh-state", + "mode": "0700", + "place": "mesh", + "type": "directory" + }, + { + "id": "state", + "mode": "0700", + "place": ".", + "type": "directory" + }, + { + "id": "valkey-data", + "mode": "0700", + "type": "directory" + }, + { + "content": "{\n \"use_default_settings\": true,\n \"server\": {\n \"secret_key\": \"STAND-IN\",\n \"base_url\": false,\n \"limiter\": false,\n \"image_proxy\": false,\n \"public_instance\": false\n },\n \"search\": {\n \"formats\": [\"html\", \"json\"]\n },\n \"valkey\": {\n \"url\": \"valkey://valkey:6379/0\"\n }\n}\n", + "id": "settings", + "merge": "json", + "mode": "0600", + "path": "${dir:state}/settings.yml", + "takes-config": [ + "general", + "brand", + "search", + "server", + "ui", + "outgoing", + "engines", + "plugins", + "categories_as_tabs", + "preferences" + ], + "type": "file" + }, + { + "content": "{}\n", + "id": "runtime-config", + "mode": "0600", + "path": "${dir:mesh-state}/config.json", + "type": "file" + } + ], + "version": "1" + }, + { + "config": { + "pooler": { + "default": { + "max-client-connections": 100, + "pool-size": 20, + "tenant": "default" + }, + "description": "The connection pooler's settings, laid over the file's: its tenant, pool size and the most client connections it takes.", + "trusted": true, + "type": "object" + } + }, + "module": "supabase", + "resources": [ + { + "id": "state", + "mode": "0700", + "place": ".", + "type": "directory" + }, + { + "id": "db-init", + "mode": "0755", + "path": "${dir:state}/db-init", + "type": "directory" + }, + { + "id": "functions-main-dir", + "mode": "0755", + "path": "${dir:state}/functions-main", + "type": "directory" + }, + { + "id": "db-data", + "mode": "0700", + "type": "directory" + }, + { + "id": "dumps", + "mode": "0700", + "path": "${dir:state}/dumps", + "type": "directory" + }, + { + "id": "db-config", + "mode": "0755", + "type": "directory" + }, + { + "id": "storage", + "mode": "0755", + "type": "directory" + }, + { + "id": "functions", + "mode": "0755", + "type": "directory" + }, + { + "content": "{\n \"pooler\": {\n \"tenant\": \"default\",\n \"pool-size\": 20,\n \"max-client-connections\": 100\n }\n}\n", + "id": "settings", + "merge": "json", + "mode": "0644", + "path": "${dir:state}/settings.json", + "takes-config": [ + "pooler" + ], + "type": "file" + }, + { + "content": "# Written by the mesh (modules/supabase): the gateway's declarative config, credentials rendered in.\n_format_version: '2.1'\n_transform: true\n\n###\n### Consumers / Users\n###\nconsumers:\n - username: DASHBOARD\n - username: anon\n keyauth_credentials:\n - key: \"STAND-IN\"\n - username: service_role\n keyauth_credentials:\n - key: \"STAND-IN\"\n\n###\n### Access Control List\n###\nacls:\n - consumer: anon\n group: anon\n - consumer: service_role\n group: admin\n\n###\n### Dashboard credentials\n###\nbasicauth_credentials:\n - consumer: DASHBOARD\n username: \"STAND-IN\"\n password: \"STAND-IN\"\n\n###\n### API Routes\n###\nservices:\n ## Open Auth routes\n - name: auth-v1-open\n url: http://supabase-auth:9999/verify\n routes:\n - name: auth-v1-open\n strip_path: true\n paths:\n - /auth/v1/verify\n plugins:\n - name: cors\n - name: auth-v1-open-callback\n url: http://supabase-auth:9999/callback\n routes:\n - name: auth-v1-open-callback\n strip_path: true\n paths:\n - /auth/v1/callback\n plugins:\n - name: cors\n - name: auth-v1-open-authorize\n url: http://supabase-auth:9999/authorize\n routes:\n - name: auth-v1-open-authorize\n strip_path: true\n paths:\n - /auth/v1/authorize\n plugins:\n - name: cors\n\n ## Secure Auth routes\n - name: auth-v1\n _comment: 'GoTrue: /auth/v1/* -> http://auth:9999/*'\n url: http://supabase-auth:9999/\n routes:\n - name: auth-v1-all\n strip_path: true\n paths:\n - /auth/v1/\n plugins:\n - name: cors\n - name: key-auth\n config:\n hide_credentials: false\n - name: acl\n config:\n hide_groups_header: true\n allow:\n - admin\n - anon\n\n ## Secure REST routes\n - name: rest-v1\n _comment: 'PostgREST: /rest/v1/* -> http://rest:3000/*'\n url: http://supabase-rest:3000/\n routes:\n - name: rest-v1-all\n strip_path: true\n paths:\n - /rest/v1/\n plugins:\n - name: cors\n - name: key-auth\n config:\n hide_credentials: true\n - name: acl\n config:\n hide_groups_header: true\n allow:\n - admin\n - anon\n\n ## Secure GraphQL routes\n - name: graphql-v1\n _comment: 'PostgREST: /graphql/v1/* -> http://rest:3000/rpc/graphql'\n url: http://supabase-rest:3000/rpc/graphql\n routes:\n - name: graphql-v1-all\n strip_path: true\n paths:\n - /graphql/v1\n plugins:\n - name: cors\n - name: key-auth\n config:\n hide_credentials: true\n - name: request-transformer\n config:\n add:\n headers:\n - Content-Profile:graphql_public\n - name: acl\n config:\n hide_groups_header: true\n allow:\n - admin\n - anon\n\n ## Secure Realtime routes\n - name: realtime-v1-ws\n _comment: 'Realtime: /realtime/v1/* -> ws://realtime:4000/socket/*'\n url: http://realtime-dev.supabase-realtime:4000/socket\n protocol: ws\n routes:\n - name: realtime-v1-ws\n strip_path: true\n paths:\n - /realtime/v1/\n plugins:\n - name: cors\n - name: key-auth\n config:\n hide_credentials: false\n - name: acl\n config:\n hide_groups_header: true\n allow:\n - admin\n - anon\n - name: realtime-v1-rest\n _comment: 'Realtime: /realtime/v1/* -> ws://realtime:4000/socket/*'\n url: http://realtime-dev.supabase-realtime:4000/api\n protocol: http\n routes:\n - name: realtime-v1-rest\n strip_path: true\n paths:\n - /realtime/v1/api\n plugins:\n - name: cors\n - name: key-auth\n config:\n hide_credentials: false\n - name: acl\n config:\n hide_groups_header: true\n allow:\n - admin\n - anon\n ## Storage routes: the storage server manages its own auth\n - name: storage-v1\n _comment: 'Storage: /storage/v1/* -> http://storage:5000/*'\n url: http://supabase-storage:5000/\n routes:\n - name: storage-v1-all\n strip_path: true\n paths:\n - /storage/v1/\n plugins:\n - name: cors\n\n ## Edge Functions routes\n - name: functions-v1\n _comment: 'Edge Functions: /functions/v1/* -> http://functions:9000/*'\n url: http://supabase-edge-functions:9000/\n routes:\n - name: functions-v1-all\n strip_path: true\n paths:\n - /functions/v1/\n plugins:\n - name: cors\n\n ## Analytics routes\n - name: analytics-v1\n _comment: 'Analytics: /analytics/v1/* -> http://logflare:4000/*'\n url: http://supabase-analytics:4000/\n routes:\n - name: analytics-v1-all\n strip_path: true\n paths:\n - /analytics/v1/\n\n ## Secure Database routes\n - name: meta\n _comment: 'pg-meta: /pg/* -> http://pg-meta:8080/*'\n url: http://supabase-meta:8080/\n routes:\n - name: meta-all\n strip_path: true\n paths:\n - /pg/\n plugins:\n - name: key-auth\n config:\n hide_credentials: false\n - name: acl\n config:\n hide_groups_header: true\n allow:\n - admin\n\n ## Protected Dashboard - catch all remaining routes\n - name: dashboard\n _comment: 'Studio: /* -> http://studio:3000/*'\n url: http://supabase-studio:3000/\n routes:\n - name: dashboard-all\n strip_path: true\n paths:\n - /\n plugins:\n - name: cors\n - name: basic-auth\n config:\n hide_credentials: true\n", + "id": "kong-conf", + "mode": "0600", + "path": "${dir:state}/kong.yml", + "type": "file" + }, + { + "content": "GOTRUE_API_HOST=0.0.0.0\nGOTRUE_API_PORT=9999\nAPI_EXTERNAL_URL=https://STAND-IN\nGOTRUE_DB_DRIVER=postgres\nGOTRUE_DB_DATABASE_URL=postgres://supabase_auth_admin:STAND-IN@supabase-db:5432/postgres\nGOTRUE_SITE_URL=https://STAND-IN\nGOTRUE_URI_ALLOW_LIST=\nGOTRUE_DISABLE_SIGNUP=true\nGOTRUE_JWT_ADMIN_ROLES=service_role\nGOTRUE_JWT_AUD=authenticated\nGOTRUE_JWT_DEFAULT_GROUP_NAME=authenticated\nGOTRUE_JWT_EXP=3600\nGOTRUE_JWT_SECRET=STAND-IN\nGOTRUE_EXTERNAL_EMAIL_ENABLED=false\nGOTRUE_EXTERNAL_ANONYMOUS_USERS_ENABLED=false\nGOTRUE_MAILER_AUTOCONFIRM=false\nGOTRUE_SMTP_ADMIN_EMAIL=admin@example.com\nGOTRUE_SMTP_HOST=supabase-mail\nGOTRUE_SMTP_PORT=2500\nGOTRUE_SMTP_USER=fake_mail_user\nGOTRUE_SMTP_PASS=fake_mail_password\nGOTRUE_SMTP_SENDER_NAME=fake_sender\nGOTRUE_MAILER_URLPATHS_INVITE=/auth/v1/verify\nGOTRUE_MAILER_URLPATHS_CONFIRMATION=/auth/v1/verify\nGOTRUE_MAILER_URLPATHS_RECOVERY=/auth/v1/verify\nGOTRUE_MAILER_URLPATHS_EMAIL_CHANGE=/auth/v1/verify\nGOTRUE_EXTERNAL_PHONE_ENABLED=false\nGOTRUE_SMS_AUTOCONFIRM=true\n", + "id": "auth-conf", + "mode": "0600", + "path": "${dir:state}/gotrue.env", + "type": "file" + }, + { + "content": "db-uri = \"postgres://authenticator:STAND-IN@supabase-db:5432/postgres\"\ndb-schemas = \"public,storage,graphql_public\"\ndb-anon-role = \"anon\"\njwt-secret = \"STAND-IN\"\ndb-use-legacy-gucs = false\napp.settings.jwt_secret = \"STAND-IN\"\napp.settings.jwt_exp = \"3600\"\n", + "id": "rest-conf", + "mode": "0600", + "path": "${dir:state}/postgrest.conf", + "type": "file" + }, + { + "content": "# Written by the mesh (modules/supabase).\n#\n# Every sink hands logflare its API key in the x-api-key header, never as an api_key query parameter:\n# logflare prints a failed request's whole URL, query string included, in its error report, so a\n# key in the URL is a key in its log (novox/hq issue 268). supabase-analytics refuses to start\n# while this file still puts the key in a URL.\napi:\n enabled: true\n address: 0.0.0.0:9001\n\nsources:\n docker_host:\n type: docker_logs\n include_containers:\n - supabase-kong\n - supabase-auth\n - supabase-rest\n - realtime-dev.supabase-realtime\n - supabase-storage\n - supabase-edge-functions\n - supabase-db\n\ntransforms:\n project_logs:\n type: remap\n inputs:\n - docker_host\n source: |-\n .project = \"default\"\n .event_message = del(.message)\n .appname = del(.container_name)\n del(.container_created_at)\n del(.container_id)\n del(.source_type)\n del(.stream)\n del(.label)\n del(.image)\n del(.host)\n del(.stream)\n router:\n type: route\n inputs:\n - project_logs\n route:\n kong: '.appname == \"supabase-kong\"'\n auth: '.appname == \"supabase-auth\"'\n rest: '.appname == \"supabase-rest\"'\n realtime: '.appname == \"realtime-dev.supabase-realtime\"'\n storage: '.appname == \"supabase-storage\"'\n functions: '.appname == \"supabase-edge-functions\"'\n db: '.appname == \"supabase-db\"'\n # Ignores non nginx errors since they are related with kong booting up\n kong_logs:\n type: remap\n inputs:\n - router.kong\n source: |-\n req, err = parse_nginx_log(.event_message, \"combined\")\n if err == null {\n .timestamp = req.timestamp\n .metadata.request.headers.referer = req.referer\n .metadata.request.headers.user_agent = req.agent\n .metadata.request.headers.cf_connecting_ip = req.client\n .metadata.request.method = req.method\n .metadata.request.path = req.path\n .metadata.request.protocol = req.protocol\n .metadata.response.status_code = req.status\n }\n if err != null {\n abort\n }\n # Ignores non nginx errors since they are related with kong booting up\n kong_err:\n type: remap\n inputs:\n - router.kong\n source: |-\n .metadata.request.method = \"GET\"\n .metadata.response.status_code = 200\n parsed, err = parse_nginx_log(.event_message, \"error\")\n if err == null {\n .timestamp = parsed.timestamp\n .severity = parsed.severity\n .metadata.request.host = parsed.host\n .metadata.request.headers.cf_connecting_ip = parsed.client\n url, err = split(parsed.request, \" \")\n if err == null {\n .metadata.request.method = url[0]\n .metadata.request.path = url[1]\n .metadata.request.protocol = url[2]\n }\n }\n if err != null {\n abort\n }\n # Gotrue logs are structured json strings which frontend parses directly. But we keep metadata for consistency.\n auth_logs:\n type: remap\n inputs:\n - router.auth\n source: |-\n parsed, err = parse_json(.event_message)\n if err == null {\n .metadata.timestamp = parsed.time\n .metadata = merge!(.metadata, parsed)\n }\n # PostgREST logs are structured so we separate timestamp from message using regex\n rest_logs:\n type: remap\n inputs:\n - router.rest\n source: |-\n parsed, err = parse_regex(.event_message, r'^(?P