From a330c564babb9ebb14e6a767b40a8609329ff07a Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 11 Oct 2026 04:43:58 +0200 Subject: [PATCH] Say a module assigned and left out of its machine's composition as a condition (issue 380) A push leaves out a module whose settings do not compose and sends the rest, which is right, but only plan said so: nfs-server ran nowhere for days while the operator believed it ran. The self-check now raises needs-operator for a setting nobody gave, naming the setting and the command, and left-out for any other cause; both warnings, cleared once the module composes or is unassigned. status and node show list the same modules as assigned, not applied. --- cmd/mesh-controller/build.go | 3 + cmd/mesh-controller/doctor.go | 5 + cmd/mesh-controller/left_out.go | 226 ++++++++++++++++++ cmd/mesh-controller/left_out_test.go | 203 ++++++++++++++++ cmd/mesh-controller/module_health.go | 4 +- cmd/mesh-controller/nodes.go | 16 +- cmd/mesh-controller/plain_words.go | 8 + cmd/mesh-controller/readable.go | 4 + cmd/mesh-controller/standing_test.go | 2 +- cmd/mesh-controller/status.go | 13 +- .../testdata/left-out/nfs-server.json | 142 +++++++++++ internal/catalogue/declaration.go | 24 +- internal/catalogue/setting_into.go | 19 +- 13 files changed, 658 insertions(+), 11 deletions(-) create mode 100644 cmd/mesh-controller/left_out.go create mode 100644 cmd/mesh-controller/left_out_test.go create mode 100644 cmd/mesh-controller/testdata/left-out/nfs-server.json diff --git a/cmd/mesh-controller/build.go b/cmd/mesh-controller/build.go index 5b772e91..ccdd8293 100644 --- a/cmd/mesh-controller/build.go +++ b/cmd/mesh-controller/build.go @@ -835,6 +835,9 @@ type answers struct { // public name on the machine went dark. The holds were correct; they were recorded only in the // machine's own state file, and the one visible symptom was a count that did not add up. untaken map[string]map[string]int + // leftOut is, per machine, every module of its set its composition leaves out, and why (novox/hq issue + // 380): assigned and not applied, which every push said only in passing. Not well while there is any. + leftOut map[string][]leftOutModule // unheld is every module on a machine whose resources are applied through a seat nothing on // that machine holds (novox/hq ADR 0207), with the modules that could hold it. Reported, not // refused, until the switch — and while there is any, the mesh is not all well: the order the diff --git a/cmd/mesh-controller/doctor.go b/cmd/mesh-controller/doctor.go index 4986d514..084a65e0 100644 --- a/cmd/mesh-controller/doctor.go +++ b/cmd/mesh-controller/doctor.go @@ -136,6 +136,11 @@ var probeRegistry = []probe{ {ID: "D-root", Asserts: "no agent can become root without a person on a machine where the router or a channel " + "proving its sender runs: not by its own account, and not through a tool that runs its command as an account " + "that can", From: "ADR 0259 §8", Kind: kindRootNotFree, Phase: 2, run: probeAgentRoot}, + // A module assigned and left out of its machine's composition (novox/hq issue 380): said on that machine, the + // operator's, never urgent; cleared once it composes again or is unassigned. + {ID: probeLeftOutID, Asserts: "no module assigned to a machine is left out of its composition unsaid: each " + + "raises needs-operator for a setting nobody gave, left-out for any other cause", From: "issue 380", + Kind: kindLeftOut, Raises: []string{kindNeedsOperator}, Phase: 1, run: probeLeftOut}, {ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals", From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs}, // The core's health definitions (novox/hq to-be 45 §8, ADR 0236): what a core component's new build is diff --git a/cmd/mesh-controller/left_out.go b/cmd/mesh-controller/left_out.go new file mode 100644 index 00000000..e530140c --- /dev/null +++ b/cmd/mesh-controller/left_out.go @@ -0,0 +1,226 @@ +package main + +import ( + "context" + "errors" + "fmt" + "sort" + "strings" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" +) + +// A module assigned to a machine and left out of its composition is said (novox/hq issue 380). +// +// **An omission is a finding, never a refusal of the push** (ADR 0163, rule 6): the machine is sent everything +// else, and the module's held things are kept. Until issue 380 the only place it showed was `plan`: nfs-server was +// assigned to the home server for days, every push left it out for a setting nobody gave, and the operator believed +// it ran. So the self-check composes every machine as the next push would (Resolution.LeftOutBecause, the push's +// own judgement) and raises a condition on that machine for each module it leaves out: +// +// - a setting nobody gave (catalogue.UnsetSettingError) is the operator's to give: kind needs-operator, naming +// the module, the setting and the command that sets it; +// - any other cause is said as a module not working is: kind left-out, a warning with the reason as evidence. +// +// Never urgent and never escalated by age (as ADR 0283 decision 5): nothing the machine ran was undone. It clears +// on the first run that no longer finds it — the module composed again, or no longer assigned. `status` and +// `node show` list the same modules under "assigned, not applied" with the same reason. +// +// A module left out because its stored manifest has a key this controller does not know is not raised here: the +// catalogue's own unknown-field condition says it once for the whole mesh (ADR 0262); it is still listed. + +const ( + // probeLeftOutID is the self-check's probe that raises and clears these conditions. + probeLeftOutID = "D-left-out" + // kindLeftOut is a module left out for any cause but a setting nobody gave; it is also every such condition's + // token, whichever its kind, so a cause that changes is the same condition said anew. + kindLeftOut = "left-out" +) + +// leftOutModule is one module of a machine's set that its composition leaves out, and why. +type leftOutModule struct { + Module string + // Setting is the setting nobody gave, when that is the cause. + Setting string + // Why is the composition's own reason, whole. + Why string + // Unread is a stored manifest this controller cannot read whole (said by the catalogue's condition). + Unread bool +} + +// leftOutOf is every module of a machine's resolution that a push would leave out, sorted. Pure: the judgement +// the push makes (catalogue.Resolution.LeftOutBecause), nothing allocated. +func leftOutOf(plan catalogue.Resolution, settings catalogue.SettingsBy, adopted bool) []leftOutModule { + because := plan.LeftOutBecause(settings, adopted) + out := make([]leftOutModule, 0, len(because)) + for module, why := range because { + l := leftOutModule{Module: module, Why: oneLine(why.Error())} + var unset *catalogue.UnsetSettingError + var unread *catalogue.UnreadManifestError + switch { + case errors.As(why, &unset): + l.Setting = unset.Setting + case errors.As(why, &unread): + l.Unread = true + } + out = append(out, l) + } + sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module }) + return out +} + +// settingCommand is the command that gives a module's setting on one machine. +func settingCommand(module, setting, node string) string { + return fmt.Sprintf("`settings set %s '{%q: …}' --node %s`", module, setting, node) +} + +// reason is why a module is left out, as `status`, `node show` and the condition's summary say it: for a setting +// nobody gave, the setting and the command that gives it; otherwise the composition's own words. +func (l leftOutModule) reason(node string) string { + if l.Setting != "" { + return fmt.Sprintf("nothing sets its setting %q, so every push leaves it out — %s sets it", l.Setting, + settingCommand(l.Module, l.Setting, node)) + } + return "every push leaves it out: " + l.Why +} + +// leftOutObservations are the conditions a machine's left-out modules raise, one each. +func leftOutObservations(node string, left []leftOutModule) []conditions.Observation { + var out []conditions.Observation + for _, l := range left { + if l.Unread { + continue + } + out = append(out, leftOutObservation(node, l)) + } + return out +} + +// leftOutObservation is one module left out of one machine's composition: needs-operator for a setting nobody +// gave, left-out otherwise; a warning either way, the operator's to resolve. +func leftOutObservation(node string, l leftOutModule) conditions.Observation { + o := conditions.Observation{Scope: conditions.ScopeModule, ID: l.Module + "." + node, Token: kindLeftOut, + Kind: kindLeftOut, Machine: node, Severity: conditions.Warning, Resolver: conditions.ResolverOperator, + Summary: fmt.Sprintf("%s is assigned to %s and not applied: %s", l.Module, node, l.reason(node)), + Said: l.Why} + w := leftOutWords(l.Module, node, l.Setting) + if l.Setting != "" { + o.Kind = kindNeedsOperator + } else { + // The composition's words may name a path or an address, which the operator's channel withholds: the + // summary sends them to the evidence, and `plan` says them in full. + o.Summary = fmt.Sprintf("%s is assigned to %s and not applied: every push leaves it out, because what is "+ + "set for it does not compose with its definition — the evidence and `plan %s` say why", l.Module, node, node) + } + o.Headline, o.Explanation, o.Needs, o.Resolved = w.Headline, w.Explanation, w.Needs, w.Resolved + return o +} + +// leftOutWords is what the operator reads of a module left out (ADR 0253): plain, the act named. +func leftOutWords(module, node, setting string) words { + w := words{ + Headline: fmt.Sprintf("%s is not applied on %s", module, node), + Explanation: fmt.Sprintf("%s is assigned to %s, and every update of %s leaves it out because what is set "+ + "for it does not fit its definition. Nothing of it changes there; the rest of %s is updated as usual.", + module, node, node, node), + Needs: fmt.Sprintf("read why in the details, then change what is set for %s or unassign it.", module), + Resolved: fmt.Sprintf("%s on %s is no longer left out", module, node), + } + if setting != "" { + w.Explanation = fmt.Sprintf("%s is assigned to %s, and every update of %s leaves it out because nothing "+ + "sets its setting %s. Nothing of it runs there until it is set; the rest of %s is updated as usual.", + module, node, node, setting, node) + w.Needs = fmt.Sprintf("set %s for %s on %s, or approve it when it is proposed to you.", setting, module, node) + // A setting's name that is not plain (a dotted key) is in the summary instead. + if _, ok := conditions.PlainWords(w, node); !ok { + w.Explanation = fmt.Sprintf("%s is assigned to %s, and every update of %s leaves it out because a "+ + "setting it needs is not set. Nothing of it runs there until it is set; the details name it.", + module, node, node) + w.Needs = fmt.Sprintf("set what %s needs on %s; the details name the setting.", module, node) + } + } + return w +} + +// probeLeftOut is the self-check's probe of issue 380: every machine's set is judged as its next push would +// judge it, and each module left out raises its condition. A machine that does not resolve is passed over — D1 +// says it — and a store that cannot be read is an error, never "nothing left out" (ADR 0227 rule 4). +func probeLeftOut(ctx context.Context, d *doctor) ([]conditions.Observation, error) { + nodes, err := d.open.inventory.Nodes(ctx) + if err != nil { + return nil, err + } + var out []conditions.Observation + for _, n := range nodes { + plan, settings, err := planFor(ctx, d.open, n.Name) + if err != nil { + if unresolvable(err) { + continue + } + return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err) + } + out = append(out, leftOutObservations(n.Name, leftOutOf(plan, settings, n.Adopted))...) + if ctx.Err() != nil { + return nil, ctx.Err() + } + } + return out, nil +} + +// notAppliedLines is a machine's "assigned, not applied" as `node show` prints it: one line a module, with the +// reason its condition says. +func notAppliedLines(node string, left []leftOutModule) []string { + if len(left) == 0 { + return nil + } + lines := []string{"", " assigned, not applied:"} + for _, l := range left { + lines = append(lines, fmt.Sprintf(" %-22s %s", l.Module, l.reason(node))) + } + return lines +} + +// machineNotApplied is one module assigned to a machine and left out of its composition, in `status --json`. +type machineNotApplied struct { + Node string `json:"node"` + Module string `json:"module"` + Setting string `json:"setting,omitempty"` + Reason string `json:"reason"` +} + +// notApplied is every machine's left-out modules, in a stated order, as `status --json` carries them. +func notApplied(left map[string][]leftOutModule) []machineNotApplied { + names := make([]string, 0, len(left)) + for name := range left { + names = append(names, name) + } + sort.Strings(names) + var out []machineNotApplied + for _, name := range names { + for _, l := range left[name] { + out = append(out, machineNotApplied{Node: name, Module: l.Module, Setting: l.Setting, Reason: l.reason(name)}) + } + } + return out +} + +// printNotApplied is status's "assigned, not applied", every machine's. +func printNotApplied(left map[string][]leftOutModule) { + rows := notApplied(left) + if len(rows) == 0 { + return + } + fmt.Printf("%d module(s) assigned, not applied — every push leaves them out, and each raises a condition:\n", + len(rows)) + for _, r := range rows { + fmt.Printf(" %-12s %-22s %s\n", r.Node, r.Module, r.Reason) + } + fmt.Println() +} + +// isLeftOutCondition is whether a condition is this probe's, which the machine's health statements neither +// raise nor clear. +func isLeftOutCondition(c conditions.Condition) bool { + return c.Source == probeLeftOutID || strings.HasSuffix(c.Key, "."+kindLeftOut) +} diff --git a/cmd/mesh-controller/left_out_test.go b/cmd/mesh-controller/left_out_test.go new file mode 100644 index 00000000..100e6a19 --- /dev/null +++ b/cmd/mesh-controller/left_out_test.go @@ -0,0 +1,203 @@ +package main + +import ( + "context" + "encoding/json" + "os" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" +) + +// novox/hq issue 380: nfs-server was assigned to the home server for days and every push left it out — "nfs-server +// has a file that says ${setting:shares}, and nothing sets shares for it" — and nothing but `plan` said so. The +// manifest is the catalogue's own at the commit that added it (mesh-catalog 48fba44), which still says +// ${setting:shares}; the reason below is the one the live push printed. + +// leftOutNFS is the resolution of a machine assigned that nfs-server, with the settings given. +func leftOutNFS(t *testing.T) catalogue.Resolution { + t.Helper() + raw, err := os.ReadFile("testdata/left-out/nfs-server.json") + if err != nil { + t.Fatal(err) + } + m, err := catalogue.ParseManifest(raw) + if err != nil { + t.Fatal(err) + } + return catalogue.Resolution{Modules: []catalogue.Manifest{m}} +} + +// sharesGiven is the operator's setting for it on the machine. +func sharesGiven(value string) catalogue.SettingsBy { + return catalogue.SettingsBy{"nfs-server": {{From: "anchor", Values: map[string]any{"shares": value}}}} +} + +func TestAModuleLeftOutForASettingNobodyGaveNeedsTheOperatorNamingTheSettingAndTheCommand(t *testing.T) { + left := leftOutOf(leftOutNFS(t), nil, false) + if len(left) != 1 || left[0].Module != "nfs-server" || left[0].Setting != "shares" { + t.Fatalf("left out: %+v", left) + } + if !strings.Contains(left[0].Why, `nothing sets "shares" for it`) { + t.Fatalf("the reason is not the push's own: %s", left[0].Why) + } + obs := leftOutObservations("anchor", left) + if len(obs) != 1 { + t.Fatalf("raised %+v", obs) + } + o := obs[0] + if o.Key() != "module.nfs-server.anchor.left-out" || o.Kind != kindNeedsOperator || o.Machine != "anchor" || + o.Severity != conditions.Warning || o.Resolver != conditions.ResolverOperator { + t.Fatalf("raised %s as %s, %s, by %s, on %q", o.Key(), o.Kind, o.Severity, o.Resolver, o.Machine) + } + for _, want := range []string{"nfs-server", "anchor", `"shares"`, "`settings set nfs-server '{\"shares\": …}' --node anchor`"} { + if !strings.Contains(o.Summary, want) { + t.Errorf("the summary does not name %s: %s", want, o.Summary) + } + } + if o.Said != left[0].Why { + t.Errorf("the evidence is not the composition's reason: %s", o.Said) + } + plainExample(t, o, "nfs-server is not applied on anchor", + "Needs you: set shares for nfs-server on anchor, or approve it when it is proposed to you. nfs-server is assigned to "+ + "anchor, and every update of anchor leaves it out because nothing sets its setting shares. Nothing of it "+ + "runs there until it is set; the rest of anchor is updated as usual.") +} + +func TestAModuleLeftOutForAnotherCauseIsAWarningWithTheReasonAsEvidence(t *testing.T) { + // A setting stored that its definition can no longer compose: one with a line break (issue 339). + left := leftOutOf(leftOutNFS(t), sharesGiven("library=/srv/library\nmedia=/srv/media"), false) + if len(left) != 1 || left[0].Setting != "" { + t.Fatalf("left out: %+v", left) + } + obs := leftOutObservations("anchor", left) + if len(obs) != 1 { + t.Fatalf("raised %+v", obs) + } + o := obs[0] + if o.Key() != "module.nfs-server.anchor.left-out" || o.Kind != kindLeftOut || o.Severity != conditions.Warning { + t.Fatalf("raised %s as %s, %s", o.Key(), o.Kind, o.Severity) + } + if !strings.Contains(o.Said, "holds a line break") || strings.Contains(o.Summary, "/srv/") { + t.Fatalf("the reason is not the evidence, or the summary carries it to the channel:\n%s\n%s", o.Summary, o.Said) + } + plainExample(t, o, "nfs-server is not applied on anchor", + "Needs you: read why in the details, then change what is set for nfs-server or unassign it. nfs-server is assigned to "+ + "anchor, and every update of anchor leaves it out because what is set for it does not fit its definition. "+ + "Nothing of it changes there; the rest of anchor is updated as usual.") +} + +// The self-check raises it, keeps it a warning however long it stands, and clears it when the module composes +// again or is no longer assigned; a machine's health statement neither clears nor raises it. +func TestALeftOutModulesConditionClearsWhenItComposesAgainOrIsUnassigned(t *testing.T) { + plan, settings := leftOutNFS(t), catalogue.SettingsBy(nil) + withProbes(t, probe{ID: probeLeftOutID, Asserts: "the test's", Kind: kindLeftOut, Phase: 1, + run: func(context.Context, *doctor) ([]conditions.Observation, error) { + return leftOutObservations("anchor", leftOutOf(plan, settings, false)), nil + }}) + store := conditions.NewInMemory() + k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store}) + defer k.Close(context.Background()) + d := &doctor{keeper: k, teller: &conditions.Told{}, host: "anchor"} + key := "module.nfs-server.anchor.left-out" + openOnes := func() map[string]conditions.Condition { + t.Helper() + open, err := k.Open(t.Context()) + if err != nil { + t.Fatal(err) + } + out := map[string]conditions.Condition{} + for _, c := range open { + out[c.Key] = c + } + return out + } + + d.runOnce(t.Context(), "a test") + c, raised := openOnes()[key] + if !raised || c.Kind != kindNeedsOperator || c.Severity != conditions.Warning { + t.Fatalf("a module left out raised %+v", openOnes()) + } + // A statement from the machine that says nothing of it — the module runs nothing there — leaves it open. + if err := judgeModuleHealth(t.Context(), nil, k, "anchor", map[string][]inventory.ResourceHealth{}, nil, + time.Now().Add(72*time.Hour)); err != nil { + t.Fatal(err) + } + if _, still := openOnes()[key]; !still { + t.Fatal("a health statement that says nothing of the module cleared its left-out condition") + } + d.runOnce(t.Context(), "a test") + if c := openOnes()[key]; c.Severity != conditions.Warning { + t.Fatalf("after a health statement and days, it is %+v", openOnes()) + } + + // The setting given: it composes, and the condition clears. + settings = sharesGiven("library=/srv/library") + d.runOnce(t.Context(), "a test") + if _, still := openOnes()[key]; still { + t.Fatalf("composed again, still open: %+v", openOnes()) + } + + // Left out again, then unassigned: no longer in the machine's set, and it clears. + settings = nil + d.runOnce(t.Context(), "a test") + if _, raised := openOnes()[key]; !raised { + t.Fatal("left out again and not raised") + } + plan = catalogue.Resolution{} + d.runOnce(t.Context(), "a test") + if _, still := openOnes()[key]; still { + t.Fatalf("unassigned, still open: %+v", openOnes()) + } +} + +func TestTheLeftOutProbeIsInTheRegistry(t *testing.T) { + for _, p := range probeRegistry { + if p.ID == probeLeftOutID { + if p.run == nil || p.Kind != kindLeftOut { + t.Fatalf("%+v", p) + } + return + } + } + t.Fatalf("no probe %s: a module left out is said nowhere", probeLeftOutID) +} + +// status and node show list it under "assigned, not applied" with the reason the condition says, and status is +// not well while there is one. +func TestStatusAndNodeListAModuleAssignedAndNotApplied(t *testing.T) { + left := map[string][]leftOutModule{"anchor": leftOutOf(leftOutNFS(t), nil, false)} + reason := leftOutObservations("anchor", left["anchor"])[0].Summary + asked := answers{leftOut: left} + if asked.well() { + t.Fatal("a mesh with a module assigned and not applied is called well") + } + shown := printed(t, func() error { return printStatus(asked) }) + if !strings.Contains(shown, "1 module(s) assigned, not applied") || !strings.Contains(shown, "nfs-server") || + !strings.Contains(shown, "`settings set nfs-server '{\"shares\": …}' --node anchor` sets it") { + t.Fatalf("status says:\n%s", shown) + } + if !strings.Contains(reason, left["anchor"][0].reason("anchor")) { + t.Fatalf("status and the condition say different reasons:\n%s\n%s", shown, reason) + } + body, err := statusAsJSON(asked) + if err != nil { + t.Fatal(err) + } + var doc struct { + NotApplied []machineNotApplied `json:"not-applied"` + } + if err := json.Unmarshal(body, &doc); err != nil || len(doc.NotApplied) != 1 || + doc.NotApplied[0].Setting != "shares" || doc.NotApplied[0].Node != "anchor" { + t.Fatalf("status --json: %v %s", err, body) + } + lines := strings.Join(notAppliedLines("anchor", left["anchor"]), "\n") + if !strings.Contains(lines, "assigned, not applied:") || !strings.Contains(lines, "nfs-server") || + !strings.Contains(lines, `nothing sets its setting "shares"`) { + t.Fatalf("node show says:\n%s", lines) + } +} diff --git a/cmd/mesh-controller/module_health.go b/cmd/mesh-controller/module_health.go index fc7cabbe..58f0d0ac 100644 --- a/cmd/mesh-controller/module_health.go +++ b/cmd/mesh-controller/module_health.go @@ -147,9 +147,11 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi } standing := map[string]conditions.Condition{} for _, c := range open { + // A module left out of the composition is the self-check's to raise and clear, never a statement's + // (novox/hq issue 380): its needs-operator is not cleared for not being in what the machine runs. if (c.Kind == kindModuleUnhealthy || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound || c.Kind == kindNeedsOperator) && - c.Subject.Machine == node { + c.Subject.Machine == node && !isLeftOutCondition(c) { standing[c.Key] = c } } diff --git a/cmd/mesh-controller/nodes.go b/cmd/mesh-controller/nodes.go index a89adcf2..f5746f65 100644 --- a/cmd/mesh-controller/nodes.go +++ b/cmd/mesh-controller/nodes.go @@ -44,7 +44,7 @@ func nodeCommand(ctx context.Context, args []string) error { if len(args) != 2 { return errors.New("node show ") } - return showNode(ctx, inv, args[1]) + return showNode(ctx, open, args[1]) case "add": return addNode(ctx, inv, args[1:]) @@ -787,7 +787,8 @@ func roughly(d time.Duration) string { // // It is also where "what should it be configured as" is read. The same line that gates an // assignment carries `card1-DP-1`, and a person composing settings for that machine needs it. -func showNode(ctx context.Context, inv *inventory.Inventory, name string) error { +func showNode(ctx context.Context, stored *stores, name string) error { + inv := stored.inventory node, err := inv.NodeByName(ctx, name) if err != nil { return err @@ -889,6 +890,17 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error if len(assigned) > 0 { fmt.Printf("\n assigned: %s\n", strings.Join(assigned, ", ")) } + // And which of them a push leaves out, and why (novox/hq issue 380): judged as the push judges it, the same + // reason its condition says. Not computable is said, never read as "all applied". + plan, settings, err := planFor(ctx, stored, name) + switch { + case err != nil: + fmt.Printf("\n whether a push leaves any assigned module out is NOT known: %s\n", oneLine(err.Error())) + default: + for _, line := range notAppliedLines(name, leftOutOf(plan, settings, node.Adopted)) { + fmt.Println(line) + } + } return nil } diff --git a/cmd/mesh-controller/plain_words.go b/cmd/mesh-controller/plain_words.go index 29e5da85..5b564929 100644 --- a/cmd/mesh-controller/plain_words.go +++ b/cmd/mesh-controller/plain_words.go @@ -231,6 +231,14 @@ var plainWordings = map[string]func(conditions.Observation) words{ w := needsOperatorWords(orModule(module), node, nil) return w }), + kindLeftOut: worded(func(o conditions.Observation) words { + // The observation carries its own words (novox/hq issue 380); these are its kind's alone. + module := "" + if o.Scope == conditions.ScopeModule && o.Machine != "" { + module = strings.TrimSuffix(o.ID, "."+o.Machine) + } + return leftOutWords(orModule(module), machineOr(o, "a machine"), "") + }), kindProviderFailing: worded(func(o conditions.Observation) words { thing, consumer := conditions.ThingWords(o), idPart(o, 2) if consumer == "" { diff --git a/cmd/mesh-controller/readable.go b/cmd/mesh-controller/readable.go index d286b693..a5b86497 100644 --- a/cmd/mesh-controller/readable.go +++ b/cmd/mesh-controller/readable.go @@ -69,6 +69,9 @@ type meshStatus struct { // **A document without this said an outage was a well mesh.** Read from what each machine // reported, so it is the machine's account and not the mesh's take-time listing. Untaken []machineUntaken `json:"untaken,omitempty"` + // NotApplied is every module assigned to a machine and left out of its composition, with why (novox/hq + // issue 380). Absent when every module composes. + NotApplied []machineNotApplied `json:"not-applied,omitempty"` // Filtered is every converged machine that is not filtered by the mesh alone (novox/hq ADR // 0168), one entry per rule set the mesh did not write — the found firewall in force again, // or a chain nobody speaks for. Absent when every converged machine is filtered by the mesh @@ -251,6 +254,7 @@ func statusAsJSON(asked answers) ([]byte, error) { } } out.Unheld = asked.unheld + out.NotApplied = notApplied(asked.leftOut) out.HandActsThisWeek, out.HandActsUnread = asked.handActs, asked.handActsUnread out.HealsThisWeek, out.HealsUnread = asked.heals, asked.healsUnread // In brief, as `conditions` lists them: status leads with every open condition, and their whole diff --git a/cmd/mesh-controller/standing_test.go b/cmd/mesh-controller/standing_test.go index e7fd3121..d174dd0c 100644 --- a/cmd/mesh-controller/standing_test.go +++ b/cmd/mesh-controller/standing_test.go @@ -66,7 +66,7 @@ func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) { } // Both machines' `node show` name it: where the provider runs, and where the consumer is. for _, node := range []string{"anchor", "laptop"} { - shown := printed(t, func() error { return showNode(ctx, open.inventory, node) }) + shown := printed(t, func() error { return showNode(ctx, open, node) }) if !strings.Contains(shown, "open condition(s) about this machine") || !strings.Contains(shown, "mesh_laptop_dashboard") { t.Fatalf("node show %s does not name it:\n%s", node, shown) } diff --git a/cmd/mesh-controller/status.go b/cmd/mesh-controller/status.go index a9699cce..91c44c9c 100644 --- a/cmd/mesh-controller/status.go +++ b/cmd/mesh-controller/status.go @@ -289,6 +289,10 @@ func printStatus(asked answers) error { fmt.Printf("\n `take ` compares what runs against what it declares, and runs it\n\n") } + // Assigned and not applied (novox/hq issue 380): before what is merely reported, because it reads like work + // finished and is none. + printNotApplied(asked.leftOut) + if len(asked.unheld) > 0 { // **Reported, and not refused yet** (novox/hq ADR 0207 §4). Each machine still resolves and // is sent what it would be; this says which of its modules depend on a seat nothing there @@ -456,6 +460,13 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) { return answers{}, err } plans[n.Name] = planned{plan, settings} + // And which of its modules a push leaves out (novox/hq issue 380), judged as the push judges it. + if left := leftOutOf(plan, settings, n.Adopted); len(left) > 0 { + if out.leftOut == nil { + out.leftOut = map[string][]leftOutModule{} + } + out.leftOut[n.Name] = left + } out.unheld = append(out.unheld, plan.Unheld...) // And which of its modules a provider leaves out of its grants, for an identity too long // for what the provision keeps (novox/hq ADR 0225) — judged from the consumer's own @@ -604,7 +615,7 @@ func untakenModules(ctx context.Context, inv *inventory.Inventory, nodes []inven // read as success for the whole of the edge cut-over outage (novox/hq 04-ISSUES/125). func (a answers) well() bool { return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 && - len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 && + len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 && len(a.leftOut) == 0 && len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.overflowing) == 0 && len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingOpen(a.pending) } diff --git a/cmd/mesh-controller/testdata/left-out/nfs-server.json b/cmd/mesh-controller/testdata/left-out/nfs-server.json new file mode 100644 index 00000000..780e09ba --- /dev/null +++ b/cmd/mesh-controller/testdata/left-out/nfs-server.json @@ -0,0 +1,142 @@ +{ + "module": "nfs-server", + "version": "1", + "upgrade": { + "policy": "record", + "why": "the folders other machines mount: a build that breaks the exports leaves every client's mount hanging or refused, and the gate on this one machine does not see the clients (hq ADR 0236, ADR 0263)" + }, + "capabilities": [ + "package-manager", + "service-manager" + ], + "provides": [ + { + "name": "nfs-share", + "scope": "mesh", + "identity": false + } + ], + "data": { + "consumers": { + "nfs-share": { + "class": "none", + "why": "the shared folders are the operator's data (hq ADR 0051): what a client writes lands in them, and they are protected where the operator declares them, never by this module, which keeps nothing of a consumer's" + } + } + }, + "claims": [ + { + "name": "node-nfs-server", + "scope": "node", + "serves": [ + "exports", + "clients", + "test", + "reload", + "adopt" + ] + } + ], + "state": [ + { + "name": "exports", + "ttl-seconds": 120, + "per-machine": true + } + ], + "tools": [ + "nfs_health" + ], + "listens": [ + { + "name": "nfs", + "port": 2049, + "protocol": "tcp", + "from": "mesh", + "fixed": true, + "why": "the shares, to the mesh's machines only (hq ADR 0263): NFS version 4 alone, which needs no other port, and never the home network, where a device that is not a node could claim any user id" + } + ], + "resources": [ + { + "id": "package", + "type": "package", + "package": "nfs-utils" + }, + { + "id": "nfs-conf", + "type": "file", + "path": "/etc/nfs.conf.d/50-mesh.conf", + "mode": "0644", + "content": "# Written by the mesh (module nfs-server, novox/hq ADR 0263). Replaced on every push; a drop-in of\n# the operator's that sorts after this one overrides it, and is theirs.\n#\n# NFS version 4 only: a client needs port 2049 and nothing else, so the module opens nothing more\n# than that, to the private network. Version 3 needs rpcbind and mountd, on ports the mesh does not open.\n[nfsd]\nvers2=n\nvers3=n\nvers4=y\nvers4.0=n\nvers4.1=y\nvers4.2=y\n" + }, + { + "id": "config-dir", + "type": "directory", + "path": "/etc/nfs-server", + "mode": "0755" + }, + { + "id": "config", + "type": "file", + "path": "/etc/nfs-server/shares.conf", + "mode": "0644", + "content": "# Written by the mesh (module nfs-server, novox/hq ADR 0263) from this machine's assignment.\n# Replaced on every push; change the `shares` setting, never this file.\n#\n# The shares: name=folder, or name=folder:ro, one share per folder. The module's process exports each\n# to the private network's range below, every client mapped to the folder's owner.\nshares=${setting:shares}\nrange=${machine:mesh-range}\n" + }, + { + "id": "run-dir", + "type": "directory", + "path": "/run/nfs-server", + "mode": "0755" + }, + { + "id": "server", + "type": "service", + "unit": "nfs-server.service", + "state": "running", + "boot": "enabled", + "restart-on": [ + "nfs-conf" + ], + "health": { + "kind": "unit" + } + }, + { + "id": "exports", + "type": "process", + "name": "nfs-server-exports", + "artifact": "tools", + "run": [ + "./nfs-server", + "exports" + ], + "restart-on": [ + "config" + ], + "health": { + "kind": "tool", + "tool": "nfs_health", + "interval": "60s", + "timeout": "10s", + "looks": 2, + "grace": "90s" + } + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/nfs-server", + "binary": "nfs-server", + "loads": [ + "nfs-server" + ] + } + ] + } +} diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index fb9010bc..a74ca6dc 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -342,18 +342,38 @@ func (e *NotMadeError) Error() string { // compose. Empty when every module composes. The same judgement SetSettings makes before storing. func (r Resolution) LeftOut(settings SettingsBy, adopted bool) map[string]string { out := map[string]string{} + for module, why := range r.LeftOutBecause(settings, adopted) { + out[module] = why.Error() + } + return out +} + +// LeftOutBecause is LeftOut with each reason as the error it was, so a reader can tell a setting nobody +// gave (an *UnsetSettingError) from any other cause and say it as the operator's to give (novox/hq issue +// 380). An UnreadManifestError for a stored manifest this controller cannot read whole. +func (r Resolution) LeftOutBecause(settings SettingsBy, adopted bool) map[string]error { + out := map[string]error{} for _, m := range r.Modules { if why := UnknownFieldReason(m); why != "" { - out[m.Module] = why + out[m.Module] = &UnreadManifestError{Module: m.Module, said: why} continue } if err := JudgeSettings(m, settings[m.Module], adopted); err != nil { - out[m.Module] = err.Error() + out[m.Module] = err } } return out } +// UnreadManifestError is a module left out because its stored manifest has a key this controller does not know +// (novox/hq ADR 0262): said once for the whole mesh, by the catalogue's own condition, not per machine. +type UnreadManifestError struct { + Module string + said string +} + +func (e *UnreadManifestError) Error() string { return e.said } + // Compose is Declaration with the owner of every resource said. func (r Resolution) Compose(with Rendering) (Composed, error) { owner := map[string]string{} diff --git a/internal/catalogue/setting_into.go b/internal/catalogue/setting_into.go index 9ee165b7..30967997 100644 --- a/internal/catalogue/setting_into.go +++ b/internal/catalogue/setting_into.go @@ -38,6 +38,17 @@ func settingsUsed(content string) []string { return keys } +// UnsetSettingError is a module whose definition says ${setting:} where nothing sets that key: typed, so +// that whoever reads why a module was left out of a machine can tell a setting nobody gave — the operator's +// to give, named with the command that gives it — from any other reason (novox/hq issue 380). Its words are +// the refusal's, unchanged. +type UnsetSettingError struct { + Module, Setting string + said string +} + +func (e *UnsetSettingError) Error() string { return e.said } + // settingInto fills a file's ${setting:…} placeholders from the layers over a module. // // The last layer setting a key wins, which is the node's over the mesh's over the module's own @@ -58,12 +69,12 @@ func settingInto(resource map[string]any, layers []Layer, module string) error { for _, key := range settingsUsed(content) { value, set := settingValue(layers, key) if !set { - return fmt.Errorf( + return &UnsetSettingError{Module: module, Setting: key, said: fmt.Sprintf( "%s has a file that says ${setting:%s}, and nothing sets %q for it — an operator's "+ "value is the assignment's, never the definition's (novox/hq ADR 0112), and only a "+ "preference has a default in the definition (ADR 0262): "+ "`settings set %s ` with {%q: …}%s", - module, key, key, module, key, orNoSettings(layers)) + module, key, key, module, key, orNoSettings(layers))} } content = strings.ReplaceAll(content, "${setting:"+key+"}", plainly(value)) } @@ -114,11 +125,11 @@ func settingIntoUnit(resource map[string]any, layers []Layer, module string) err for _, key := range settingsUsed(unit) { value, set := settingValue(layers, key) if !set { - return fmt.Errorf( + return &UnsetSettingError{Module: module, Setting: key, said: fmt.Sprintf( "%s has a service whose unit says ${setting:%s}, and nothing sets %q for it — an operator's "+ "value is the assignment's, never the definition's (novox/hq ADR 0112): "+ "`settings set %s ` with {%q: …}%s", - module, key, key, module, key, orNoSettings(layers)) + module, key, key, module, key, orNoSettings(layers))} } v := plainly(value) if !unitPart.MatchString(v) {