diff --git a/cmd/mesh-controller/doctor.go b/cmd/mesh-controller/doctor.go index 084a65e0..ff21a2b6 100644 --- a/cmd/mesh-controller/doctor.go +++ b/cmd/mesh-controller/doctor.go @@ -74,8 +74,12 @@ type probe struct { // probeRegistry is the registry, in to-be 45's order. **The registry is the design's live form**: a // probe added to a design is a row added here. var probeRegistry = []probe{ - {ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation", - From: "issues 236, 263, 275", Kind: "declaration-refused", Raises: []string{kindAwaitingPush}, Phase: 1, + // D1 also says every module assigned and left out of a machine's declaration (novox/hq issue 380), from the + // resolution it already makes: needs-operator for a setting nobody gave, left-out for any other cause. + {ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation; no module " + + "assigned to a machine is left out of its declaration unsaid", + From: "issues 236, 263, 275, 380", Kind: "declaration-refused", + Raises: []string{kindAwaitingPush, kindLeftOut, kindNeedsOperator}, Phase: 1, run: probeDeclarations}, {ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6", From: "issue 262", Kind: "resolver-wrong", Phase: 1, run: probeResolvers}, @@ -136,11 +140,6 @@ var probeRegistry = []probe{ {ID: "D-root", Asserts: "no agent can become root without a person on a machine where the router or a channel " + "proving its sender runs: not by its own account, and not through a tool that runs its command as an account " + "that can", From: "ADR 0259 §8", Kind: kindRootNotFree, Phase: 2, run: probeAgentRoot}, - // A module assigned and left out of its machine's composition (novox/hq issue 380): said on that machine, the - // operator's, never urgent; cleared once it composes again or is unassigned. - {ID: probeLeftOutID, Asserts: "no module assigned to a machine is left out of its composition unsaid: each " + - "raises needs-operator for a setting nobody gave, left-out for any other cause", From: "issue 380", - Kind: kindLeftOut, Raises: []string{kindNeedsOperator}, Phase: 1, run: probeLeftOut}, {ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals", From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs}, // The core's health definitions (novox/hq to-be 45 §8, ADR 0236): what a core component's new build is diff --git a/cmd/mesh-controller/left_out.go b/cmd/mesh-controller/left_out.go index e530140c..c66a21d3 100644 --- a/cmd/mesh-controller/left_out.go +++ b/cmd/mesh-controller/left_out.go @@ -1,7 +1,6 @@ package main import ( - "context" "errors" "fmt" "sort" @@ -11,12 +10,12 @@ import ( "github.com/novox/mesh-controller/internal/conditions" ) -// A module assigned to a machine and left out of its composition is said (novox/hq issue 380). +// A module assigned to a machine and left out of its declaration is said (novox/hq issue 380). // // **An omission is a finding, never a refusal of the push** (ADR 0163, rule 6): the machine is sent everything // else, and the module's held things are kept. Until issue 380 the only place it showed was `plan`: nfs-server was -// assigned to the home server for days, every push left it out for a setting nobody gave, and the operator believed -// it ran. So the self-check composes every machine as the next push would (Resolution.LeftOutBecause, the push's +// assigned to the home server for days, every send left it out for a setting nobody gave, and the operator believed +// it ran. So the self-check (D1) judges every machine as the next send would (Resolution.LeftOutBecause, the send's // own judgement) and raises a condition on that machine for each module it leaves out: // // - a setting nobody gave (catalogue.UnsetSettingError) is the operator's to give: kind needs-operator, naming @@ -31,19 +30,20 @@ import ( // catalogue's own unknown-field condition says it once for the whole mesh (ADR 0262); it is still listed. const ( - // probeLeftOutID is the self-check's probe that raises and clears these conditions. - probeLeftOutID = "D-left-out" + // probeLeftOutID is the self-check's probe that raises and clears these conditions: D1, which already + // resolves every machine (probeDeclarations), so the judgement costs no resolution of its own. + probeLeftOutID = "D1" // kindLeftOut is a module left out for any cause but a setting nobody gave; it is also every such condition's // token, whichever its kind, so a cause that changes is the same condition said anew. kindLeftOut = "left-out" ) -// leftOutModule is one module of a machine's set that its composition leaves out, and why. +// leftOutModule is one module of a machine's set that its declaration leaves out, and why. type leftOutModule struct { Module string // Setting is the setting nobody gave, when that is the cause. Setting string - // Why is the composition's own reason, whole. + // Why is the declaration's own reason, whole. Why string // Unread is a stored manifest this controller cannot read whole (said by the catalogue's condition). Unread bool @@ -76,13 +76,13 @@ func settingCommand(module, setting, node string) string { } // reason is why a module is left out, as `status`, `node show` and the condition's summary say it: for a setting -// nobody gave, the setting and the command that gives it; otherwise the composition's own words. +// nobody gave, the setting and the command that gives it; otherwise the declaration's own words. func (l leftOutModule) reason(node string) string { if l.Setting != "" { - return fmt.Sprintf("nothing sets its setting %q, so every push leaves it out — %s sets it", l.Setting, + return fmt.Sprintf("nothing sets its setting %q, so every send leaves it out of its declaration — %s sets it", l.Setting, settingCommand(l.Module, l.Setting, node)) } - return "every push leaves it out: " + l.Why + return "every send leaves it out of its declaration: " + l.Why } // leftOutObservations are the conditions a machine's left-out modules raise, one each. @@ -97,7 +97,7 @@ func leftOutObservations(node string, left []leftOutModule) []conditions.Observa return out } -// leftOutObservation is one module left out of one machine's composition: needs-operator for a setting nobody +// leftOutObservation is one module left out of one machine's declaration: needs-operator for a setting nobody // gave, left-out otherwise; a warning either way, the operator's to resolve. func leftOutObservation(node string, l leftOutModule) conditions.Observation { o := conditions.Observation{Scope: conditions.ScopeModule, ID: l.Module + "." + node, Token: kindLeftOut, @@ -108,10 +108,10 @@ func leftOutObservation(node string, l leftOutModule) conditions.Observation { if l.Setting != "" { o.Kind = kindNeedsOperator } else { - // The composition's words may name a path or an address, which the operator's channel withholds: the + // The words of why may name a path or an address, which the operator's channel withholds: the // summary sends them to the evidence, and `plan` says them in full. - o.Summary = fmt.Sprintf("%s is assigned to %s and not applied: every push leaves it out, because what is "+ - "set for it does not compose with its definition — the evidence and `plan %s` say why", l.Module, node, node) + o.Summary = fmt.Sprintf("%s is assigned to %s and not applied: every send leaves it out of its declaration, "+ + "because what is set for it does not fit its manifest — the evidence and `plan %s` say why", l.Module, node, node) } o.Headline, o.Explanation, o.Needs, o.Resolved = w.Headline, w.Explanation, w.Needs, w.Resolved return o @@ -121,20 +121,20 @@ func leftOutObservation(node string, l leftOutModule) conditions.Observation { func leftOutWords(module, node, setting string) words { w := words{ Headline: fmt.Sprintf("%s is not applied on %s", module, node), - Explanation: fmt.Sprintf("%s is assigned to %s, and every update of %s leaves it out because what is set "+ - "for it does not fit its definition. Nothing of it changes there; the rest of %s is updated as usual.", + Explanation: fmt.Sprintf("%s is assigned to %s, and every send to %s leaves it out of the declaration because what "+ + "is set for it does not fit its manifest. Nothing of it changes there; the rest of %s is sent as usual.", module, node, node, node), Needs: fmt.Sprintf("read why in the details, then change what is set for %s or unassign it.", module), Resolved: fmt.Sprintf("%s on %s is no longer left out", module, node), } if setting != "" { - w.Explanation = fmt.Sprintf("%s is assigned to %s, and every update of %s leaves it out because nothing "+ - "sets its setting %s. Nothing of it runs there until it is set; the rest of %s is updated as usual.", + w.Explanation = fmt.Sprintf("%s is assigned to %s, and every send to %s leaves it out of the declaration because "+ + "nothing sets its setting %s. Nothing of it runs there until it is set; the rest of %s is sent as usual.", module, node, node, setting, node) w.Needs = fmt.Sprintf("set %s for %s on %s, or approve it when it is proposed to you.", setting, module, node) // A setting's name that is not plain (a dotted key) is in the summary instead. if _, ok := conditions.PlainWords(w, node); !ok { - w.Explanation = fmt.Sprintf("%s is assigned to %s, and every update of %s leaves it out because a "+ + w.Explanation = fmt.Sprintf("%s is assigned to %s, and every send to %s leaves it out of the declaration because a "+ "setting it needs is not set. Nothing of it runs there until it is set; the details name it.", module, node, node) w.Needs = fmt.Sprintf("set what %s needs on %s; the details name the setting.", module, node) @@ -143,31 +143,6 @@ func leftOutWords(module, node, setting string) words { return w } -// probeLeftOut is the self-check's probe of issue 380: every machine's set is judged as its next push would -// judge it, and each module left out raises its condition. A machine that does not resolve is passed over — D1 -// says it — and a store that cannot be read is an error, never "nothing left out" (ADR 0227 rule 4). -func probeLeftOut(ctx context.Context, d *doctor) ([]conditions.Observation, error) { - nodes, err := d.open.inventory.Nodes(ctx) - if err != nil { - return nil, err - } - var out []conditions.Observation - for _, n := range nodes { - plan, settings, err := planFor(ctx, d.open, n.Name) - if err != nil { - if unresolvable(err) { - continue - } - return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err) - } - out = append(out, leftOutObservations(n.Name, leftOutOf(plan, settings, n.Adopted))...) - if ctx.Err() != nil { - return nil, ctx.Err() - } - } - return out, nil -} - // notAppliedLines is a machine's "assigned, not applied" as `node show` prints it: one line a module, with the // reason its condition says. func notAppliedLines(node string, left []leftOutModule) []string { @@ -181,12 +156,15 @@ func notAppliedLines(node string, left []leftOutModule) []string { return lines } -// machineNotApplied is one module assigned to a machine and left out of its composition, in `status --json`. +// machineNotApplied is one module assigned to a machine and left out of its declaration, in `status --json`. type machineNotApplied struct { Node string `json:"node"` Module string `json:"module"` Setting string `json:"setting,omitempty"` Reason string `json:"reason"` + // UnreadManifest is a module left out because this controller cannot read its manifest whole: it raises no + // condition of its own on the machine, since the catalogue's unknown-field condition says it once (ADR 0262). + UnreadManifest bool `json:"unread-manifest,omitempty"` } // notApplied is every machine's left-out modules, in a stated order, as `status --json` carries them. @@ -199,7 +177,8 @@ func notApplied(left map[string][]leftOutModule) []machineNotApplied { var out []machineNotApplied for _, name := range names { for _, l := range left[name] { - out = append(out, machineNotApplied{Node: name, Module: l.Module, Setting: l.Setting, Reason: l.reason(name)}) + out = append(out, machineNotApplied{Node: name, Module: l.Module, Setting: l.Setting, Reason: l.reason(name), + UnreadManifest: l.Unread}) } } return out @@ -211,16 +190,33 @@ func printNotApplied(left map[string][]leftOutModule) { if len(rows) == 0 { return } - fmt.Printf("%d module(s) assigned, not applied — every push leaves them out, and each raises a condition:\n", - len(rows)) + unread := 0 + for _, r := range rows { + if r.UnreadManifest { + unread++ + } + } + // Which raise a condition is said, not implied (review of #223): one whose manifest this controller cannot + // read is listed here and raises none of its own on the machine — the catalogue's condition says it. + raises := "each raises a condition on its machine" + switch { + case unread == len(rows): + raises = "none raises a condition on its machine: this controller cannot read their manifests, which the " + + "catalogue's own condition says" + case unread > 0: + raises += fmt.Sprintf(", except the %d whose manifest this controller cannot read, which the catalogue's own "+ + "condition says", unread) + } + fmt.Printf("%d module(s) assigned, not applied — every send leaves them out of their declaration; %s:\n", + len(rows), raises) for _, r := range rows { fmt.Printf(" %-12s %-22s %s\n", r.Node, r.Module, r.Reason) } fmt.Println() } -// isLeftOutCondition is whether a condition is this probe's, which the machine's health statements neither -// raise nor clear. +// isLeftOutCondition is whether a condition is a module left out of its declaration, which the machine's health +// statements neither raise nor clear: by its token, which every one carries whatever its kind. func isLeftOutCondition(c conditions.Condition) bool { - return c.Source == probeLeftOutID || strings.HasSuffix(c.Key, "."+kindLeftOut) + return c.Subject.Scope == conditions.ScopeModule && strings.HasSuffix(c.Key, "."+kindLeftOut) } diff --git a/cmd/mesh-controller/left_out_test.go b/cmd/mesh-controller/left_out_test.go index 100e6a19..695eb588 100644 --- a/cmd/mesh-controller/left_out_test.go +++ b/cmd/mesh-controller/left_out_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "os" + "slices" "strings" "testing" "time" @@ -11,9 +12,10 @@ import ( "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" ) -// novox/hq issue 380: nfs-server was assigned to the home server for days and every push left it out — "nfs-server +// novox/hq issue 380: nfs-server was assigned to the home server for days and every send left it out — "nfs-server // has a file that says ${setting:shares}, and nothing sets shares for it" — and nothing but `plan` said so. The // manifest is the catalogue's own at the commit that added it (mesh-catalog 48fba44), which still says // ${setting:shares}; the reason below is the one the live push printed. @@ -60,16 +62,16 @@ func TestAModuleLeftOutForASettingNobodyGaveNeedsTheOperatorNamingTheSettingAndT } } if o.Said != left[0].Why { - t.Errorf("the evidence is not the composition's reason: %s", o.Said) + t.Errorf("the evidence is not the reason the send gives: %s", o.Said) } plainExample(t, o, "nfs-server is not applied on anchor", "Needs you: set shares for nfs-server on anchor, or approve it when it is proposed to you. nfs-server is assigned to "+ - "anchor, and every update of anchor leaves it out because nothing sets its setting shares. Nothing of it "+ - "runs there until it is set; the rest of anchor is updated as usual.") + "anchor, and every send to anchor leaves it out of the declaration because nothing sets its setting shares. "+ + "Nothing of it runs there until it is set; the rest of anchor is sent as usual.") } func TestAModuleLeftOutForAnotherCauseIsAWarningWithTheReasonAsEvidence(t *testing.T) { - // A setting stored that its definition can no longer compose: one with a line break (issue 339). + // A setting stored that its manifest can no longer take: one with a line break (issue 339). left := leftOutOf(leftOutNFS(t), sharesGiven("library=/srv/library\nmedia=/srv/media"), false) if len(left) != 1 || left[0].Setting != "" { t.Fatalf("left out: %+v", left) @@ -87,8 +89,8 @@ func TestAModuleLeftOutForAnotherCauseIsAWarningWithTheReasonAsEvidence(t *testi } plainExample(t, o, "nfs-server is not applied on anchor", "Needs you: read why in the details, then change what is set for nfs-server or unassign it. nfs-server is assigned to "+ - "anchor, and every update of anchor leaves it out because what is set for it does not fit its definition. "+ - "Nothing of it changes there; the rest of anchor is updated as usual.") + "anchor, and every send to anchor leaves it out of the declaration because what is set for it does not fit "+ + "its manifest. Nothing of it changes there; the rest of anchor is sent as usual.") } // The self-check raises it, keeps it a warning however long it stands, and clears it when the module composes @@ -158,7 +160,7 @@ func TestALeftOutModulesConditionClearsWhenItComposesAgainOrIsUnassigned(t *test func TestTheLeftOutProbeIsInTheRegistry(t *testing.T) { for _, p := range probeRegistry { if p.ID == probeLeftOutID { - if p.run == nil || p.Kind != kindLeftOut { + if p.run == nil || !slices.Contains(p.Raises, kindLeftOut) || !slices.Contains(p.Raises, kindNeedsOperator) { t.Fatalf("%+v", p) } return @@ -201,3 +203,49 @@ func TestStatusAndNodeListAModuleAssignedAndNotApplied(t *testing.T) { t.Fatalf("node show says:\n%s", lines) } } + +// The skip is this probe's alone (review of #223): a part waiting for the operator (ADR 0283) keeps its own +// needs-operator condition, `module...needs-operator`, which still clears on the first statement that no +// longer names it — beside a left-out condition on the same machine, which stays. +func TestAWaitsNeedsOperatorStillClearsWhenItsStatementStopsNamingItBesideALeftOutOne(t *testing.T) { + store := conditions.NewInMemory() + k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store}) + defer k.Close(context.Background()) + left := leftOutObservations("anchor", leftOutOf(leftOutNFS(t), nil, false)) + if err := k.Reconcile(t.Context(), probeLeftOutID, left); err != nil { + t.Fatal(err) + } + waiting := map[string][]inventory.ResourceHealth{"notes": {{Module: "notes", Resource: "server", State: link.StateWaiting, + Waits: []inventory.Wait{{Setting: "domain", What: "the domain it serves"}}}}} + if err := judgeModuleHealth(t.Context(), nil, k, "anchor", waiting, map[string]int{"notes": 2}, time.Now()); err != nil { + t.Fatal(err) + } + waitKey, leftKey := needsOperatorKey("notes", "anchor"), "module.nfs-server.anchor.left-out" + open := func() map[string]conditions.Condition { + t.Helper() + list, err := k.Open(t.Context()) + if err != nil { + t.Fatal(err) + } + out := map[string]conditions.Condition{} + for _, c := range list { + out[c.Key] = c + } + return out + } + if c, raised := open()[waitKey]; !raised || c.Kind != kindNeedsOperator { + t.Fatalf("the wait raised %+v", open()) + } + if c := open()[leftKey]; c.Kind != kindNeedsOperator { + t.Fatalf("the left-out condition is not open as needs-operator: %+v", open()) + } + if err := judgeModuleHealth(t.Context(), nil, k, "anchor", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil { + t.Fatal(err) + } + if _, still := open()[waitKey]; still { + t.Fatalf("the statement no longer names the wait, and its needs-operator is still open: %+v", open()) + } + if _, still := open()[leftKey]; !still { + t.Fatalf("the left-out condition was cleared by a health statement: %+v", open()) + } +} diff --git a/cmd/mesh-controller/probes.go b/cmd/mesh-controller/probes.go index 22ef45a3..481571f7 100644 --- a/cmd/mesh-controller/probes.go +++ b/cmd/mesh-controller/probes.go @@ -78,6 +78,11 @@ func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err) } var problems, foreseen []string + // Each module the next send leaves out of this machine's declaration (novox/hq issue 380), judged from this + // resolution as the send judges it: a finding, never a refusal. + if err == nil { + out = append(out, leftOutObservations(n.Name, leftOutOf(plan, settings, n.Adopted))...) + } if err == nil && gensErr == nil { var declared sendable if declared, err = declarationWith(ctx, open, n.Name, plan, settings, gens, Foreseeing); err == nil { diff --git a/mesh-controller b/mesh-controller index b8fa8cd1..7059703d 100755 Binary files a/mesh-controller and b/mesh-controller differ