diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index feb6aa62..2036a129 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -20,11 +20,13 @@ import ( "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay" "git.novox.be/novox/mesh-sdk/go/declaration" + "git.novox.be/novox/mesh-sdk/go/views" "github.com/novox/mesh-host/internal/agent" "github.com/novox/mesh-host/internal/apply" "github.com/novox/mesh-host/internal/compose/frombus" "github.com/novox/mesh-host/internal/link" "github.com/novox/mesh-host/internal/liveness" + "github.com/novox/mesh-host/internal/seatwait" "github.com/novox/mesh-host/internal/selfcompose" "github.com/novox/mesh-host/internal/store" ) @@ -598,6 +600,9 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus } // The marks of the haves run off the composition pass, each bounded (see marker). marks := newMarker(run) + findings := findingsOfTheNode{session} + heldFindings := &seatwait.HeldKeeper{Node: node, Observer: "node-engine", Writer: findings, + Left: func() ([]views.Finding, error) { return findings.left(node) }, Say: say} hooks := composerHooks{ seatList: seats, seatListFailed: err != nil, @@ -626,6 +631,11 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus // A module held alone marks that module, never the others (issue 1126); the marking never holds the pass. marks.set("refused", func(ctx context.Context) { now.NotComposedEach(ctx, why, held) }) } + // And each module held says it as the finding `module-held`, cleared when it is composed again or no longer + // wanted here. A node refused whole (why) says nothing of its modules: their findings stand as they were. + if why == "" && run.Err() == nil { + marks.set("module-held", func(context.Context) { heldFindings.Reconcile(held) }) + } }, // And what it keeps of a module as it runs, for a manifest that does not say it yet: on the module's have (ADR 0601 §4). kept: func(kept map[string][]string) { diff --git a/cmd/mesh-host/seatlist.go b/cmd/mesh-host/seatlist.go index 7ae8c9fe..dd07ddb6 100644 --- a/cmd/mesh-host/seatlist.go +++ b/cmd/mesh-host/seatlist.go @@ -107,10 +107,16 @@ func (realClock) AfterFunc(d time.Duration, f func()) seatwait.Timer { type findingsOfTheNode struct{ session *link.Session } func (w findingsOfTheNode) encode(f views.Finding) (string, []byte, error) { - key := conditions.Key(conditions.ScopeMachine, f.ID, f.Kind) + // The seat list's finding is of the machine; a held module's is of the module on a machine, `.`. + scope, machine, source := conditions.ScopeMachine, f.ID, "seat-list-unread" + if f.Kind == seatwait.HeldKind { + scope, source = conditions.ScopeModule, seatwait.HeldKind + machine = f.ID[strings.LastIndex(f.ID, ".")+1:] + } + key := conditions.Key(scope, f.ID, f.Kind) v := condseat.FindingsByKeyValue{V: 1, Key: key, Kind: f.Kind, Observer: f.Observer, Severity: conditions.Warning, - Subject: condseat.Subject{Scope: conditions.ScopeMachine, ID: f.ID, Machine: f.ID}, Summary: f.Summary, - Headline: f.Headline, Explanation: f.Explanation, Source: "seat-list-unread", At: f.At.UTC()} + Subject: condseat.Subject{Scope: scope, ID: f.ID, Machine: machine}, Summary: f.Summary, + Headline: f.Headline, Explanation: f.Explanation, Source: source, At: f.At.UTC()} if err := conditions.CheckFinding(v); err != nil { return "", nil, err } @@ -118,6 +124,26 @@ func (w findingsOfTheNode) encode(f views.Finding) (string, []byte, error) { return key, raw, err } +// left is the findings of held modules an earlier run of this engine left on node: its keys of this kind. +func (w findingsOfTheNode) left(node string) ([]views.Finding, error) { + keys, err := w.session.FindingKeys() + if err != nil { + return nil, err + } + var out []views.Finding + for _, k := range keys { + id, ok := strings.CutPrefix(k, conditions.ScopeModule+".") + if !ok { + continue + } + if id, ok = strings.CutSuffix(id, "."+seatwait.HeldKind); !ok || !strings.HasSuffix(id, "."+node) { + continue + } + out = append(out, views.Finding{Observer: "node-engine", Kind: seatwait.HeldKind, ID: id}) + } + return out, nil +} + func (w findingsOfTheNode) Put(f views.Finding) error { key, raw, err := w.encode(f) if err != nil { diff --git a/cmd/mesh-host/seatlist_unread_test.go b/cmd/mesh-host/seatlist_unread_test.go index ef6c1208..3e76ac4c 100644 --- a/cmd/mesh-host/seatlist_unread_test.go +++ b/cmd/mesh-host/seatlist_unread_test.go @@ -3,6 +3,8 @@ package main import ( "testing" + "git.novox.be/novox/mesh-sdk/go/views" + "github.com/novox/mesh-host/internal/seatwait" "github.com/novox/mesh-host/internal/selfcompose" ) @@ -19,4 +21,10 @@ func TestAReaderThatCouldNotStartHoldsTheComposer(t *testing.T) { if got := composerEnv(nil, "n1", func(string) {}, composerHooks{}).SeatList; got != nil { t.Errorf("a composer with no reader asked for gates by %v, want none (the shadow)", got) } + // The finding of a held module is of the module on its machine, in a layout the condition keeper accepts (design 47, test 55). + key, raw, err := findingsOfTheNode{}.encode(views.Finding{Observer: "node-engine", Kind: seatwait.HeldKind, + ID: seatwait.HeldID("mesh-grants", "ace"), Summary: "mesh-grants is held on ace: x", Headline: "h", Explanation: "e"}) + if err != nil || key != "module.mesh-grants.ace.module-held" || len(raw) == 0 { + t.Errorf("a held module's finding encodes as %q, %v", key, err) + } } diff --git a/internal/link/findings.go b/internal/link/findings.go index c0356f9c..a4a46390 100644 --- a/internal/link/findings.go +++ b/internal/link/findings.go @@ -57,6 +57,19 @@ func (q *Session) HasFinding(key string) (bool, error) { return true, nil } +// FindingKeys lists the keys of this engine's findings over the link open now; none when the bucket holds none. +func (q *Session) FindingKeys() ([]string, error) { + kv, err := q.findings() + if err != nil { + return nil, err + } + keys, err := kv.Keys() + if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { + return nil, err + } + return keys, nil +} + // DeleteFinding clears one finding of this engine over the link open now; a finding already gone is cleared. func (q *Session) DeleteFinding(key string) error { kv, err := q.findings() diff --git a/internal/seatwait/held.go b/internal/seatwait/held.go new file mode 100644 index 00000000..b2682d50 --- /dev/null +++ b/internal/seatwait/held.go @@ -0,0 +1,117 @@ +// HeldKeeper keeps the node-engine's finding `module-held`: a module the node leaves out of its pass because its +// claim cannot be held (its seat's definition missing, removed or refused, a claim another holder keeps, a credential the +// vault refuses) is no silent leave-out. The finding names the node, the module and why, and the condition keeper keeps the +// condition (novox/hq ADR 0564 §2) until the module is composed again or is no longer wanted on the node, when the finding +// is cleared (design 47, *A claim problem holds its module, never the node's pass*; test 55). +package seatwait + +import ( + "fmt" + "sort" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/views" +) + +// HeldKind is the finding of a held module; its id is `.`. +const HeldKind = "module-held" + +// HeldKeeper writes one finding for each module the node holds, and clears it when the module is no longer held. +type HeldKeeper struct { + Node string + Observer string + Writer views.FindingWriter + // Left lists the findings of this kind an earlier run of the engine left, once, at the first Reconcile: a module held + // then and composed while the engine was away must not keep its condition. Nil leaves none. + Left func() ([]views.Finding, error) + // Now is the clock; nil is time.Now. + Now func() time.Time + // Say tells the node's journal; nil says nothing. + Say func(string) + + mu sync.Mutex + swept bool + written map[string]views.Finding // by module +} + +// HeldID is the finding's id for a module on a node. +func HeldID(module, node string) string { return module + "." + node } + +// Reconcile makes the findings those of held (each module held and why): one is written for a module not yet written or +// held for other words, and one is cleared for a module no longer held, composed or no longer wanted here. A write or a +// clear that fails is said and tried again at the next Reconcile. +func (k *HeldKeeper) Reconcile(held map[string]string) { + k.mu.Lock() + defer k.mu.Unlock() + if k.written == nil { + k.written = map[string]views.Finding{} + } + if !k.swept && k.Left != nil { + left, err := k.Left() + if err != nil { + k.say(fmt.Sprintf("the findings an earlier run left of held modules cannot be read: %v", err)) + } else { + k.swept = true + for _, f := range left { + module := strings.TrimSuffix(f.ID, "."+k.Node) + if _, ok := k.written[module]; !ok { + k.written[module] = f + } + } + } + } else if k.Left == nil { + k.swept = true + } + names := make([]string, 0, len(k.written)) + for m := range k.written { + names = append(names, m) + } + sort.Strings(names) + for _, m := range names { + if _, still := held[m]; still { + continue + } + if err := k.Writer.Delete(k.written[m]); err != nil { + k.say(fmt.Sprintf("the finding that %s is held on %s cannot be cleared: %v", m, k.Node, err)) + continue + } + delete(k.written, m) + } + names = names[:0] + for m := range held { + names = append(names, m) + } + sort.Strings(names) + for _, m := range names { + why := held[m] + if was, ok := k.written[m]; ok && was.Summary != "" && strings.HasSuffix(was.Summary, ": "+why) { + continue + } + f := views.Finding{Observer: k.Observer, Kind: HeldKind, ID: HeldID(m, k.Node), + Summary: fmt.Sprintf("%s is held on %s: %s", m, k.Node, why), + Headline: fmt.Sprintf("%s is held on %s", m, k.Node), + Explanation: "The node leaves this module out of its pass, keeps what runs of it and composes and applies every other module. The finding clears when the module is composed again or is no longer wanted on the node.", + At: k.now()} + if err := k.Writer.Put(f); err != nil { + k.say(fmt.Sprintf("the finding that %s is held on %s cannot be written: %v", m, k.Node, err)) + continue + } + k.written[m] = f + k.say(f.Summary) + } +} + +func (k *HeldKeeper) now() time.Time { + if k.Now != nil { + return k.Now() + } + return time.Now() +} + +func (k *HeldKeeper) say(s string) { + if k.Say != nil { + k.Say(s) + } +} diff --git a/internal/seatwait/seatwait_test.go b/internal/seatwait/seatwait_test.go index f34aa5cf..3e945bd9 100644 --- a/internal/seatwait/seatwait_test.go +++ b/internal/seatwait/seatwait_test.go @@ -167,13 +167,40 @@ func caseAFindingOfAnEarlierProcessIsClearedAtTheFirstFullRead(t *testing.T) { } } +// Design 47, test 55: a module held for its claim is a finding that names the node, the module and why, written once, +// and cleared when the module is composed again or no longer wanted, also one an earlier run of the engine left. +func caseAHeldModuleIsAFindingClearedWhenItIsComposedOrNoLongerWanted(t *testing.T) { + findings := &fakeFindings{} + k := &HeldKeeper{Node: "ace", Observer: "node-engine", Writer: findings, + Left: func() ([]views.Finding, error) { + return []views.Finding{{Kind: HeldKind, ID: HeldID("gone", "ace")}}, nil + }} + k.Reconcile(map[string]string{"mesh-grants": "its seat is refused", "other": "no claim"}) + if len(findings.put) != 2 || findings.put[0].ID != "mesh-grants.ace" || len(findings.deleted) != 1 || findings.deleted[0].ID != "gone.ace" { + t.Fatalf("wrote %v, cleared %v", findings.put, findings.deleted) + } + k.Reconcile(map[string]string{"mesh-grants": "its seat is refused", "other": "no claim"}) + if len(findings.put) != 2 { + t.Fatalf("a module held for the same words was written again: %v", findings.put) + } + k.Reconcile(map[string]string{"mesh-grants": "its seat is refused"}) + if len(findings.deleted) != 2 || findings.deleted[1].ID != "other.ace" { + t.Fatalf("a module composed again kept its finding: %v", findings.deleted) + } + k.Reconcile(nil) + if len(findings.deleted) != 3 { + t.Fatalf("a module no longer wanted kept its finding: %v", findings.deleted) + } +} + // ADR 0636 §1, the node's finding of a seat list unread. func TestTheSeatListWait(t *testing.T) { for name, run := range map[string]func(*testing.T){ - "five minutes write one finding and the first full read clears it": caseTheNodeWritesOneFindingAfterFiveMinutesAndClearsItAtTheFirstFullRead, - "a read within the bound writes nothing": caseAReadWithinTheBoundWritesNothing, - "a finding that cannot be written is said and tried again": caseAFindingThatCannotBeWrittenIsSaidAndTriedAgain, - "a finding of an earlier process is cleared at the first full read": caseAFindingOfAnEarlierProcessIsClearedAtTheFirstFullRead, + "five minutes write one finding and the first full read clears it": caseTheNodeWritesOneFindingAfterFiveMinutesAndClearsItAtTheFirstFullRead, + "a read within the bound writes nothing": caseAReadWithinTheBoundWritesNothing, + "a finding that cannot be written is said and tried again": caseAFindingThatCannotBeWrittenIsSaidAndTriedAgain, + "a finding of an earlier process is cleared at the first full read": caseAFindingOfAnEarlierProcessIsClearedAtTheFirstFullRead, + "a held module is a finding cleared when it is composed or no longer wanted": caseAHeldModuleIsAFindingClearedWhenItIsComposedOrNoLongerWanted, } { t.Run(name, run) }