diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 3ed9dab2..72177123 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -425,9 +425,9 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, hub := process.NewEventHub(logger.Printf) // The seat list (seatlist.go), read over the connection and heard on the hub, given to catalogue at every change, and a // pass made then: registered before the hub starts. - gate := &seatGate{} + gate := seatlist.NewPart("assigner", seatListAgainCallers) if bus != nil { - gate.follow(seatlist.ProcessBus(bus.Call, hub), wake, logger.Printf) + gate.Follow(seatlist.ProcessBus(bus.Call, hub), wake, logger.Printf) } // The indexer's manifests, modules and module graph, through its first views and views family, never its buckets // (DLV-F2r, ADR 0631): the first pass waits for the three answers, as it waits for every watch to hand what stands. @@ -502,7 +502,8 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, logger.Printf("a pipeline's end or a registration is not heard, so the wants keep the trunk-only form until the assigner starts again: %v", err) } } - hearReadsChanged(ctx, js, hub, reads, findings, gate.eventsLive, logger.Printf) + hearReadsChanged(ctx, js, hub, reads, findings, gate.EventsLive, logger.Printf) + hearRunning(ctx, js, w, wake, findings, logger.Printf) } if bus != nil && kvs[buckets.Puts] != nil { p.acts = newPutActs(bus, wake, logger.Printf) @@ -602,7 +603,7 @@ func hearReadsChanged(ctx context.Context, js jetstream.JetStream, hub *process. } w.Heard(process.EventsStream) if live != nil { - live() // the seat list, unread while the seat was silent, is read now (seatGate.eventsLive) + live() // the seat list, unread while the seat was silent, is read now (seatlist.Part.EventsLive) } go func() { <-ctx.Done() diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index 8c5d6aa2..ef0ccc4f 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -18,6 +18,8 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate" "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay" "git.novox.be/novox/mesh-sdk/go/motion" + "git.novox.be/novox/mesh-sdk/go/seatlist" + "sync" oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" "git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient" @@ -79,7 +81,8 @@ type passer struct { seat *seatActs // seats is the seat list the process reads (seatlist.go): until it is read, seat-holders is not written and a proposal // laid again is not ended on a seat it does not name (ADR 0636 §1). Nil reads nothing. - seats *seatGate + seats *seatlist.Part + seatWaitSaid sync.Once publish publisher saidProposed map[string]bool // copied says the retired state was copied into `assignments` by this process, or found empty (copyRetired). @@ -1248,11 +1251,11 @@ func (p *passer) seatHolders(ctx context.Context) { if len(assignments) == 0 { return // no assignments read yet: nothing is known of who holds what, and nothing is deleted } - if !p.seats.read() { + if !p.seats.Heard() { // ADR 0636 §1: which claim names which seat is the list's, so nothing is written from the bootstrap alone; the // reader's change makes a pass. if p.seats != nil { - p.seats.waitSaid.Do(func() { p.logf("seat-holders is not written until the seat list is read: %s", p.seats.health()) }) + p.seatWaitSaid.Do(func() { p.logf("seat-holders is not written until the seat list is read: %s", p.seats.Health()) }) } return } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index 5e8e1ce4..716ac50c 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -651,8 +651,21 @@ func putOfTheBus(t *testing.T) { t.Errorf("the warrant was recorded as %q", call) } world(buckets.Wants, "want.novox.nats", wantsKV.values["want.novox.nats"]) - running, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Composed: digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) - world(buckets.Have, hkey, running) + // Design 47 G §309: the put ends on the node-engine seat's running naming its fingerprint, never on the have. + haveRunning, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Composed: digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) + world(buckets.Have, hkey, haveRunning) + if got := step(); got.Step != puts.Wanted { + t.Fatalf("the have runs it and no running says so, the put stands at %s: %s", got.Step, got.Words) + } + if err := p.w.hearRunning("mesh.seat.node-engine.event.running.novox.nats", []byte(`{"v":1,"at":"2026-11-10T20:00:00Z","node":"novox","module":"nats"}`)); err != nil { + t.Fatal(err) + } + if got := step(); got.Step != puts.Wanted { + t.Fatalf("a running without a fingerprint ends none, the put stands at %s: %s", got.Step, got.Words) + } + if err := p.w.hearRunning("mesh.seat.node-engine.event.running.novox.nats", []byte(`{"v":1,"at":"2026-11-10T20:01:00Z","node":"novox","module":"nats","fingerprint":"`+fp+`"}`)); err != nil { + t.Fatal(err) + } if got := step(); got.Step != puts.Holding { t.Fatalf("running healthy, the put stands at %s: %s", got.Step, got.Words) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go index 77ccfaf2..0322e53c 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go @@ -221,6 +221,9 @@ func (p *passer) putWorld(node, module string, put puts.Put, manifest catalogue. } } } + if r, ok := p.w.runningOf(node, module); ok { + w.Running = &r + } if hkey, err := nodeengineclient.HaveKey(node, module); err == nil { if raw := p.w.value(buckets.Have, hkey); raw != nil { if h, err := nodeengineclient.DecodeHave(raw); err == nil { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/running.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/running.go new file mode 100644 index 00000000..1aa6d079 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/running.go @@ -0,0 +1,88 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-assigner/internal/assign" + "mesh-delivery-assigner/internal/assignment" +) + +// What each node runs is heard, never asked (design 47 G §199, §309): the node-engine seat's event `running..` is +// kept last per subject on its own stream, and the assigner's durable consumer on it starts from the last message of every +// subject, so a node away still has its last word here. The assigner counts a provider healthy by it (rule 5) and ends a put on it. + +// hearRunning keeps each `running` the stream holds and each it says from now, and wakes the pass on every one. +func hearRunning(ctx context.Context, js jetstream.JetStream, w *world, wake func(), findings assign.Findings, logf func(string, ...any)) { + acct := process.Account{Node: thisNode, Module: assignment.Module} + nh := notHeardWriter(findings, logf) + cc, err := acct.HearSeatEvents(ctx, js, nodeengine.StreamRunning, func(m jetstream.Msg) { + if err := m.Ack(); err != nil { + logf("acknowledging %s: %v", m.Subject(), err) + } + if err := w.hearRunning(m.Subject(), m.Data()); err != nil { + logf("%v", err) + return + } + wake() + }, logf) + if err != nil { + nh.NotHeard(nodeengine.StreamRunning, "what each node runs is not heard until the assigner starts again, so a provider is counted healthy by its have and a put does not end", err) + return + } + w.mu.Lock() + w.runningHeard = true + w.mu.Unlock() + nh.Heard(nodeengine.StreamRunning) + go func() { + <-ctx.Done() + cc.Stop() + }() +} + +// hearRunning keeps one `running` event said on subject. +func (w *world) hearRunning(subject string, data []byte) error { + var r nodeengine.Running + if err := json.Unmarshal(data, &r); err != nil { + return fmt.Errorf("%s holds what is not a running: %w", subject, err) + } + if r.Node == "" || r.Module == "" { + return fmt.Errorf("%s names no node or no module, so it is kept for none", subject) + } + w.mu.Lock() + w.running[r.Node+"."+r.Module] = r + w.mu.Unlock() + return nil +} + +// runningOf is the last `running` heard of module on node. +func (w *world) runningOf(node, module string) (nodeengine.Running, bool) { + w.mu.Lock() + defer w.mu.Unlock() + r, ok := w.running[node+"."+module] + return r, ok +} + +// runningNow is every `running` heard, by node and module; nil while the seat's events are not heard (the rules then read the +// have). +func (w *world) runningNow() map[string]map[string]nodeengine.Running { + w.mu.Lock() + defer w.mu.Unlock() + if !w.runningHeard { + return nil + } + out := map[string]map[string]nodeengine.Running{} + for _, r := range w.running { + if out[r.Node] == nil { + out[r.Node] = map[string]nodeengine.Running{} + } + out[r.Node][r.Module] = r + } + return out +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go index dfb5d04d..8efd714f 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go @@ -440,10 +440,10 @@ func (p *passer) mustAnswer(ctx context.Context, prop mma.Proposal, rev uint64, func (p *passer) layAgain(ctx context.Context, prop mma.Proposal, rev uint64, current map[string]mma.Assignments, revs map[string]uint64, why string) { held, recorded := p.lays(ctx) again, laid, findings := assign.Propose(assign.ProposeIn{ID: prop.ID, By: prop.By, Why: prop.Why, At: prop.At, Changes: prop.Changes, - Current: current, Revisions: revs, Held: held, Recorded: recorded, SeatListRead: p.seats.read()}) + Current: current, Revisions: revs, Held: held, Recorded: recorded, SeatListRead: p.seats.Heard()}) if slices.ContainsFunc(findings, plc.WaitsForSeatList) { // ADR 0636 §1: nothing is ended on a list not read; the reader's change makes a pass, which lays it again. - p.logf("the proposal %s is not laid again yet: %s", prop.ID, p.seats.health()) + p.logf("the proposal %s is not laid again yet: %s", prop.ID, p.seats.Health()) return } if len(findings) > 0 { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go index 6d63a9f9..e459a792 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go @@ -19,6 +19,7 @@ import ( mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient" mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/seatlist" "git.novox.be/novox/mesh-sdk/go/views" "mesh-delivery-assigner/internal/assign" @@ -86,7 +87,7 @@ type seatTools struct { now func() time.Time // seats is the seat list the bundle reads (seatlist.go): until it is read, propose judges no mesh seat the bootstrapped // definitions do not name as held once (ADR 0636 §1). Nil reads nothing. - seatList *seatGate + seatList *seatlist.Part } func newSeatTools(st seatStore, ask func(string, any) (json.RawMessage, error)) *seatTools { @@ -265,7 +266,7 @@ func (s *seatTools) doPropose(args map[string]any, by string) (any, error) { enrollers := plc.EnrollersOf(current) now := s.now() p, laid, findings := assign.Propose(assign.ProposeIn{ID: plc.NewProposalID(now), By: by, Why: arg(args, "why"), At: now, - Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded, Warrant: warrant, SeatListRead: s.seatList.read()}) + Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded, Warrant: warrant, SeatListRead: s.seatList.Heard()}) if refusal := plc.AddNodeRefusal(p, moduleOfCaller(by, enrollers), enrollers); refusal != "" { findings = append(findings, refusal) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go index 704f820c..c884c495 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go @@ -257,7 +257,7 @@ func TestTheSeatsVerbs(t *testing.T) { // Such a session is proved as its machine's tool runner (issue 627); the healers too, no other module. for caller, refused := range map[string]bool{"person.jochen": false, "shanks.node-tools": false, "ace.mesh-healers": false, "ace.messenger": true} { - if err := seatlist.Refusal(seatListAgainTool, seatListAgainCallers, caller); (err != nil) != refused { + if err := seatlist.Refusal("assigner_seat_list_again", seatListAgainCallers, caller); (err != nil) != refused { t.Errorf("%s: refused=%v, err %v", caller, refused, err) } } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seatlist.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seatlist.go index ec80d6e0..61e8d076 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seatlist.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seatlist.go @@ -1,103 +1,13 @@ package main -import ( - "errors" - "sync" - "sync/atomic" +import stdio "git.novox.be/novox/mesh-sdk/go" - stdio "git.novox.be/novox/mesh-sdk/go" - "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/seatlist" -) - -// The assigner reads the seat list as every part does (novox/hq ADR 0635 §1, ADR 0636 §1, §7; issue 1966): through the seat -// mesh-catalog's verb seat-definitions and its two events, from the bootstrapped definitions until the seat has answered. What -// the reader holds is given to catalogue at every change, which the check of a proposal and the seat-holders state read: which -// mesh seat is replicated, and which name is a seat's. Both of its runtimes hold one: the process (the proposals it lays again, -// the state seat-holders) over its connection and the event hub, and the tools' bundle (the seat's verb propose) over the -// runtime. Until the list is read, a mesh seat the bootstrapped definitions do not name is never judged held once +// The assigner reads the seat list as every part does, through mesh-sdk's one wiring (go/seatlist.Part; design 47 G §204): its +// reader over the seat mesh-catalog's verb seat-definitions and its two events, its tools `assigner_seat_list` and +// `assigner_seat_list_again`. What the reader holds is given to catalogue at every change, which the check of a proposal and the +// seat-holders state read; until the list is read, a mesh seat the bootstrapped definitions do not name is never judged held once // (plc.CheckAssignments), and seat-holders is not written. -// Seat list tools (ADR 0636 §7): what the reader holds, and reading it again now, which every operator's session and the -// healers may ask. -const ( - seatListTool = "assigner_seat_list" - seatListAgainTool = "assigner_seat_list_again" -) - // A call through the mesh MCP server is proved only as its machine's tool runner (.node-tools, issue 627), so // node-tools stands for an operator's session until a session is proved (issue 1966). var seatListAgainCallers = []string{stdio.AnyPerson, "node-tools", "mesh-healers"} - -// seatGate is the reader a runtime of the assigner judges by, set once it is started; before, the list counts as unread. -type seatGate struct { - reader atomic.Pointer[seatlist.Reader] - // waitSaid says the journal was told that seat-holders waits for the list, so it is told once per wait. - waitSaid sync.Once -} - -// read says the seat mesh-catalog has answered the reader: the list is read (seatlist.Status's Heard). A nil gate, or one -// whose reader has not started, has read nothing. -func (g *seatGate) read() bool { - if g == nil { - return false - } - r := g.reader.Load() - return r != nil && r.Status().Heard -} - -// health is why the list is not read, or empty. -func (g *seatGate) health() string { - if g == nil { - return "the assigner follows no seat list" - } - r := g.reader.Load() - if r == nil { - return "the seat list reader has not started yet" - } - return r.Status().Health() -} - -// follow starts the reader over bus: what it holds is given to catalogue at once and at every change, and changed is called -// then (the process makes a pass, which lays again what waited). -func (g *seatGate) follow(bus seatlist.Bus, changed func(), logf func(string, ...any)) *seatlist.Reader { - reader, err := seatlist.Start(seatlist.Config{Bus: bus, Bootstrap: seatlist.Bootstrapped(), - OnChange: func(seats []catalogue.Seat) { - catalogue.UseSeats(seats) - if changed != nil { - changed() - } - }}) - if err != nil { - logf("the assigner cannot follow its seat list, so a mesh seat the bootstrapped definitions do not name waits for it: %v", err) - } - if reader != nil { - catalogue.UseSeats(reader.Seats()) - g.reader.Store(reader) - } - return reader -} - -// eventsLive is called once the process's consumer on EVENTS is heard: the seat may have said it speaks before that consumer -// existed, whose events start at the stream's end, so while the list is unread it is read now (ADR 0636 §1). -func (g *seatGate) eventsLive() { - if r := g.reader.Load(); r != nil { - r.ReloadIfUnread() - } -} - -// tools are the reader's two tools over whatever reader the gate holds at the call. -func (g *seatGate) tools() []stdio.Tool { - shape := seatlist.Tools(&seatlist.Reader{}, seatListTool, seatListAgainTool, seatListAgainCallers) - for i := range shape { - run := i - shape[i].Run = func(args map[string]any) (any, error) { - r := g.reader.Load() - if r == nil { - return nil, errors.New("the seat list reader has not started yet: the runtime has not served this module") - } - return seatlist.Tools(r, seatListTool, seatListAgainTool, seatListAgainCallers)[run].Run(args) - } - } - return shape -} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seatlist_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seatlist_test.go index 67f44e2f..8cf119c9 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seatlist_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seatlist_test.go @@ -3,6 +3,9 @@ package main import ( "encoding/json" "errors" + "os" + "path/filepath" + "strings" "sync" "testing" "time" @@ -50,21 +53,44 @@ func (standInCatalog) Subscribe(string, func(string, json.RawMessage) error) err // gateOver is a seat gate following the stand-in, once its first read has ended. A read one gives catalogue the test list // (catalogue.UseSeats), as a host's does, for the tests after it too: the list is the same everywhere. -func gateOver(t *testing.T, silent bool) *seatGate { +func gateOver(t *testing.T, silent bool) *seatlist.Part { t.Helper() if silent { // A silent seat reached no one: its reader asks again within the bus's retry bound (issue 1966), which a test // that waits for it to stay unread keeps short. t.Cleanup(busretry.Waits(20*time.Millisecond, time.Millisecond, 5*time.Millisecond)) } - g := &seatGate{} - r := g.follow(standInCatalog{silent: silent}, nil, t.Logf) + g := seatlist.NewPart("assigner", seatListAgainCallers) + r := g.Follow(standInCatalog{silent: silent}, nil, t.Logf) if r == nil { t.Fatal("the reader did not start") } <-r.Ready() - if g.read() == silent { - t.Fatalf("the gate read %v over a stand-in silent %v: %s", g.read(), silent, g.health()) + if g.Heard() == silent { + t.Fatalf("the gate read %v over a stand-in silent %v: %s", g.Heard(), silent, g.Health()) } return g } + +// Design 47 G §204: the seat list's reader is wired once, by mesh-sdk's go/seatlist.Part; no part wires its own. A search of the +// assigner's code finds no start of a reader and no tools of its own. +func TestTheSeatListIsWiredByTheSDKAlone(t *testing.T) { + files, err := filepath.Glob("*.go") + if err != nil { + t.Fatal(err) + } + for _, f := range files { + if strings.HasSuffix(f, "_test.go") { + continue + } + raw, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) + } + for _, own := range []string{"seatlist.Start(", "seatlist.Tools(", "OnChange:"} { + if strings.Contains(string(raw), own) { + t.Errorf("%s wires the seat list itself (%s), where mesh-sdk's seatlist.Part does it", f, own) + } + } + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go index 9b946428..1c7ec141 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go @@ -386,15 +386,15 @@ func serveTools() error { } }() // The seat list (seatlist.go), read through the runtime once Serve has its channel: the seat's verbs judge by it. - gate := &seatGate{} + gate := seatlist.NewPart("assigner", seatListAgainCallers) go func() { <-stdio.Served() - gate.follow(seatlist.StdioBus(), nil, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: "+f+"\n", a...) }) + gate.Follow(seatlist.StdioBus(), nil, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: "+f+"\n", a...) }) }() seatVerbs := newSeatTools(runtimeStore{}, stdio.Ask) seatVerbs.seatList = gate tools := append(s.tools(), seatVerbs.tools()...) - tools = append(tools, gate.tools()...) + tools = append(tools, gate.Tools()...) wants := newWantsTools(runtimeStore{}, stdio.Ask, func(e, id string, b any) error { // The seat's event, said on the seat: a claimed seat's event is never the module's own. return stdio.SeatPublish(wantsEventSubject(e), b, id) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go index a5a1ca9a..be881eb6 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go @@ -52,6 +52,10 @@ type world struct { // sourceGroups is, by revision of the gate's source.main, the release (group) that moved it, from every revision its // history keeps (watched with history): a build's source revision names the release that carries it (ADR 0601 §1). sourceGroups map[uint64]string + // running is, by `.`, the last `running` the node-engine seat said of it, kept from its stream by the assigner's + // durable consumer (design 47 G §199, §309); runningHeard says that consumer is bound. + running map[string]nodeengine.Running + runningHeard bool } // entryMeta is a value's revision in its bucket and when it was written. @@ -70,7 +74,7 @@ type parsed struct { func newWorld() *world { return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{}, meta: map[string]entryMeta{}, trunkBuilt: map[string]builds.BuildLogEntry{}, assignmentKeys: mmac.NewKeys(), - sourceGroups: map[uint64]string{}} + sourceGroups: map[uint64]string{}, running: map[string]nodeengine.Running{}} } // putEntry is put, keeping the value's revision and the time it was written. @@ -391,7 +395,7 @@ func (w *world) wantsOf() map[string]map[string]mwc.Want { // nodes is what the six rules read of the nodes: presence, haves, and every provision's providers with their nodes. func (w *world) nodes(manifests map[string]catalogue.Manifest, assigned map[string][]string) assign.Nodes { - n := assign.Nodes{Online: map[string]bool{}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{}, Providers: map[string]map[string][]string{}, + n := assign.Nodes{Online: map[string]bool{}, Running: w.runningNow(), Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{}, Providers: map[string]map[string][]string{}, Reads: map[string][]string{}} for key, value := range w.read(buckets.Presence) { if node, online, ok := presenceOf(key, value); ok { diff --git a/modules/mesh-delivery-assigner/internal/assign/put.go b/modules/mesh-delivery-assigner/internal/assign/put.go index 34d7ae9b..eb139832 100644 --- a/modules/mesh-delivery-assigner/internal/assign/put.go +++ b/modules/mesh-delivery-assigner/internal/assign/put.go @@ -2,7 +2,6 @@ package assign import ( "fmt" - "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" mwc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshwantsclient" @@ -50,7 +49,10 @@ type PutWorld struct { SnapshotFailed string // Have is what the node says it runs of the module; nil when it says nothing. Have *nodeengine.HaveByNodeModuleValue - Now time.Time + // Running is the node-engine seat's last `running` of the module on the node; nil when it said none (design 47 G §309). A put + // ends on it, never on the have. + Running *nodeengine.Running + Now time.Time } // PutNext is what a step decides: the put as it stands after it, and the acts the pass does for it. @@ -154,26 +156,18 @@ func StepPut(w PutWorld) PutNext { return next } -// applied is a wanted put once the node's have says how it went: holding or done when it runs the composition healthy. +// applied is a wanted put once the node says how it went: holding or done when the node-engine seat's `running` names the put's +// fingerprint (design 47, *Compositions, wants and the apply*: from G a put ends on `running..` naming its +// fingerprint, never on the have). A `running` without a fingerprint, or one that says the module stopped, ends none. The have +// still says a failed health check in the put's words. func applied(w PutWorld, p puts.Put) puts.Put { - h := w.Have - if h == nil || h.Fp != p.FP { - return p - } - // A put without a hold ends on the fingerprint alone: a node that composes its own digests no composition of the put's, and - // a put that waits for that one pins the module's want for good (issue 1966). A held put keeps its composition. - if !p.Hold && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK { + if r := w.Running; r != nil && !r.Stopped && r.Fingerprint != "" && r.Fingerprint == p.FP { + if p.Hold { + return p.Said(w.Now, puts.Holding, fmt.Sprintf("%s runs %s at %s healthy; held there until released", w.Node, w.Module, short(p.FP))) + } return p.Said(w.Now, puts.Done, fmt.Sprintf("%s runs %s at %s healthy; the rules write its want again", w.Node, w.Module, short(p.FP))) } - if h.Composed != p.Composed { - return p - } - switch { - case h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK && p.Hold: - return p.Said(w.Now, puts.Holding, fmt.Sprintf("%s runs %s at %s healthy; held there until released", w.Node, w.Module, short(p.FP))) - case h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK: - return p.Said(w.Now, puts.Done, fmt.Sprintf("%s runs %s at %s healthy; the rules write its want again", w.Node, w.Module, short(p.FP))) - case failed(*h): + if h := w.Have; h != nil && h.Fp == p.FP && h.Composed == p.Composed && failed(*h) { return p.Said(w.Now, p.Step, fmt.Sprintf("%s runs %s at %s and its health check failed: %s", w.Node, w.Module, short(p.FP), h.Detail)) } return p diff --git a/modules/mesh-delivery-assigner/internal/assign/wants.go b/modules/mesh-delivery-assigner/internal/assign/wants.go index 6e7845cd..63a15400 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants.go @@ -109,6 +109,10 @@ func (m Module) composition(node, fp string) (ComposedDigest, bool) { type Nodes struct { Online map[string]bool Haves map[string]map[string]nodeengine.HaveByNodeModuleValue + // Running is, by node and module, the last `running` the node-engine seat said (design 47 G §309): a provider runs healthy + // while the build it runs does, as this says it, whatever a newer want of it waits for (rule 5, main file *A provider + // before its consumers*). Nil: the seat's events are not heard, and the have says it. + Running map[string]map[string]nodeengine.Running // Providers is, by provision, the modules that provide it, with the nodes each is assigned to. Providers map[string]map[string][]string // Reads is, by node, the reader capabilities its node-engine states in its node record (nodeengine.NodeRecord.ManifestReads, @@ -276,6 +280,17 @@ func (m Module) MissingProvisions(node string) []string { return out } +// runsHealthy says whether module runs healthy on node: from G the node-engine seat's last `running` of it names a build and +// does not say it stopped; with the seat's events not heard, the have's phase and health. +func (n Nodes) runsHealthy(node, module string) bool { + if n.Running != nil { + r, ok := n.Running[node][module] + return ok && !r.Stopped && r.Fingerprint != "" + } + h, ok := n.Haves[node][module] + return ok && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK +} + // providedHealthy is rule 5 for one provision: some module providing it runs healthy on the node, or, assigned // elsewhere only, at its canary. A provision no module provides is not a module's to judge (the mesh or the node // offers it) and holds. @@ -295,7 +310,7 @@ func providedHealthy(provision, node string, n Nodes) (bool, string) { if !slices.Contains(nodes, node) { at = Canary(nodes, n.Online) } - if h, ok := n.Haves[at][p]; ok && at != "" && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK { + if at != "" && n.runsHealthy(at, p) { return true, "" } } diff --git a/modules/mesh-delivery-assigner/internal/assign/wants_test.go b/modules/mesh-delivery-assigner/internal/assign/wants_test.go index 1c38e3a7..e3d46708 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants_test.go @@ -923,7 +923,7 @@ func stepsOfAPut(t *testing.T) { w := bus() w.Put = w.Put.Said(now, puts.Wanted, "wanted") w.Put.Composed = d.Digest - w.Have = &nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: d.Digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} + w.Running = &nodeengine.Running{Node: "novox", Module: "nats", Fingerprint: fNew} return w }, puts.Holding, "held there until released", false, false, false}, {"wanted without hold: done", func() PutWorld { @@ -931,7 +931,7 @@ func stepsOfAPut(t *testing.T) { w.Put.Hold = false w.Put = w.Put.Said(now, puts.Wanted, "wanted") w.Put.Composed = d.Digest - w.Have = &nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: d.Digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} + w.Running = &nodeengine.Running{Node: "novox", Module: "nats", Fingerprint: fNew} return w }, puts.Done, "the rules write its want again", false, false, false}, {"wanted without hold, the node composes its own: done at the fingerprint", func() PutWorld { @@ -939,14 +939,27 @@ func stepsOfAPut(t *testing.T) { w.Put.Hold = false w.Put = w.Put.Said(now, puts.Wanted, "wanted") w.Put.Composed = d.Digest - w.Have = &nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: dig(fOld), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} + w.Running = &nodeengine.Running{Node: "novox", Module: "nats", Fingerprint: fNew} return w }, puts.Done, "the rules write its want again", false, false, false}, - {"wanted with hold, the node composes its own: stays wanted", func() PutWorld { + {"wanted, the node's running names no fingerprint: stays wanted (the have runs it, which ends nothing)", func() PutWorld { w := bus() w.Put = w.Put.Said(now, puts.Wanted, "wanted") w.Put.Composed = d.Digest - w.Have = &nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: dig(fOld), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} + w.Running = &nodeengine.Running{Node: "novox", Module: "nats"} + w.Have = &nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: d.Digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} + return w + }, puts.Wanted, "wanted", false, false, false}, + {"wanted, the node's running says it stopped: stays wanted", func() PutWorld { + w := bus() + w.Put = w.Put.Said(now, puts.Wanted, "wanted") + w.Running = &nodeengine.Running{Node: "novox", Module: "nats", Fingerprint: fNew, Stopped: true} + return w + }, puts.Wanted, "wanted", false, false, false}, + {"wanted, the node's running names another build: stays wanted", func() PutWorld { + w := bus() + w.Put = w.Put.Said(now, puts.Wanted, "wanted") + w.Running = &nodeengine.Running{Node: "novox", Module: "nats", Fingerprint: fOld} return w }, puts.Wanted, "wanted", false, false, false}, {"released: nothing more", func() PutWorld { w := bus(); w.Put = w.Put.Said(now, puts.Released, "released"); return w }, puts.Released, "released", false, false, false}, @@ -1546,3 +1559,27 @@ func TestAHeldModuleWithNoWantIsNotOnNeitherList(t *testing.T) { } } } + +// Main file *A provider before its consumers* (design 47 G §309): from G a provider runs healthy while the build it runs is +// healthy, as the node-engine seat's `running` says it, whatever a newer want of it waits for (its have's phase waiting-yes). +func TestAProviderRunsHealthyByItsRunning(t *testing.T) { + waiting := nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseWaitingYes, Health: nodeengineclient.HealthOK} + for _, c := range []struct { + name string + running map[string]map[string]nodeengine.Running + ok bool + }{ + {"its new want waits for a yes, its old build runs", map[string]map[string]nodeengine.Running{"novox": {"nats": {Node: "novox", Module: "nats", Fingerprint: fOld}}}, true}, + {"its running build is stopped", map[string]map[string]nodeengine.Running{"novox": {"nats": {Node: "novox", Module: "nats", Stopped: true}}}, false}, + {"it said no running", map[string]map[string]nodeengine.Running{"novox": {}}, false}, + {"its running names no build", map[string]map[string]nodeengine.Running{"novox": {"nats": {Node: "novox", Module: "nats"}}}, false}, + {"the seat's events are not heard: the have says it, and waiting-yes is not running", nil, false}, + } { + n := Nodes{Online: map[string]bool{"novox": true}, Running: c.running, + Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"novox": {"nats": waiting}}, + Providers: map[string]map[string][]string{"bus-state": {"nats": {"novox"}}}} + if ok, words := providedHealthy("bus-state", "novox", n); ok != c.ok { + t.Errorf("%s: healthy = %v (%s), want %v", c.name, ok, words, c.ok) + } + } +} diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index 1db630c4..ba1b4796 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -123,8 +123,8 @@ func TestManifest(t *testing.T) { } // ADR 0618 §4: and it hears each node's engine say its reads changed; ADR 0636 §4 (issue 1966): and the seat list's two events; // issue 1966 note 151: and the teller's ask-ended, its subject the one the assigner hears. - if !slices.Equal(m.Consumes, []string{"mesh-catalog.registration-changed", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-delivery-indexer.views.graph", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", buckets.TellerModule + "." + tellerevents.AskEnded, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-engine.manifest-reads-changed"}) { - t.Errorf("consumes = %v, want the indexer's views of graph, manifests and modules, the teller's ask-ended and the engines' reads changed", m.Consumes) + if !slices.Equal(m.Consumes, []string{"mesh-catalog.registration-changed", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-delivery-indexer.views.graph", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", buckets.TellerModule + "." + tellerevents.AskEnded, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-engine.manifest-reads-changed", "node-engine.running"}) { + t.Errorf("consumes = %v, want the indexer's views of graph, manifests and modules, the teller's ask-ended, the engines' reads changed and what they run", m.Consumes) } if want := []string{"mesh-delivery-indexer.graph", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "seat:mesh-catalog.registrations", "seat:mesh-catalog.seat-definitions", // ADR 0636 §4: the seat list diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index d6252329..e69ae235 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -48,7 +48,8 @@ "mesh-delivery-teller.ask-ended", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", - "node-engine.manifest-reads-changed" + "node-engine.manifest-reads-changed", + "node-engine.running" ], "invokes": [ "mesh-delivery-indexer.graph",