Merge branch 'feat/47-G-build' into feat/47-G-build-watchman

This commit is contained in:
2026-11-11 12:14:30 +01:00
17 changed files with 243 additions and 147 deletions
@@ -425,9 +425,9 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
hub := process.NewEventHub(logger.Printf)
// The seat list (seatlist.go), read over the connection and heard on the hub, given to catalogue at every change, and a
// pass made then: registered before the hub starts.
gate := &seatGate{}
gate := seatlist.NewPart("assigner", seatListAgainCallers)
if bus != nil {
gate.follow(seatlist.ProcessBus(bus.Call, hub), wake, logger.Printf)
gate.Follow(seatlist.ProcessBus(bus.Call, hub), wake, logger.Printf)
}
// The indexer's manifests, modules and module graph, through its first views and views family, never its buckets
// (DLV-F2r, ADR 0631): the first pass waits for the three answers, as it waits for every watch to hand what stands.
@@ -502,7 +502,8 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
logger.Printf("a pipeline's end or a registration is not heard, so the wants keep the trunk-only form until the assigner starts again: %v", err)
}
}
hearReadsChanged(ctx, js, hub, reads, findings, gate.eventsLive, logger.Printf)
hearReadsChanged(ctx, js, hub, reads, findings, gate.EventsLive, logger.Printf)
hearRunning(ctx, js, w, wake, findings, logger.Printf)
}
if bus != nil && kvs[buckets.Puts] != nil {
p.acts = newPutActs(bus, wake, logger.Printf)
@@ -602,7 +603,7 @@ func hearReadsChanged(ctx context.Context, js jetstream.JetStream, hub *process.
}
w.Heard(process.EventsStream)
if live != nil {
live() // the seat list, unread while the seat was silent, is read now (seatGate.eventsLive)
live() // the seat list, unread while the seat was silent, is read now (seatlist.Part.EventsLive)
}
go func() {
<-ctx.Done()
@@ -18,6 +18,8 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
"git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay"
"git.novox.be/novox/mesh-sdk/go/motion"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"sync"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient"
@@ -79,7 +81,8 @@ type passer struct {
seat *seatActs
// seats is the seat list the process reads (seatlist.go): until it is read, seat-holders is not written and a proposal
// laid again is not ended on a seat it does not name (ADR 0636 §1). Nil reads nothing.
seats *seatGate
seats *seatlist.Part
seatWaitSaid sync.Once
publish publisher
saidProposed map[string]bool
// copied says the retired state was copied into `assignments` by this process, or found empty (copyRetired).
@@ -1248,11 +1251,11 @@ func (p *passer) seatHolders(ctx context.Context) {
if len(assignments) == 0 {
return // no assignments read yet: nothing is known of who holds what, and nothing is deleted
}
if !p.seats.read() {
if !p.seats.Heard() {
// ADR 0636 §1: which claim names which seat is the list's, so nothing is written from the bootstrap alone; the
// reader's change makes a pass.
if p.seats != nil {
p.seats.waitSaid.Do(func() { p.logf("seat-holders is not written until the seat list is read: %s", p.seats.health()) })
p.seatWaitSaid.Do(func() { p.logf("seat-holders is not written until the seat list is read: %s", p.seats.Health()) })
}
return
}
@@ -651,8 +651,21 @@ func putOfTheBus(t *testing.T) {
t.Errorf("the warrant was recorded as %q", call)
}
world(buckets.Wants, "want.novox.nats", wantsKV.values["want.novox.nats"])
running, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Composed: digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
world(buckets.Have, hkey, running)
// Design 47 G §309: the put ends on the node-engine seat's running naming its fingerprint, never on the have.
haveRunning, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Composed: digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
world(buckets.Have, hkey, haveRunning)
if got := step(); got.Step != puts.Wanted {
t.Fatalf("the have runs it and no running says so, the put stands at %s: %s", got.Step, got.Words)
}
if err := p.w.hearRunning("mesh.seat.node-engine.event.running.novox.nats", []byte(`{"v":1,"at":"2026-11-10T20:00:00Z","node":"novox","module":"nats"}`)); err != nil {
t.Fatal(err)
}
if got := step(); got.Step != puts.Wanted {
t.Fatalf("a running without a fingerprint ends none, the put stands at %s: %s", got.Step, got.Words)
}
if err := p.w.hearRunning("mesh.seat.node-engine.event.running.novox.nats", []byte(`{"v":1,"at":"2026-11-10T20:01:00Z","node":"novox","module":"nats","fingerprint":"`+fp+`"}`)); err != nil {
t.Fatal(err)
}
if got := step(); got.Step != puts.Holding {
t.Fatalf("running healthy, the put stands at %s: %s", got.Step, got.Words)
}
@@ -221,6 +221,9 @@ func (p *passer) putWorld(node, module string, put puts.Put, manifest catalogue.
}
}
}
if r, ok := p.w.runningOf(node, module); ok {
w.Running = &r
}
if hkey, err := nodeengineclient.HaveKey(node, module); err == nil {
if raw := p.w.value(buckets.Have, hkey); raw != nil {
if h, err := nodeengineclient.DecodeHave(raw); err == nil {
@@ -0,0 +1,88 @@
package main
import (
"context"
"encoding/json"
"fmt"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assignment"
)
// What each node runs is heard, never asked (design 47 G §199, §309): the node-engine seat's event `running.<node>.<module>` is
// kept last per subject on its own stream, and the assigner's durable consumer on it starts from the last message of every
// subject, so a node away still has its last word here. The assigner counts a provider healthy by it (rule 5) and ends a put on it.
// hearRunning keeps each `running` the stream holds and each it says from now, and wakes the pass on every one.
func hearRunning(ctx context.Context, js jetstream.JetStream, w *world, wake func(), findings assign.Findings, logf func(string, ...any)) {
acct := process.Account{Node: thisNode, Module: assignment.Module}
nh := notHeardWriter(findings, logf)
cc, err := acct.HearSeatEvents(ctx, js, nodeengine.StreamRunning, func(m jetstream.Msg) {
if err := m.Ack(); err != nil {
logf("acknowledging %s: %v", m.Subject(), err)
}
if err := w.hearRunning(m.Subject(), m.Data()); err != nil {
logf("%v", err)
return
}
wake()
}, logf)
if err != nil {
nh.NotHeard(nodeengine.StreamRunning, "what each node runs is not heard until the assigner starts again, so a provider is counted healthy by its have and a put does not end", err)
return
}
w.mu.Lock()
w.runningHeard = true
w.mu.Unlock()
nh.Heard(nodeengine.StreamRunning)
go func() {
<-ctx.Done()
cc.Stop()
}()
}
// hearRunning keeps one `running` event said on subject.
func (w *world) hearRunning(subject string, data []byte) error {
var r nodeengine.Running
if err := json.Unmarshal(data, &r); err != nil {
return fmt.Errorf("%s holds what is not a running: %w", subject, err)
}
if r.Node == "" || r.Module == "" {
return fmt.Errorf("%s names no node or no module, so it is kept for none", subject)
}
w.mu.Lock()
w.running[r.Node+"."+r.Module] = r
w.mu.Unlock()
return nil
}
// runningOf is the last `running` heard of module on node.
func (w *world) runningOf(node, module string) (nodeengine.Running, bool) {
w.mu.Lock()
defer w.mu.Unlock()
r, ok := w.running[node+"."+module]
return r, ok
}
// runningNow is every `running` heard, by node and module; nil while the seat's events are not heard (the rules then read the
// have).
func (w *world) runningNow() map[string]map[string]nodeengine.Running {
w.mu.Lock()
defer w.mu.Unlock()
if !w.runningHeard {
return nil
}
out := map[string]map[string]nodeengine.Running{}
for _, r := range w.running {
if out[r.Node] == nil {
out[r.Node] = map[string]nodeengine.Running{}
}
out[r.Node][r.Module] = r
}
return out
}
@@ -440,10 +440,10 @@ func (p *passer) mustAnswer(ctx context.Context, prop mma.Proposal, rev uint64,
func (p *passer) layAgain(ctx context.Context, prop mma.Proposal, rev uint64, current map[string]mma.Assignments, revs map[string]uint64, why string) {
held, recorded := p.lays(ctx)
again, laid, findings := assign.Propose(assign.ProposeIn{ID: prop.ID, By: prop.By, Why: prop.Why, At: prop.At, Changes: prop.Changes,
Current: current, Revisions: revs, Held: held, Recorded: recorded, SeatListRead: p.seats.read()})
Current: current, Revisions: revs, Held: held, Recorded: recorded, SeatListRead: p.seats.Heard()})
if slices.ContainsFunc(findings, plc.WaitsForSeatList) {
// ADR 0636 §1: nothing is ended on a list not read; the reader's change makes a pass, which lays it again.
p.logf("the proposal %s is not laid again yet: %s", prop.ID, p.seats.health())
p.logf("the proposal %s is not laid again yet: %s", prop.ID, p.seats.Health())
return
}
if len(findings) > 0 {
@@ -19,6 +19,7 @@ import (
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-assigner/internal/assign"
@@ -86,7 +87,7 @@ type seatTools struct {
now func() time.Time
// seats is the seat list the bundle reads (seatlist.go): until it is read, propose judges no mesh seat the bootstrapped
// definitions do not name as held once (ADR 0636 §1). Nil reads nothing.
seatList *seatGate
seatList *seatlist.Part
}
func newSeatTools(st seatStore, ask func(string, any) (json.RawMessage, error)) *seatTools {
@@ -265,7 +266,7 @@ func (s *seatTools) doPropose(args map[string]any, by string) (any, error) {
enrollers := plc.EnrollersOf(current)
now := s.now()
p, laid, findings := assign.Propose(assign.ProposeIn{ID: plc.NewProposalID(now), By: by, Why: arg(args, "why"), At: now,
Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded, Warrant: warrant, SeatListRead: s.seatList.read()})
Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded, Warrant: warrant, SeatListRead: s.seatList.Heard()})
if refusal := plc.AddNodeRefusal(p, moduleOfCaller(by, enrollers), enrollers); refusal != "" {
findings = append(findings, refusal)
}
@@ -257,7 +257,7 @@ func TestTheSeatsVerbs(t *testing.T) {
// Such a session is proved as its machine's tool runner (issue 627); the healers too, no other module.
for caller, refused := range map[string]bool{"person.jochen": false, "shanks.node-tools": false, "ace.mesh-healers": false,
"ace.messenger": true} {
if err := seatlist.Refusal(seatListAgainTool, seatListAgainCallers, caller); (err != nil) != refused {
if err := seatlist.Refusal("assigner_seat_list_again", seatListAgainCallers, caller); (err != nil) != refused {
t.Errorf("%s: refused=%v, err %v", caller, refused, err)
}
}
@@ -1,103 +1,13 @@
package main
import (
"errors"
"sync"
"sync/atomic"
import stdio "git.novox.be/novox/mesh-sdk/go"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/seatlist"
)
// The assigner reads the seat list as every part does (novox/hq ADR 0635 §1, ADR 0636 §1, §7; issue 1966): through the seat
// mesh-catalog's verb seat-definitions and its two events, from the bootstrapped definitions until the seat has answered. What
// the reader holds is given to catalogue at every change, which the check of a proposal and the seat-holders state read: which
// mesh seat is replicated, and which name is a seat's. Both of its runtimes hold one: the process (the proposals it lays again,
// the state seat-holders) over its connection and the event hub, and the tools' bundle (the seat's verb propose) over the
// runtime. Until the list is read, a mesh seat the bootstrapped definitions do not name is never judged held once
// The assigner reads the seat list as every part does, through mesh-sdk's one wiring (go/seatlist.Part; design 47 G §204): its
// reader over the seat mesh-catalog's verb seat-definitions and its two events, its tools `assigner_seat_list` and
// `assigner_seat_list_again`. What the reader holds is given to catalogue at every change, which the check of a proposal and the
// seat-holders state read; until the list is read, a mesh seat the bootstrapped definitions do not name is never judged held once
// (plc.CheckAssignments), and seat-holders is not written.
// Seat list tools (ADR 0636 §7): what the reader holds, and reading it again now, which every operator's session and the
// healers may ask.
const (
seatListTool = "assigner_seat_list"
seatListAgainTool = "assigner_seat_list_again"
)
// A call through the mesh MCP server is proved only as its machine's tool runner (<node>.node-tools, issue 627), so
// node-tools stands for an operator's session until a session is proved (issue 1966).
var seatListAgainCallers = []string{stdio.AnyPerson, "node-tools", "mesh-healers"}
// seatGate is the reader a runtime of the assigner judges by, set once it is started; before, the list counts as unread.
type seatGate struct {
reader atomic.Pointer[seatlist.Reader]
// waitSaid says the journal was told that seat-holders waits for the list, so it is told once per wait.
waitSaid sync.Once
}
// read says the seat mesh-catalog has answered the reader: the list is read (seatlist.Status's Heard). A nil gate, or one
// whose reader has not started, has read nothing.
func (g *seatGate) read() bool {
if g == nil {
return false
}
r := g.reader.Load()
return r != nil && r.Status().Heard
}
// health is why the list is not read, or empty.
func (g *seatGate) health() string {
if g == nil {
return "the assigner follows no seat list"
}
r := g.reader.Load()
if r == nil {
return "the seat list reader has not started yet"
}
return r.Status().Health()
}
// follow starts the reader over bus: what it holds is given to catalogue at once and at every change, and changed is called
// then (the process makes a pass, which lays again what waited).
func (g *seatGate) follow(bus seatlist.Bus, changed func(), logf func(string, ...any)) *seatlist.Reader {
reader, err := seatlist.Start(seatlist.Config{Bus: bus, Bootstrap: seatlist.Bootstrapped(),
OnChange: func(seats []catalogue.Seat) {
catalogue.UseSeats(seats)
if changed != nil {
changed()
}
}})
if err != nil {
logf("the assigner cannot follow its seat list, so a mesh seat the bootstrapped definitions do not name waits for it: %v", err)
}
if reader != nil {
catalogue.UseSeats(reader.Seats())
g.reader.Store(reader)
}
return reader
}
// eventsLive is called once the process's consumer on EVENTS is heard: the seat may have said it speaks before that consumer
// existed, whose events start at the stream's end, so while the list is unread it is read now (ADR 0636 §1).
func (g *seatGate) eventsLive() {
if r := g.reader.Load(); r != nil {
r.ReloadIfUnread()
}
}
// tools are the reader's two tools over whatever reader the gate holds at the call.
func (g *seatGate) tools() []stdio.Tool {
shape := seatlist.Tools(&seatlist.Reader{}, seatListTool, seatListAgainTool, seatListAgainCallers)
for i := range shape {
run := i
shape[i].Run = func(args map[string]any) (any, error) {
r := g.reader.Load()
if r == nil {
return nil, errors.New("the seat list reader has not started yet: the runtime has not served this module")
}
return seatlist.Tools(r, seatListTool, seatListAgainTool, seatListAgainCallers)[run].Run(args)
}
}
return shape
}
@@ -3,6 +3,9 @@ package main
import (
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
@@ -50,21 +53,44 @@ func (standInCatalog) Subscribe(string, func(string, json.RawMessage) error) err
// gateOver is a seat gate following the stand-in, once its first read has ended. A read one gives catalogue the test list
// (catalogue.UseSeats), as a host's does, for the tests after it too: the list is the same everywhere.
func gateOver(t *testing.T, silent bool) *seatGate {
func gateOver(t *testing.T, silent bool) *seatlist.Part {
t.Helper()
if silent {
// A silent seat reached no one: its reader asks again within the bus's retry bound (issue 1966), which a test
// that waits for it to stay unread keeps short.
t.Cleanup(busretry.Waits(20*time.Millisecond, time.Millisecond, 5*time.Millisecond))
}
g := &seatGate{}
r := g.follow(standInCatalog{silent: silent}, nil, t.Logf)
g := seatlist.NewPart("assigner", seatListAgainCallers)
r := g.Follow(standInCatalog{silent: silent}, nil, t.Logf)
if r == nil {
t.Fatal("the reader did not start")
}
<-r.Ready()
if g.read() == silent {
t.Fatalf("the gate read %v over a stand-in silent %v: %s", g.read(), silent, g.health())
if g.Heard() == silent {
t.Fatalf("the gate read %v over a stand-in silent %v: %s", g.Heard(), silent, g.Health())
}
return g
}
// Design 47 G §204: the seat list's reader is wired once, by mesh-sdk's go/seatlist.Part; no part wires its own. A search of the
// assigner's code finds no start of a reader and no tools of its own.
func TestTheSeatListIsWiredByTheSDKAlone(t *testing.T) {
files, err := filepath.Glob("*.go")
if err != nil {
t.Fatal(err)
}
for _, f := range files {
if strings.HasSuffix(f, "_test.go") {
continue
}
raw, err := os.ReadFile(f)
if err != nil {
t.Fatal(err)
}
for _, own := range []string{"seatlist.Start(", "seatlist.Tools(", "OnChange:"} {
if strings.Contains(string(raw), own) {
t.Errorf("%s wires the seat list itself (%s), where mesh-sdk's seatlist.Part does it", f, own)
}
}
}
}
@@ -386,15 +386,15 @@ func serveTools() error {
}
}()
// The seat list (seatlist.go), read through the runtime once Serve has its channel: the seat's verbs judge by it.
gate := &seatGate{}
gate := seatlist.NewPart("assigner", seatListAgainCallers)
go func() {
<-stdio.Served()
gate.follow(seatlist.StdioBus(), nil, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: "+f+"\n", a...) })
gate.Follow(seatlist.StdioBus(), nil, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: "+f+"\n", a...) })
}()
seatVerbs := newSeatTools(runtimeStore{}, stdio.Ask)
seatVerbs.seatList = gate
tools := append(s.tools(), seatVerbs.tools()...)
tools = append(tools, gate.tools()...)
tools = append(tools, gate.Tools()...)
wants := newWantsTools(runtimeStore{}, stdio.Ask, func(e, id string, b any) error {
// The seat's event, said on the seat: a claimed seat's event is never the module's own.
return stdio.SeatPublish(wantsEventSubject(e), b, id)
@@ -52,6 +52,10 @@ type world struct {
// sourceGroups is, by revision of the gate's source.main, the release (group) that moved it, from every revision its
// history keeps (watched with history): a build's source revision names the release that carries it (ADR 0601 §1).
sourceGroups map[uint64]string
// running is, by `<node>.<module>`, the last `running` the node-engine seat said of it, kept from its stream by the assigner's
// durable consumer (design 47 G §199, §309); runningHeard says that consumer is bound.
running map[string]nodeengine.Running
runningHeard bool
}
// entryMeta is a value's revision in its bucket and when it was written.
@@ -70,7 +74,7 @@ type parsed struct {
func newWorld() *world {
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{},
meta: map[string]entryMeta{}, trunkBuilt: map[string]builds.BuildLogEntry{}, assignmentKeys: mmac.NewKeys(),
sourceGroups: map[uint64]string{}}
sourceGroups: map[uint64]string{}, running: map[string]nodeengine.Running{}}
}
// putEntry is put, keeping the value's revision and the time it was written.
@@ -391,7 +395,7 @@ func (w *world) wantsOf() map[string]map[string]mwc.Want {
// nodes is what the six rules read of the nodes: presence, haves, and every provision's providers with their nodes.
func (w *world) nodes(manifests map[string]catalogue.Manifest, assigned map[string][]string) assign.Nodes {
n := assign.Nodes{Online: map[string]bool{}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{}, Providers: map[string]map[string][]string{},
n := assign.Nodes{Online: map[string]bool{}, Running: w.runningNow(), Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{}, Providers: map[string]map[string][]string{},
Reads: map[string][]string{}}
for key, value := range w.read(buckets.Presence) {
if node, online, ok := presenceOf(key, value); ok {
@@ -2,7 +2,6 @@ package assign
import (
"fmt"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
mwc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshwantsclient"
@@ -50,7 +49,10 @@ type PutWorld struct {
SnapshotFailed string
// Have is what the node says it runs of the module; nil when it says nothing.
Have *nodeengine.HaveByNodeModuleValue
Now time.Time
// Running is the node-engine seat's last `running` of the module on the node; nil when it said none (design 47 G §309). A put
// ends on it, never on the have.
Running *nodeengine.Running
Now time.Time
}
// PutNext is what a step decides: the put as it stands after it, and the acts the pass does for it.
@@ -154,26 +156,18 @@ func StepPut(w PutWorld) PutNext {
return next
}
// applied is a wanted put once the node's have says how it went: holding or done when it runs the composition healthy.
// applied is a wanted put once the node says how it went: holding or done when the node-engine seat's `running` names the put's
// fingerprint (design 47, *Compositions, wants and the apply*: from G a put ends on `running.<node>.<module>` naming its
// fingerprint, never on the have). A `running` without a fingerprint, or one that says the module stopped, ends none. The have
// still says a failed health check in the put's words.
func applied(w PutWorld, p puts.Put) puts.Put {
h := w.Have
if h == nil || h.Fp != p.FP {
return p
}
// A put without a hold ends on the fingerprint alone: a node that composes its own digests no composition of the put's, and
// a put that waits for that one pins the module's want for good (issue 1966). A held put keeps its composition.
if !p.Hold && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK {
if r := w.Running; r != nil && !r.Stopped && r.Fingerprint != "" && r.Fingerprint == p.FP {
if p.Hold {
return p.Said(w.Now, puts.Holding, fmt.Sprintf("%s runs %s at %s healthy; held there until released", w.Node, w.Module, short(p.FP)))
}
return p.Said(w.Now, puts.Done, fmt.Sprintf("%s runs %s at %s healthy; the rules write its want again", w.Node, w.Module, short(p.FP)))
}
if h.Composed != p.Composed {
return p
}
switch {
case h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK && p.Hold:
return p.Said(w.Now, puts.Holding, fmt.Sprintf("%s runs %s at %s healthy; held there until released", w.Node, w.Module, short(p.FP)))
case h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK:
return p.Said(w.Now, puts.Done, fmt.Sprintf("%s runs %s at %s healthy; the rules write its want again", w.Node, w.Module, short(p.FP)))
case failed(*h):
if h := w.Have; h != nil && h.Fp == p.FP && h.Composed == p.Composed && failed(*h) {
return p.Said(w.Now, p.Step, fmt.Sprintf("%s runs %s at %s and its health check failed: %s", w.Node, w.Module, short(p.FP), h.Detail))
}
return p
@@ -109,6 +109,10 @@ func (m Module) composition(node, fp string) (ComposedDigest, bool) {
type Nodes struct {
Online map[string]bool
Haves map[string]map[string]nodeengine.HaveByNodeModuleValue
// Running is, by node and module, the last `running` the node-engine seat said (design 47 G §309): a provider runs healthy
// while the build it runs does, as this says it, whatever a newer want of it waits for (rule 5, main file *A provider
// before its consumers*). Nil: the seat's events are not heard, and the have says it.
Running map[string]map[string]nodeengine.Running
// Providers is, by provision, the modules that provide it, with the nodes each is assigned to.
Providers map[string]map[string][]string
// Reads is, by node, the reader capabilities its node-engine states in its node record (nodeengine.NodeRecord.ManifestReads,
@@ -276,6 +280,17 @@ func (m Module) MissingProvisions(node string) []string {
return out
}
// runsHealthy says whether module runs healthy on node: from G the node-engine seat's last `running` of it names a build and
// does not say it stopped; with the seat's events not heard, the have's phase and health.
func (n Nodes) runsHealthy(node, module string) bool {
if n.Running != nil {
r, ok := n.Running[node][module]
return ok && !r.Stopped && r.Fingerprint != ""
}
h, ok := n.Haves[node][module]
return ok && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK
}
// providedHealthy is rule 5 for one provision: some module providing it runs healthy on the node, or, assigned
// elsewhere only, at its canary. A provision no module provides is not a module's to judge (the mesh or the node
// offers it) and holds.
@@ -295,7 +310,7 @@ func providedHealthy(provision, node string, n Nodes) (bool, string) {
if !slices.Contains(nodes, node) {
at = Canary(nodes, n.Online)
}
if h, ok := n.Haves[at][p]; ok && at != "" && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK {
if at != "" && n.runsHealthy(at, p) {
return true, ""
}
}
@@ -923,7 +923,7 @@ func stepsOfAPut(t *testing.T) {
w := bus()
w.Put = w.Put.Said(now, puts.Wanted, "wanted")
w.Put.Composed = d.Digest
w.Have = &nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: d.Digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
w.Running = &nodeengine.Running{Node: "novox", Module: "nats", Fingerprint: fNew}
return w
}, puts.Holding, "held there until released", false, false, false},
{"wanted without hold: done", func() PutWorld {
@@ -931,7 +931,7 @@ func stepsOfAPut(t *testing.T) {
w.Put.Hold = false
w.Put = w.Put.Said(now, puts.Wanted, "wanted")
w.Put.Composed = d.Digest
w.Have = &nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: d.Digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
w.Running = &nodeengine.Running{Node: "novox", Module: "nats", Fingerprint: fNew}
return w
}, puts.Done, "the rules write its want again", false, false, false},
{"wanted without hold, the node composes its own: done at the fingerprint", func() PutWorld {
@@ -939,14 +939,27 @@ func stepsOfAPut(t *testing.T) {
w.Put.Hold = false
w.Put = w.Put.Said(now, puts.Wanted, "wanted")
w.Put.Composed = d.Digest
w.Have = &nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: dig(fOld), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
w.Running = &nodeengine.Running{Node: "novox", Module: "nats", Fingerprint: fNew}
return w
}, puts.Done, "the rules write its want again", false, false, false},
{"wanted with hold, the node composes its own: stays wanted", func() PutWorld {
{"wanted, the node's running names no fingerprint: stays wanted (the have runs it, which ends nothing)", func() PutWorld {
w := bus()
w.Put = w.Put.Said(now, puts.Wanted, "wanted")
w.Put.Composed = d.Digest
w.Have = &nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: dig(fOld), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
w.Running = &nodeengine.Running{Node: "novox", Module: "nats"}
w.Have = &nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: d.Digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
return w
}, puts.Wanted, "wanted", false, false, false},
{"wanted, the node's running says it stopped: stays wanted", func() PutWorld {
w := bus()
w.Put = w.Put.Said(now, puts.Wanted, "wanted")
w.Running = &nodeengine.Running{Node: "novox", Module: "nats", Fingerprint: fNew, Stopped: true}
return w
}, puts.Wanted, "wanted", false, false, false},
{"wanted, the node's running names another build: stays wanted", func() PutWorld {
w := bus()
w.Put = w.Put.Said(now, puts.Wanted, "wanted")
w.Running = &nodeengine.Running{Node: "novox", Module: "nats", Fingerprint: fOld}
return w
}, puts.Wanted, "wanted", false, false, false},
{"released: nothing more", func() PutWorld { w := bus(); w.Put = w.Put.Said(now, puts.Released, "released"); return w }, puts.Released, "released", false, false, false},
@@ -1546,3 +1559,27 @@ func TestAHeldModuleWithNoWantIsNotOnNeitherList(t *testing.T) {
}
}
}
// Main file *A provider before its consumers* (design 47 G §309): from G a provider runs healthy while the build it runs is
// healthy, as the node-engine seat's `running` says it, whatever a newer want of it waits for (its have's phase waiting-yes).
func TestAProviderRunsHealthyByItsRunning(t *testing.T) {
waiting := nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseWaitingYes, Health: nodeengineclient.HealthOK}
for _, c := range []struct {
name string
running map[string]map[string]nodeengine.Running
ok bool
}{
{"its new want waits for a yes, its old build runs", map[string]map[string]nodeengine.Running{"novox": {"nats": {Node: "novox", Module: "nats", Fingerprint: fOld}}}, true},
{"its running build is stopped", map[string]map[string]nodeengine.Running{"novox": {"nats": {Node: "novox", Module: "nats", Stopped: true}}}, false},
{"it said no running", map[string]map[string]nodeengine.Running{"novox": {}}, false},
{"its running names no build", map[string]map[string]nodeengine.Running{"novox": {"nats": {Node: "novox", Module: "nats"}}}, false},
{"the seat's events are not heard: the have says it, and waiting-yes is not running", nil, false},
} {
n := Nodes{Online: map[string]bool{"novox": true}, Running: c.running,
Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"novox": {"nats": waiting}},
Providers: map[string]map[string][]string{"bus-state": {"nats": {"novox"}}}}
if ok, words := providedHealthy("bus-state", "novox", n); ok != c.ok {
t.Errorf("%s: healthy = %v (%s), want %v", c.name, ok, words, c.ok)
}
}
}
@@ -123,8 +123,8 @@ func TestManifest(t *testing.T) {
}
// ADR 0618 §4: and it hears each node's engine say its reads changed; ADR 0636 §4 (issue 1966): and the seat list's two events;
// issue 1966 note 151: and the teller's ask-ended, its subject the one the assigner hears.
if !slices.Equal(m.Consumes, []string{"mesh-catalog.registration-changed", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-delivery-indexer.views.graph", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", buckets.TellerModule + "." + tellerevents.AskEnded, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-engine.manifest-reads-changed"}) {
t.Errorf("consumes = %v, want the indexer's views of graph, manifests and modules, the teller's ask-ended and the engines' reads changed", m.Consumes)
if !slices.Equal(m.Consumes, []string{"mesh-catalog.registration-changed", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-delivery-indexer.views.graph", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", buckets.TellerModule + "." + tellerevents.AskEnded, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-engine.manifest-reads-changed", "node-engine.running"}) {
t.Errorf("consumes = %v, want the indexer's views of graph, manifests and modules, the teller's ask-ended, the engines' reads changed and what they run", m.Consumes)
}
if want := []string{"mesh-delivery-indexer.graph", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules",
"seat:mesh-catalog.registrations", "seat:mesh-catalog.seat-definitions", // ADR 0636 §4: the seat list
+2 -1
View File
@@ -48,7 +48,8 @@
"mesh-delivery-teller.ask-ended",
"mesh-pipeline-runner.pipeline-ended",
"mesh-pipeline-runner.group-ended",
"node-engine.manifest-reads-changed"
"node-engine.manifest-reads-changed",
"node-engine.running"
],
"invokes": [
"mesh-delivery-indexer.graph",