MH-3: the apply order by rank, the revert, the backup holder and the bus module first, the engine's own module last

settle applies the changed wants in the order of design 47 H 771: the node's own composition,
a reverted module, the holder of node-backup and the bus module (by the claims of the manifests
the composer holds), each pipeline's wants by rank (older source first), the rest by name, the
engine's own module last. A want of a higher rank waits on this machine alone for the lower
ranks of its pipeline, held in its have as a wait of step placing and judged again at the next
settle. The want's pipeline and rank are the mesh-delivery contract's stand-in until its commit.

Builds: design 47 H §771; H-A-46, H-A-59, H-A-67, H-A-68

Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
This commit is contained in:
2026-11-20 02:37:59 +01:00
parent d2b6ecdf0e
commit 949b782a1e
7 changed files with 449 additions and 37 deletions
+54 -13
View File
@@ -193,6 +193,11 @@ type Env struct {
// transient failure only on RetryTransient or a change of either key.
Backoff func(try int) time.Duration
Say func(string)
// Claims says whether the manifest the node holds for a module claims a seat, and whether that manifest is read: the
// holder of the seat node-backup and the bus module are applied first (design 47 H §771). Nil reads none.
Claims func(ctx context.Context, module, seat string) (claims, known bool)
// OwnModule is the node-engine's own module, applied last (design 47 H §771); nil, or "", names none.
OwnModule func() string
// SelfComposed says the compositions are the node's own (novox/hq ADR 0547 §2, issue 1126): Composed hands what the
// node composed itself, unsigned, and a want names the build alone. The composition composed for the want's build is
// the one applied, when it differs from what runs.
@@ -266,7 +271,9 @@ type Agent struct {
// heldAlone is the modules the composer holds alone (NotComposedEach): what runs of each in the node's own
// composition is held out of the node's own apply until a composition of its own is handed (heldOut).
heldAlone map[string]bool
retry chan []string
// placing is, by module, what a want of a higher rank waits on here (design 47 H §771): judged again at every settle.
placing map[string]string
retry chan []string
// reconcile carries the judge's edges: a resource found not active (ADR 0568 §2).
reconcile chan []string
// failedAgain carries the modules whose failed apply a person asked to try again (RetryFailed).
@@ -860,23 +867,57 @@ func (a *Agent) sayOnce(key, line string) {
a.env.Say(line)
}
// settle applies what the dirty modules' wants allow: the node's own first, then each module in name order.
// settle applies what the dirty modules' wants allow, in the order of design 47 H §771 (orderOf): the node's own composition
// first, the holder of the seat node-backup and the bus module, each pipeline's wants by rank, the rest by name, and the
// engine's own module last. A want of a higher rank waits, on this machine alone, until the lower ranks of its pipeline are
// applied or failed, and is held in its have, said as a wait of step placing; it is judged again at the next settle.
func (a *Agent) settle(ctx context.Context, dirty map[string]bool) {
order := make([]string, 0, len(dirty))
for m := range dirty {
order = append(order, m)
a.mu.Lock()
for m := range a.placing {
dirty[m] = true
}
sort.Slice(order, func(i, j int) bool {
if (order[i] == NodeModule) != (order[j] == NodeModule) {
return order[i] == NodeModule
a.mu.Unlock()
modules := make([]string, 0, len(dirty))
for m := range dirty {
modules = append(modules, m)
}
rows, _ := a.orderOf(ctx, modules)
todo := make([]string, 0, len(rows))
for _, r := range rows {
todo = append(todo, r.module)
}
// Each pass applies the first want that waits on nothing, and starts again from the top: a lower rank applied may release a
// want that stands earlier in the order.
for len(todo) > 0 && ctx.Err() == nil {
at := -1
for i, m := range todo {
a.mu.Lock()
waits := a.waitsOnLocked(m)
a.mu.Unlock()
if len(waits) == 0 {
at = i
break
}
}
return order[i] < order[j]
})
for _, m := range order {
if ctx.Err() != nil {
return
if at < 0 {
break
}
m := todo[at]
todo = append(todo[:at], todo[at+1:]...)
a.mu.Lock()
_, waited := a.placing[m]
delete(a.placing, m)
a.mu.Unlock()
a.settleOne(ctx, m)
if waited {
a.endPlacing(ctx, m)
}
}
for _, m := range todo {
a.mu.Lock()
waits := a.waitsOnLocked(m)
a.mu.Unlock()
a.placingHave(ctx, m, waits)
}
}
+105 -20
View File
@@ -135,6 +135,8 @@ type bed struct {
settledRan []bool
ag *Agent // the agent run() runs
env Env
// runs is, by module, a build the agent already runs when run() starts it.
runs map[string]string
}
func newBed(t *testing.T) *bed {
@@ -214,6 +216,9 @@ func (b *bed) run() (stop func()) {
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
b.ag = New(b.env)
for m, fp := range b.runs {
b.ag.running[m] = nodeengine.HaveByNodeModuleValue{Fp: fp, Composed: "sha256:" + strings.Repeat("1", 64), Phase: nodeengineclient.PhaseRunning}
}
go func() { defer close(done); _ = b.ag.Run(ctx) }()
return func() { cancel(); <-done }
}
@@ -980,27 +985,107 @@ func TestARemovedWantRemovesThatModuleOnly(t *testing.T) {
})
}
// The node's own composition is applied before any module's, whichever key came first.
func TestTheNodesOwnCompositionComesBeforeAnyModules(t *testing.T) {
b := newBed(t)
fp := fpOf("a")
ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
wk, wraw := wantFor(t, "web", fp, dig, "")
nk, nraw, ndig := composition(t, NodeModule, "", false)
nwk := meshwants.WantsByWantNodeNodeKey(node)
nwraw, err := bucketstest.NodeWant(meshwants.WantsByWantNodeNodeValue{Composed: ndig})
if err != nil {
t.Fatal(err)
// Design 47 H §771: the node-engine applies the wants changed at once in one order: the node's own composition, a reverted
// module, the holder of the seat node-backup and the bus module (found by their claims), each pipeline's wants by rank (the
// older source first), the rest by name, and its own module last; a want of a higher rank waits, on this machine, for the lower
// ranks of its pipeline, held in its have as a wait of step placing, and the read apply-order answers the order with why.
func TestTheOrderTheChangedWantsAreAppliedIn(t *testing.T) {
type mod struct {
name string
pipeline string // the want's pipeline, with its rank
rank int64
claims string // the seat its manifest claims
runs string // the build it runs before, the want naming another (a revert when it is in no pipeline)
late bool // its composition is made after the start
}
b.comps.before(ck, craw)
b.comps.before(nk, nraw)
b.wants.before(wk, wraw)
b.wants.before(nwk, nwraw)
stop := b.run()
defer stop()
eventually(t, "both not applied", func() bool { return len(b.appliedList()) == 2 })
if got := b.appliedList(); got[0] != NodeModule || got[1] != "web" {
t.Fatalf("the order was %v, the node's own first", got)
base := []mod{{name: "zeta1", pipeline: "main.7", rank: 1}, {name: "alpha2", pipeline: "main.7", rank: 2}, {name: "old", pipeline: "main.5", rank: 1},
{name: "zeta"}, {name: "alpha"}, {name: "bk", claims: SeatBackup}, {name: "bus", claims: SeatBus}, {name: "engine"}}
for _, row := range []struct {
name string
mods []mod
order []string // what is applied, in order, once the late compositions are made
first []string // what is applied before them
read []string // what apply-order answers at the start: module position:waits
}{
{"every class in its place", base, []string{NodeModule, "bk", "bus", "old", "zeta1", "alpha2", "alpha", "zeta", "engine"}, nil,
nil},
{"a revert comes first", append([]mod{{name: "rev", runs: "z"}}, base...),
[]string{NodeModule, "rev", "bk", "bus", "old", "zeta1", "alpha2", "alpha", "zeta", "engine"}, nil, nil},
{"a higher rank waits for its lower rank", []mod{{name: "zeta1", pipeline: "main.7", rank: 1, late: true}, {name: "alpha2", pipeline: "main.7", rank: 2},
{name: "other"}}, []string{NodeModule, "other", "zeta1", "alpha2"}, []string{NodeModule, "other"},
[]string{"zeta1:1:", "alpha2:2:zeta1"}},
} {
t.Run(row.name, func(t *testing.T) {
b := newBed(t)
b.runs = map[string]string{}
b.env.OwnModule = func() string { return "engine" }
b.env.Claims = func(_ context.Context, module, seat string) (bool, bool) {
for _, m := range row.mods {
if m.name == module {
return m.claims == seat, true
}
}
return false, false
}
var late []string
for i, m := range row.mods {
fp := fpOf(string("0123456789abcdef"[i]))
if m.runs != "" {
b.runs[m.name] = fpOf(m.runs)
}
ck, craw, dig := composition(t, m.name, fp, false, file(m.name, "x", "1"))
raw, err := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: fp, Composed: dig, Pipeline: m.pipeline, Rank: m.rank})
if err != nil {
t.Fatal(err)
}
if m.late {
late = append(late, ck, string(craw))
} else {
b.comps.before(ck, craw)
}
b.wants.before(meshwants.WantsByWantNodeModuleKey(node, m.name), raw)
}
nk, nraw, ndig := composition(t, NodeModule, "", false)
nwraw, err := bucketstest.NodeWant(meshwants.WantsByWantNodeNodeValue{Composed: ndig})
if err != nil {
t.Fatal(err)
}
b.comps.before(nk, nraw)
b.wants.before(meshwants.WantsByWantNodeNodeKey(node), nwraw)
stop := b.run()
defer stop()
if row.first != nil {
eventually(t, "the first wants are applied", func() bool { return len(b.appliedList()) == len(row.first) })
b.idle()
if got := b.appliedList(); !slices.Equal(got, row.first) {
t.Fatalf("applied %v before the late compositions, want %v", got, row.first)
}
var got []string
for _, r := range b.ag.ApplyOrder(context.Background()) {
got = append(got, fmt.Sprintf("%s:%d:%s", r.Module, r.Position, strings.Join(r.Waits, ",")))
}
// The node's own is applied; the rest stand in the order, each with what it waits on.
if !slices.Equal(got, row.read) {
t.Fatalf("apply-order answers %v, want %v", got, row.read)
}
if h := b.haves.last("alpha2"); h.Phase != string(nodeengineclient.PhaseHeld) || h.Wait == nil || h.Wait.Step != nodeengine.WaitStepPlacing || h.Wait.On != "zeta1" {
t.Fatalf("alpha2's have is %s with wait %+v, want held in a wait of step placing on zeta1", h.Phase, h.Wait)
}
for i := 0; i < len(late); i += 2 {
b.comps.put(late[i], []byte(late[i+1]))
}
}
eventually(t, "every want is applied", func() bool { return len(b.appliedList()) == len(row.order) })
b.idle()
if got := b.appliedList(); !slices.Equal(got, row.order) {
t.Fatalf("applied %v, want %v", got, row.order)
}
if row.first != nil {
if h := b.haves.last("alpha2"); h.Wait != nil || h.Phase != string(nodeengineclient.PhaseRunning) {
t.Fatalf("alpha2's have is %s with wait %+v after the lower rank applied, want running and no wait", h.Phase, h.Wait)
}
}
})
}
}
+249
View File
@@ -0,0 +1,249 @@
package agent
import (
"context"
"fmt"
"sort"
"strconv"
"strings"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
)
// The order on one machine (design 47 H §771, H-A-46, H-A-59): the node-engine applies its changed wants, one module at a time,
// the node's own composition first, a reverted module, then the holder of the seat node-backup and the bus module, then each pipeline's wants by
// rank (the older source first), then the rest by name, and its own module last. It reads the order from the wants alone and
// consumes no event of the runner.
const (
// SeatBackup and SeatBus are the seats whose holders are applied first on a machine: found by the claim of the manifest the
// composer holds for the module, never by a module's name.
SeatBackup = "node-backup"
SeatBus = "mesh-broker"
)
// order classes: the lower is applied first.
const (
classNode = iota
classRevert
classBackup
classBus
classPipeline
classRest
classOwn
)
// rowOf is one module's place in the order.
type rowOf struct {
module string
class int
pipeline string
branch string
source int64
rank int64
}
// changedLocked says a module's want is not applied yet: it is there, and what runs of the module is not what it names
// (a.mu held).
func (a *Agent) changedLocked(module string) bool {
w, ok := a.wants[module]
if !ok || !w.Present {
return false
}
r, runs := a.running[module]
if w.Want.Removed {
return runs && r.Phase != nodeengineclient.PhaseRemoved
}
if module == NodeModule {
return !runs || r.Composed != w.Want.Composed || r.Phase == nodeengineclient.PhaseFailed
}
return !(runs && r.Fp == w.Want.Fp && r.Phase != nodeengineclient.PhaseRemoved && r.Phase != nodeengineclient.PhaseFailed)
}
// revertLocked says a module's want is a revert: it names no pipeline and no rank (a placement's build) and a fingerprint other
// than the one the module runs (design 47 H §771, H-A-68; a.mu held).
func (a *Agent) revertLocked(module string) bool {
w, ok := a.wants[module]
if !ok || !w.Present || w.Want.Removed || module == NodeModule || w.Want.Pipeline != "" || w.Want.Rank > 0 || w.Want.Fp == "" {
return false
}
r, runs := a.running[module]
return runs && r.Phase != nodeengineclient.PhaseRemoved && r.Fp != "" && r.Fp != w.Want.Fp
}
// waitsOn is the modules a want waits on: the changed wants of the same pipeline of a lower rank, that did not fail here
// (a.mu held). A want of no pipeline or of no rank waits on none.
func (a *Agent) waitsOnLocked(module string) []string {
w := a.wants[module].Want
if w.Pipeline == "" || w.Rank <= 0 {
return []string{}
}
out := []string{}
for m, other := range a.wants {
if m == module || other.Want.Pipeline != w.Pipeline || other.Want.Rank <= 0 || other.Want.Rank >= w.Rank || !a.changedLocked(m) {
continue
}
// A lower rank that failed here ends the wait; one retried for a failure of the bus does not (it still waits).
if att, tried := a.attempt[m]; tried && att.FP == other.Want.Fp && !att.Held && !att.Transient && !a.runsLocked(m) {
continue
}
out = append(out, m)
}
sort.Strings(out)
return out
}
// runsLocked says the module runs what its want names (a.mu held).
func (a *Agent) runsLocked(module string) bool {
r, runs := a.running[module]
return runs && r.Fp == a.wants[module].Want.Fp && r.Phase != nodeengineclient.PhaseRemoved && r.Phase != nodeengineclient.PhaseFailed
}
// pipelineParts splits a pipeline's key `<branch>.<source>` into its branch and its source number.
func pipelineParts(key string) (branch string, source int64) {
i := strings.LastIndex(key, ".")
if i < 0 {
return key, 0
}
n, err := strconv.ParseInt(key[i+1:], 10, 64)
if err != nil {
return key, 0
}
return key[:i], n
}
// claims says whether the manifest the node holds for the module claims the seat; a module whose manifest is not read claims none.
func (a *Agent) claims(ctx context.Context, module, seat string) bool {
if a.env.Claims == nil || module == NodeModule {
return false
}
claims, known := a.env.Claims(ctx, module, seat)
return claims && known
}
// orderOf is the modules in the order they are applied, each with its place and why; modules are the ones asked for, whatever
// the state of their wants.
func (a *Agent) orderOf(ctx context.Context, modules []string) ([]rowOf, map[string]string) {
own := ""
if a.env.OwnModule != nil {
own = a.env.OwnModule()
}
// The claims are asked of the composer before a.mu is taken: it asks what the agent runs.
backups, buses := map[string]bool{}, map[string]bool{}
for _, m := range modules {
backups[m] = a.claims(ctx, m, SeatBackup)
buses[m] = !backups[m] && a.claims(ctx, m, SeatBus)
}
a.mu.Lock()
defer a.mu.Unlock()
reverts := map[string]bool{}
for _, m := range modules {
reverts[m] = a.revertLocked(m)
}
rows := make([]rowOf, 0, len(modules))
why := map[string]string{}
for _, m := range modules {
w := a.wants[m].Want
r := rowOf{module: m, class: classRest}
switch {
case m == NodeModule:
r.class, why[m] = classNode, "the node's own composition, which every module's openings and accounts depend on"
case m == own && own != "":
r.class, why[m] = classOwn, "the node-engine's own module, applied last"
case reverts[m]:
r.class, why[m] = classRevert, "a revert to the build "+w.Fp+", applied first"
case backups[m]:
r.class, why[m] = classBackup, "holds the seat "+SeatBackup+", applied first"
case buses[m]:
r.class, why[m] = classBus, "claims the seat "+SeatBus+", the bus module, applied first"
case w.Pipeline != "" && w.Rank > 0:
r.class, r.pipeline, r.rank = classPipeline, w.Pipeline, w.Rank
r.branch, r.source = pipelineParts(w.Pipeline)
why[m] = fmt.Sprintf("rank %d of the pipeline %s", w.Rank, w.Pipeline)
default:
why[m] = "in no pipeline: by name"
}
rows = append(rows, r)
}
sort.SliceStable(rows, func(i, j int) bool {
x, y := rows[i], rows[j]
if x.class != y.class {
return x.class < y.class
}
if x.class == classPipeline {
if x.source != y.source {
return x.source < y.source
}
if x.branch != y.branch {
return x.branch < y.branch
}
if x.rank != y.rank {
return x.rank < y.rank
}
}
return x.module < y.module
})
return rows, why
}
// ApplyOrder is the order this machine's node-engine will apply its changed wants in, one row per want not applied yet (the verb
// apply-order of the seat node-engine, design 47 H §771).
func (a *Agent) ApplyOrder(ctx context.Context) []nodeengine.ApplyOrderAnswerOrderItem {
a.mu.Lock()
var modules []string
for m := range a.wants {
if a.changedLocked(m) {
modules = append(modules, m)
}
}
a.mu.Unlock()
rows, why := a.orderOf(ctx, modules)
out := make([]nodeengine.ApplyOrderAnswerOrderItem, 0, len(rows))
a.mu.Lock()
defer a.mu.Unlock()
for i, r := range rows {
w := a.wants[r.module].Want
item := nodeengine.ApplyOrderAnswerOrderItem{Module: r.module, Position: int64(i + 1), Why: why[r.module], Waits: a.waitsOnLocked(r.module)}
if r.class == classPipeline || (w.Pipeline != "" && w.Rank > 0) {
item.Pipeline, item.Rank = w.Pipeline, w.Rank
}
out = append(out, item)
}
return out
}
// placingHave writes a module's have held for the lower rank of its pipeline it waits on, once per distinct wait, the module
// keeping what it runs (design 47 H §771, H-A-59). The reason `placing` is a hold the engine judges again at a start.
func (a *Agent) placingHave(ctx context.Context, module string, waits []string) {
on := strings.Join(waits, ", ")
words := fmt.Sprintf("%s waits for %s of its pipeline, of a lower rank, to be applied on this machine", module, on)
a.mu.Lock()
if a.placing == nil {
a.placing = map[string]string{}
}
same := a.placing[module] == on
a.placing[module] = on
prev, runs := a.running[module]
a.mu.Unlock()
if same {
return
}
if !runs {
if h, found, _ := a.readHave(ctx, module); found {
prev.Fp, prev.Composed = h.FP, h.Composed
}
}
a.env.Say(words)
a.writeHave(ctx, Have{Module: module, FP: prev.Fp, Composed: prev.Composed, Phase: string(nodeengineclient.PhaseHeld), Detail: words,
Wait: &nodeengine.Wait{Owner: nodeengine.Seat, Step: nodeengine.WaitStepPlacing, Reason: nodeengine.WaitStepPlacing, On: on, Since: a.now(), Words: words}})
}
// endPlacing ends the wait of a module whose want was held for a lower rank and is judged again: when its have still carries the
// wait and the module runs a build, the have says it running again, the wait ended.
func (a *Agent) endPlacing(ctx context.Context, module string) {
h, found, _ := a.readHave(ctx, module)
if !found || h.Wait == nil || h.Wait.Reason != nodeengine.WaitStepPlacing || h.FP == "" {
return
}
a.writeHave(ctx, Have{Module: module, FP: h.FP, Composed: h.Composed, Phase: string(nodeengineclient.PhaseRunning)})
}
+7 -3
View File
@@ -1,6 +1,10 @@
package agent
import "strings"
import (
"strings"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
)
// EntryAsked and EntryMissing open the detail of a have held for the entry the want names (design 47 G section 421 and
// section 614): the entry is asked of the catalog half and not yet answered, or the half answered that it holds none. Both
@@ -11,14 +15,14 @@ const (
)
// judgedAgain says a held have is a hold the node-engine judges again at a start, not one that waits for the operator's answer:
// its wait's reason (the structured field the have carries) is that its entry is not held (§421), whatever words its detail
// its wait's reason (the structured field the have carries) is that its entry is not held (§421) or it waits for a lower rank of its pipeline (§771), whatever words its detail
// opens with. Only a failed backup or restore point waits for an answer (ADR 0568 §4); g14, 2026-11-18: feh held "whether ..." with the reason entry-missing was kept the same way.
func judgedAgain(h Have) bool {
if h.Wait == nil {
return false
}
switch h.Wait.Reason {
case strings.TrimSuffix(EntryAsked, ":"), strings.TrimSuffix(EntryMissing, ":"):
case strings.TrimSuffix(EntryAsked, ":"), strings.TrimSuffix(EntryMissing, ":"), nodeengine.WaitStepPlacing:
return true
}
return false