Design 47 H1: the one chain (take-in, forge bridge, pipeline runner, assigner, teller, view, watchman, grants; checker gone) #263

Merged
mesh-tools merged 104 commits from feat/47-H1 into main 2026-11-20 07:15:13 +00:00
44 changed files with 758 additions and 5863 deletions
Showing only changes of commit 1cf2ea46ff - Show all commits
@@ -102,10 +102,6 @@ func TestATrunkEntryTakesItsVersionFromTheReleaseOfItsFingerprintsSource(t *test
}
// The release names each artifact by its whole reference, as the catalog says it, and the parts are keyed so (A-96); the
// assigner reads the entry's fp-src from its inputs (A-419).
// A-429: a released version's build counts as done for rule 1; a kept build (no version) and an unknown one do not.
if !releasedBuild(p.namedBuild("mail", "t", 7), "t") || releasedBuild(p.namedBuild("mail", "t", 7), "k") || releasedBuild(p.namedBuild("mail", "t", 8), "t") || releasedBuild(nil, "t") {
t.Error("rule 1's released build: the release of the entry's fingerprint counts, a kept build, another source or none does not")
}
if v, _, _ := fromEntry(meshcatalog.CatalogEntry{Module: "mail", Fp: "t", Inputs: &meshcatalog.EntryInputs{Src: 9, FpSrc: 7}}); v.FpSrc != 7 || v.Src != 9 {
t.Errorf("the entry's module value carries fp-src %d and src %d, want 7 and 9", v.FpSrc, v.Src)
}
@@ -130,25 +126,3 @@ func TestATrunkEntryTakesItsVersionFromTheReleaseOfItsFingerprintsSource(t *test
t.Error("no revision kept names a release")
}
}
// A-162: the assigner knows a failed build from its `build.ended`: the error whole; a done end or none heard is not failed.
func TestTheStandingOfABuildIsHeardFromItsEnd(t *testing.T) {
w := newWorld()
say := func(e builds.BuildEnded) {
raw, _ := json.Marshal(e)
if err := w.ends.put(raw); err != nil {
t.Fatal(err)
}
}
if failed, _ := w.buildStanding("web", "fp1"); failed {
t.Error("a build nothing was heard of failed")
}
say(builds.BuildEnded{Module: "web", Fingerprint: "fp1", Outcome: builds.BuildFailed, Error: "transport fault: connection"})
if failed, why := w.buildStanding("web", "fp1"); !failed || why != "transport fault: connection" {
t.Errorf("failed end: failed %v why %q", failed, why)
}
say(builds.BuildEnded{Module: "web", Fingerprint: "fp1", Outcome: builds.BuildDone})
if failed, _ := w.buildStanding("web", "fp1"); failed {
t.Error("a later done end leaves the build failed")
}
}
@@ -504,7 +504,7 @@ func caseOf(half mcc.Half) (map[string]composer.Manifest, map[string]json.RawMes
if e.Verdict == "" || e.Verdict == mcc.VerdictAccepted {
continue
}
v := meshcatalog.VerdictsByDigestValue{V: 1, Module: name, State: e.Verdict}
v := caseVerdict{V: 1, Module: name, State: e.Verdict}
if e.Verdict != mcc.VerdictRefused {
v.State = mcc.VerdictManifestWaits
}
@@ -520,6 +520,17 @@ func caseOf(half mcc.Half) (map[string]composer.Manifest, map[string]json.RawMes
return manifests, verdicts, none, nil
}
// caseVerdict is a manifest's verdict as the composer's case reads it, by an entry's own verdict: the form the seat mesh-catalog's
// state verdicts had, which left the seat with the take-in's validation (design 47 H §745, H-A-79).
type caseVerdict struct {
V int64 `json:"v,omitempty"`
Module string `json:"module"`
State string `json:"state"`
Problems []string `json:"problems,omitempty"`
Unknown []string `json:"unknown,omitempty"`
Warnings []string `json:"warnings,omitempty"`
}
// nodeEngineEntry is the entry of the module that claims the seat node-engine, or false.
func nodeEngineEntry(entries map[string]meshcatalog.CatalogEntry) (meshcatalog.CatalogEntry, bool) {
for _, name := range slices.Sorted(mapsKeys(entries)) {
@@ -346,7 +346,7 @@ func TestOneCompositionRunPerCandidateAndMachine(t *testing.T) {
sup := asks(2)
readers(runKeyOf("run-r3", "n1"), 2)
readers(runKeyOf("run-r3", "n2"), 2)
pe, _ := json.Marshal(mpr.PipelineEnded{V: 1, Pipeline: "run-r3.a", Module: "a", Branch: trunkBranch, Commit: commit, Outcome: mpr.PipelineEndedOutcomeSuperseded, Run: "r3", Steps: []mpr.StepResult{}})
pe, _ := json.Marshal(mpr.PipelineEnded{V: 1, Pipeline: "run-r3.a", Branch: trunkBranch, Outcome: mpr.PipelineEndedOutcomeSuperseded, Run: "r3", Movers: []mpr.MoverEnded{{Module: "a", Steps: []mpr.StepResult{}}}})
if _, err := b.js.Publish(b.ctx, "mesh.seat."+mpr.Seat+".event."+mpr.EventPipelineEnded+".n1", pe); err != nil {
t.Fatal(err)
}
@@ -3,7 +3,6 @@ package main
import (
"context"
"encoding/json"
"strings"
"sync"
"time"
@@ -11,7 +10,6 @@ import (
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatdef"
)
// The composition step (design 47 G §22, §354, §63): the assigner takes the seat mesh-wants' accepted kind `step`, the
@@ -40,8 +38,6 @@ type compositionSteps struct {
ended func(id string, run meshwants.StepRun, at time.Time)
now func() time.Time
logf func(string, ...any)
// asker is the name the step's progress names as its asker, read when it is said (stepAskerOf).
asker func() string
// ticks is a ticker of every d, its channel and its stop (time.NewTicker); nil is the clock's. A test hands its own.
ticks func(d time.Duration) (<-chan time.Time, func())
mu sync.Mutex
@@ -49,32 +45,6 @@ type compositionSteps struct {
stops map[string]context.CancelFunc
}
// stepAskerOf is the asker a composition step's progress names: the one caller the seat mesh-wants' definition lists for its
// accepted kind step, read from the seat list when it is said, since that subject names no asker (design 47 G A-294, which
// corrects A-267). Empty while the seat list holds no definition naming exactly one.
func stepAskerOf(definition func() (seatdef.Definition, bool)) func() string {
return func() string {
d, ok := definition()
if !ok {
return ""
}
var one string
for _, m := range d.Accepts {
if m.Name != meshwants.AcceptStep || m.Callers == nil {
continue
}
for _, e := range m.Callers.List {
name, isSeat := strings.CutPrefix(e, seatdef.CallerSeatPrefix)
if !isSeat || (one != "" && one != name) {
return ""
}
one = name
}
}
return one
}
}
// progressEvery is how often a running composition step says its progress (design 47 G §496, A-262).
const progressEvery = time.Minute
@@ -154,7 +124,7 @@ func (c *compositionSteps) say1(ctx context.Context, id string, run meshwants.St
stopWait()
}()
phase := &stepPhase{phase: meshwants.StepProgressEventPhaseFetch}
stop := c.sayProgress(id, phase)
stop := c.sayProgress(id, run.Asker, phase)
o := how(context.WithValue(ctx, phaseKey{}, phase), id, run)
stop()
if ctx.Err() != nil {
@@ -222,7 +192,7 @@ func (c *compositionSteps) leave(ids []string) {
// sayProgress says the step's progress on the seat's step-progress.<node> once a minute until the returned stop is called (design
// 47 G §496, A-262): a ticker inside the running step, as the build seats' step runner says its own beat.
func (c *compositionSteps) sayProgress(id string, phase *stepPhase) (stop func()) {
func (c *compositionSteps) sayProgress(id, asker string, phase *stepPhase) (stop func()) {
ticks := c.ticks
if ticks == nil {
ticks = func(d time.Duration) (<-chan time.Time, func()) { t := time.NewTicker(d); return t.C, t.Stop }
@@ -239,7 +209,7 @@ func (c *compositionSteps) sayProgress(id string, phase *stepPhase) (stop func()
return
case <-tick:
at := c.now().UTC()
p := meshwants.StepProgressEvent{Ask: id, Asker: c.asker(), Phase: phase.get(), At: at, Shared: phase.waitsOn()}
p := meshwants.StepProgressEvent{Ask: id, Asker: asker, Phase: phase.get(), At: at, Shared: phase.waitsOn()}
if err := c.say(meshwants.EventStepProgress+"."+c.node, p, id+".progress."+at.Format(time.RFC3339Nano)); err != nil {
c.logf("saying the progress of the composition step %s: %v", id, err)
}
@@ -3,7 +3,6 @@ package main
import (
"context"
"encoding/json"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"slices"
"strings"
"sync"
@@ -84,23 +83,7 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) {
stopped := make(chan struct{})
at := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
release := make(chan struct{})
// A-294: the asker is the one caller mesh-wants' definition lists for step, read from the seat list; none while unread
// or while it lists more than one.
runner := seatdef.Definition{Seat: meshwants.Seat, Accepts: []seatdef.Message{{Name: meshwants.AcceptStep,
Callers: &seatdef.Callers{List: []string{"seat:mesh-pipeline-runner"}}}}}
two := seatdef.Definition{Seat: meshwants.Seat, Accepts: []seatdef.Message{{Name: meshwants.AcceptStep,
Callers: &seatdef.Callers{List: []string{"seat:mesh-pipeline-runner", "seat:mesh-catalog"}}}}}
for _, d := range []struct {
def seatdef.Definition
read bool
want string
}{{runner, true, "mesh-pipeline-runner"}, {runner, false, ""}, {two, true, ""}} {
if got := stepAskerOf(func() (seatdef.Definition, bool) { return d.def, d.read })(); got != d.want {
t.Errorf("the asker of %+v (read %v) = %q, want %q", d.def.Accepts, d.read, got, d.want)
}
}
c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return at }, logf: t.Logf,
asker: stepAskerOf(func() (seatdef.Definition, bool) { return runner, true }),
ticks: func(d time.Duration) (<-chan time.Time, func()) {
if d != time.Minute {
t.Errorf("the progress ticks every %s, want a minute", d)
@@ -113,7 +96,8 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) {
<-release
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed}
}}
body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4.m", Step: meshwants.StepRunStepComposition, Module: "m", Branch: "main"})
// A11/AC1 (design 47 H §752, H-A-88): with no callers in mesh-wants, the progress names the asker the step run carries.
body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4", Step: meshwants.StepRunStepComposition, Module: "m", Branch: "main", Asker: "mesh-pipeline-runner"})
if !c.take(context.Background(), "ask-1", body) {
t.Fatal("the composition step is not taken")
}
@@ -6,7 +6,6 @@ import (
"encoding/hex"
"encoding/json"
"fmt"
"maps"
"slices"
"strings"
"time"
@@ -37,8 +36,8 @@ type graphKeeper struct {
said string
// full says the next revision did not fit the store, so no want is written (graphfill.go); started says the first pass judged the fill.
full, started bool
// pins removes a revision that ceased to be current once nothing pins it (pins.go); nil keeps every revision.
pins *pinKeeper
// revisions removes a revision that ceased to be current (revisions.go, H-A-86); nil keeps every revision.
revisions *revisionRemover
// computed is the revision computed by the last pass, with its entries: its edges.
computed assigngraph.Computed
}
@@ -51,9 +50,6 @@ func (p *passer) keepGraph(ctx context.Context) {
}
c := assigngraph.Compute(p.graphInputs())
g.computed = c
if p.ends != nil {
p.ends.setRunner(assigngraph.Runner(slices.Collect(maps.Values(c.Entries)))) // the runner's member of a group goes first (A-334)
}
for _, u := range c.Uncomputed {
if g.said != u {
g.said = u
@@ -87,8 +83,8 @@ func (p *passer) keepGraph(ctx context.Context) {
return
}
p.fillingAfter(ctx, used)
if g.pins != nil {
g.pins.judge() // the revision before ceased to be current
if g.revisions != nil {
g.revisions.judge() // the revision before ceased to be current
}
id := fmt.Sprintf("%s-%s-%d", meshwants.EventAssignmentRevision, rev.Kind, rev.Revision)
if err := p.wantsSay(meshwants.EventAssignmentRevision, assigngraph.Event(rev), id); err != nil {
@@ -637,7 +637,7 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
if err != nil || want.Fp != fNew {
t.Fatalf("the canary ace's want = %+v, %v", want, err)
}
if w := b.wantNow(t, "shanks", "web", "shanks' first want at what it runs", func(w mwc.Want) bool { return w.Fp != "" }); w.Fp != fOld || w.Composed != "" || !w.Self || w.Newest != fNew {
if w := b.wantNow(t, "shanks", "web", "shanks' first want at what it runs", func(w mwc.Want) bool { return w.Fp != "" }); w.Fp != fOld || !w.Self {
t.Errorf("shanks' want = %+v, want the fingerprint it runs, the node composing it, held back from F", w)
}
// DLV-O5 (ADR 0615 §5, ADR 0619 §2): each want is said on the assigner's views family, its value whole at its revision,
@@ -665,7 +665,7 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
// there by itself: the condition is cleared.
healthyF, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
b.putJSON(t, buckets.Have, "ace.web", healthyF)
if w := b.wantNow(t, "g14", "web", "g14's first want at F once the canary runs it healthy", func(w mwc.Want) bool { return w.Fp != "" }); w.Fp != fNew || w.Composed != "" || !w.Self || w.Newest != "" {
if w := b.wantNow(t, "g14", "web", "g14's first want at F once the canary runs it healthy", func(w mwc.Want) bool { return w.Fp != "" }); w.Fp != fNew || !w.Self {
t.Errorf("g14's want = %+v, want F, the node composing it", w)
}
cleared := false
@@ -689,7 +689,7 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
b.putJSON(t, plc.AssignmentsBucket(), k, values[k])
}
}
if w := b.wantNow(t, "ace", "web", "ace's want once placed: the build alone, composed by the node", func(w mwc.Want) bool { return w.Self }); w.Composed != "" || w.Fp != fNew {
if w := b.wantNow(t, "ace", "web", "ace's want once placed: the build alone, composed by the node", func(w mwc.Want) bool { return w.Self }); w.Fp != fNew {
t.Errorf("ace's want once placed = %+v", w)
}
nkey, _ := mwc.WantNodeKey("ace")
@@ -720,7 +720,7 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
if err != nil {
t.Fatal(err)
}
b.putJSON(t, buckets.Provisions, "ace.app.database", raw)
b.putJSON(t, "mesh-provisioner_provisions", "ace.app.database", raw)
}
// The provisioner keys it from the manifests, as soon as they are known and whether the provider is placed yet or not.
asked(meshprovisionerclient.Asked, "the provider's build is not done")
@@ -842,8 +842,8 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
t.Errorf("g14, whose engine answers nothing and which has no record, reads %+v", s)
}
if w := b.wantNow(t, "ace", "forms", "the want held on ace, naming the form", func(w mwc.Want) bool {
return w.Newest == formsNew && strings.Contains(w.Words, "form:new")
}); w.Fp != formsOld || w.Rule != mwc.RuleComposed {
return strings.Contains(w.Words, "form:new")
}); w.Fp != formsOld {
t.Errorf("forms' want = %+v, want it held at what ace runs under rule 7", w)
}
// Issue 1966 note 170, 1966/AC170-4: ace's engine says its statement, and while the assigner's ask of it is held,
@@ -916,32 +916,6 @@ func (s bedState) Await(key string, after uint64, done func([]byte) bool, bound
return nil, fmt.Errorf("no answer at %s within %s", key, bound)
}
// 1297/B9 (the assigner's half, ADR 0528 clause 6): started whole, the assigner holds a fingerprint whose health check
// failed on a node and writes the ask about it in its own asks, which the view and the teller read; its asks bucket is
// the one its manifest declares (ownBuckets, held equal to the manifest by TestOpensWhatItsManifestNames).
func TestAHeldFingerprintIsAskedAbout(t *testing.T) {
b := startBed(t, "")
fNew := meshcatalogclient.SourcePrefix2 + strings.Repeat("c", 64)
digest := b.manifest(t, "web", `{"module":"web","version":"1"}`)
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fNew, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
b.putModule(t, "web", mod)
failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"})
b.putJSON(t, buckets.Have, "ace.web", failed)
b.place(t, "ace", "web")
b.start(t)
hkey, _ := mwc.HeldKey("web", fNew)
// 1906/H3: the held record names the machine and keeps the health, when and the node's words.
if h, err := mwc.DecodeHeld(b.get(t, buckets.Wants(), hkey)); err != nil || h.Node != "ace" ||
!strings.HasPrefix(h.Words, "health failed, phase running, seen at ") || !strings.HasSuffix(h.Words, ": connection refused") {
t.Fatalf("held = %+v, %v", h, err)
}
akey, _ := asks.HoldAskKey("web", fNew)
a, err := decodeAsk(akey, b.get(t, buckets.Asks(), akey))
if err != nil || a.Asking <= 0 || a.Ask.ID != meshdeliveryclient.AskingID(akey, a.Asking) || len(a.Ask.Options) != 2 {
t.Fatalf("the ask = %+v, %v", a, err)
}
}
// conditionsSaid reads what the assigner said of its conditions, by finding key: each a finding in its own bucket, which
// the condition keeper carries (novox/hq ADR 0590 §3, issue 1126), raised while it stands and cleared once deleted, each
// with the words its last finding said and the subject the finding names (`<module>[.<fingerprint>]`).
@@ -972,47 +946,6 @@ func (b *bed) conditionsSaid(t *testing.T) map[string]oc.Condition {
return out
}
// 1906/D3 (ADR 0539 §19), whole on the bus: a module holding data whose new fingerprint failed its health check on the
// canary is held, not set back, and asked about (restore it, or leave it stopped); the condition names the module, the node
// and the check's words.
func TestAFailedDataModuleIsAskedToRestore(t *testing.T) {
b := startBed(t, "")
fNew := meshcatalogclient.SourcePrefix2 + strings.Repeat("d", 64)
digest := b.manifest(t, "mail", `{"module":"mail","version":"1","resources":[{"id":"data","type":"directory","path":"/var/lib/mail"}],"data":{"own":[{"id":"mail","path":"${dir:data}","class":"irreplaceable","backup":"copy"}]}}`)
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fNew, Manifest: digest, Repository: "novox/mail", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
b.putModule(t, "mail", mod)
failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"})
b.putJSON(t, buckets.Have, "ace.mail", failed)
wantKey, _ := mwc.WantKey("ace", "mail")
want, _ := mwc.EncodeWant(mwc.Want{Fp: fNew, Composed: "sha256:" + strings.Repeat("c", 64)})
b.putJSON(t, buckets.Wants(), wantKey, want)
b.place(t, "ace", "mail")
b.start(t)
akey, _ := asks.RestoreAskKey("ace", "mail", fNew)
a, err := decodeAsk(akey, b.get(t, buckets.Asks(), akey))
if err != nil || len(a.Ask.Options) != 2 || a.Ask.Options[0].ID != "restore" || a.Ask.Options[1].ID != "leave-stopped" {
t.Fatalf("the restore ask = %+v, %v", a, err)
}
if w, err := mwc.DecodeWant(b.get(t, buckets.Wants(), wantKey)); err != nil || w.Fp != fNew {
t.Errorf("the data module was set back: want = %+v, %v", w, err)
}
key := ""
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && key == ""; time.Sleep(100 * time.Millisecond) {
for k, c := range b.conditionsSaid(t) {
if c.State == occ.StateRaised && strings.HasPrefix(c.Subject.ID, "mail.") && strings.HasSuffix(k, ".held") {
if c.Subject.Machine != "ace" || !strings.Contains(c.Explanation, "connection refused") {
t.Fatalf("the condition = %+v, want it to name ace and the check's words", c)
}
key = k
}
}
}
if key == "" {
t.Fatal("no condition for the held fingerprint was said")
}
}
// 1906/D8 (scenario D of acceptance test 18, ADR 0539 §19), whole on the bus: the assigner is stopped while it writes the wants
// of forty modules and another is started; every module's want is written, and each exactly once, by revision: nothing lost
// and nothing written again.
@@ -1136,43 +1069,6 @@ func (b *bed) hasKey(bucket, key string) bool {
return err == nil
}
// 1906/D4 (ADR 0539 §19), whole on the bus: the operator's release of a held fingerprint writes its want again, by the rules,
// and clears the condition.
func TestAReleaseAnswerWritesTheWantAgain(t *testing.T) {
b := startBed(t, "")
fNew, fOld := meshcatalogclient.SourcePrefix2+strings.Repeat("b", 64), meshcatalogclient.SourcePrefix2+strings.Repeat("a", 64)
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
b.compose(t, "ace", 'b')
b.compose(t, "ace", 'a')
healthy, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
b.putJSON(t, buckets.Have, "ace.web", healthy) // it ran the older fingerprint healthy before
failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"})
b.putJSON(t, buckets.Have, "ace.web", failed)
b.putWant(t, "ace", "web", mwc.Want{Fp: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
b.start(t)
w := b.wantNow(t, "ace", "web", "set back to the older fingerprint", func(w mwc.Want) bool { return w.Fp == fOld })
if w.Newest != fNew {
t.Fatalf("the set-back want = %+v", w)
}
hold, _ := asks.HoldAskKey("web", fNew)
b.get(t, buckets.Asks(), hold)
b.answer(t, hold, "release")
w = b.wantNow(t, "ace", "web", "the held fingerprint's want written again", func(w mwc.Want) bool { return w.Fp == fNew })
if w.Composed != "" || !w.Self || w.Rule != 0 {
t.Errorf("the want = %+v, want F by the rules, the node composing it", w)
}
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
cleared := false
for k, c := range b.conditionsSaid(t) {
cleared = cleared || (strings.HasPrefix(k, "module.web.") && c.State == occ.StateCleared && strings.HasSuffix(k, ".held"))
}
if cleared {
return
}
}
t.Error("the condition of the held fingerprint was not cleared")
}
// askNow waits for the backup ask at key to be the one numbered asking and answers it.
func (b *bed) askNow(t *testing.T, key string, asking int) decodedAsk {
t.Helper()
@@ -1213,7 +1109,7 @@ func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) {
}
b.answerTo(t, akey, mwc.BackupAgain, a.Ask.ID)
w1 := b.wantNow(t, "ace", "web", "the answer in the want", func(w mwc.Want) bool { return w.Backup == mwc.BackupAgain })
if w1.Fp != fNew || w1.Composed != "" || !w1.Self {
if w1.Fp != fNew || !w1.Self {
t.Errorf("the want = %+v", w1)
}
// the node retries: its have is backing up, not held. The ask stands and its condition is not cleared (A-453a).
@@ -1260,112 +1156,6 @@ func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) {
const recorded = `{"module":"web","version":"1","upgrade":{"policy":"record","why":"a person takes each build"}}`
// 1906/H2, whole on the bus: given a fingerprint held after one failed check, when every node that runs the module then runs
// it healthy, the assigner lifts the hold by itself: the held record and its ask are taken away, the held condition is
// cleared saying why, and the want stays at the fingerprint.
func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
b := startBed(t, "")
fNew := meshcatalogclient.SourcePrefix2 + strings.Repeat("b", 64)
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
b.compose(t, "ace", 'b')
failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"})
b.putJSON(t, buckets.Have, "ace.web", failed)
b.putWant(t, "ace", "web", mwc.Want{Fp: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
b.start(t)
hkey, _ := mwc.HeldKey("web", fNew)
akey, _ := asks.HoldAskKey("web", fNew)
b.get(t, buckets.Wants(), hkey)
b.get(t, buckets.Asks(), akey)
healthy, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
b.putJSON(t, buckets.Have, "ace.web", healthy)
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
if b.hasKey(buckets.Wants(), hkey) || b.hasKey(buckets.Asks(), akey) {
continue
}
for k, c := range b.conditionsSaid(t) {
if strings.HasPrefix(k, "module.web.") && c.State == occ.StateCleared && strings.HasSuffix(k, ".held") {
if w := b.wantNow(t, "ace", "web", "the want at the lifted fingerprint", func(w mwc.Want) bool { return w.Fp == fNew }); w.Rule != 0 {
t.Errorf("the want = %+v", w)
}
return
}
}
}
t.Fatalf("the hold was not lifted: held %v, ask %v", b.hasKey(buckets.Wants(), hkey), b.hasKey(buckets.Asks(), akey))
}
// 1906/H2, whole on the bus: a hold lifted by itself and written again after the assigner was started again is asked under a
// new id, so the teller raises it; the answer to the first hold does not carry over (1906/H4).
func TestAHoldWrittenAgainAfterALiftAndARestartIsAskedAgain(t *testing.T) {
b := startBed(t, "")
fNew := meshcatalogclient.SourcePrefix2 + strings.Repeat("b", 64)
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
b.compose(t, "ace", 'b')
failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"})
healthy, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
b.putJSON(t, buckets.Have, "ace.web", failed)
b.putWant(t, "ace", "web", mwc.Want{Fp: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
_, stop := b.startStoppable(t)
hkey, _ := mwc.HeldKey("web", fNew)
akey, _ := asks.HoldAskKey("web", fNew)
first, err := decodeAsk(akey, b.get(t, buckets.Asks(), akey))
if err != nil {
t.Fatal(err)
}
b.putJSON(t, buckets.Have, "ace.web", healthy)
for deadline := time.Now().Add(15 * time.Second); b.hasKey(buckets.Wants(), hkey); time.Sleep(100 * time.Millisecond) {
if time.Now().After(deadline) {
t.Fatal("the hold was not lifted")
}
}
stop()
time.Sleep(1100 * time.Millisecond) // the next hold is written in another second
b.answerTo(t, akey, "keep", first.Ask.ID) // an answer to the first hold, which the later hold must not take
b.putJSON(t, buckets.Have, "ace.web", failed)
_, stop = b.startStoppable(t)
again, err := decodeAsk(akey, b.get(t, buckets.Asks(), akey))
if err != nil {
t.Fatal(err)
}
if again.Ask.ID == first.Ask.ID {
t.Fatalf("the hold written again is asked under the first id %s", first.Ask.ID)
}
// Issue 1966 note 155, ADR 0668 §4: that asking lapses at the operator channel while no assigner runs, so the teller's
// ask-ended is said before the next one's consumer binds; at its bind the next reads the operator-channel seat's asks and
// writes the hold again under a new asking.
stop()
nc, err := nats.Connect(b.srv.ClientURL())
if err != nil {
t.Fatal(err)
}
defer nc.Close()
for _, subject := range []string{"mesh.seat.operator-channel.tool.asks", "mesh.seat.operator-channel.call.asks.*"} {
if _, err := nc.Subscribe(subject, func(m *nats.Msg) {
answer, _ := json.Marshal(map[string]any{"asks": []map[string]string{{"key": "mesh-delivery-teller." + again.Ask.ID, "state": "expired"}}, "matched": 1})
reply, _ := json.Marshal(map[string]any{"result": json.RawMessage(answer)})
_ = m.Respond(reply)
}); err != nil {
t.Fatal(err)
}
}
subscribed(t, nc)
b.start(t)
for deadline := time.Now().Add(15 * time.Second); ; time.Sleep(50 * time.Millisecond) {
if third, err := decodeAsk(akey, b.get(t, buckets.Asks(), akey)); err == nil && third.Ask.ID != again.Ask.ID {
break
}
if time.Now().After(deadline) {
t.Fatalf("the hold whose asking %s lapsed before the assigner's bind was not asked again", again.Ask.ID)
}
}
b.putJSON(t, buckets.Have, "ace.web", healthy)
for deadline := time.Now().Add(15 * time.Second); b.hasKey(buckets.Wants(), hkey); time.Sleep(100 * time.Millisecond) {
if time.Now().After(deadline) {
t.Fatal("the stale \"keep\" of the first hold kept the second one held")
}
}
}
// bedStore is the runtime's states over the bed's buckets, as the seat's verbs read and write them: the assigner's own by
// their local names, another module's `<module>.<state>`.
type bedStore struct{ b *bed }
@@ -77,7 +77,6 @@ func main() {
var readBuckets = []string{
buckets.Grants,
buckets.Have,
buckets.Provisions,
}
// readFamilies are the states kept one bucket per node the assigner reads, each named in its manifest's reads: every
@@ -90,7 +89,7 @@ func ownBuckets() []string {
return []string{buckets.Asks(), buckets.Differences(),
buckets.Puts(), buckets.Wants(), plc.AssignmentsBucket(), plc.ProposalsBucket(),
plc.CandidatesBucket(), plc.PulsesBucket(), assign.FindingsBucket(), assign.PoliciesBucket(), plc.SeatHoldersBucket(),
engineReadsBucket(), retiredBucket(), copiesBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.Inputs(), buckets.CompositionRuns(), pinsBucket()}
engineReadsBucket(), retiredBucket(), copiesBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.Inputs(), buckets.CompositionRuns()}
}
// engineReadsState is the assigner's state of what each node's engine states it reads (ADR 0618 §3), one key `<node>`, as
@@ -137,7 +136,6 @@ var keptStates = []announce.State{
{Name: buckets.PutWaitsState, Readable: true}, // the put waits said (putwaits.go, design 47 G A-211)
{Name: buckets.InputsState, Readable: true}, // read by the tool process alone: the verb input and assigner_inputs (design 47 G §354)
{Name: buckets.CompositionRunsState, Readable: true}, // the composition runs, read by the process and its tools (design 47 G §719, A-468c)
{Name: pinsState, Readable: true}, // the records pinning each assignment graph revision (design 47 G §579)
}
// kvSet is every bucket the assigner opened, by name.
@@ -166,12 +164,6 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
continue
}
if name == buckets.Provisions && err != nil {
// The seat mesh-provisioner's state, with its own delivery (ADR 0570 §4): bound once the bus says it was made;
// until then no want waits for a provision, as before.
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
continue
}
if name == assign.PoliciesBucket() && err != nil {
// The seat mesh-wants' state, new with it (ADR 0565 §7): bound once the bus says it was made; until then no
// person's policy is read, so the controller's beside each composition holds, as before.
@@ -195,11 +187,6 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
logf("the bucket %s cannot be opened (%v): it is read by the tool process alone", name, err)
continue
}
if name == pinsBucket() && err != nil {
// New with the pins (design 47 G §579): without it no pin is kept and no revision is removed.
logf("the bucket %s cannot be opened (%v): no pin is kept and no revision removed until it starts again", name, err)
continue
}
if name == buckets.Waits() && err != nil {
// New with the wait on an outside cause (design 47 G §706): without it no wait is made, and a failing check is judged
// as before.
@@ -234,8 +221,8 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
// states and those new with a delivery of their own, bound once made, and the two that choose the grants' source.
func startsWithout(name string) bool {
switch name {
case plc.SeatHoldersBucket(), buckets.Provisions, assign.PoliciesBucket(), assign.FindingsBucket(),
buckets.Puts(), engineReadsBucket(), retiredBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.Inputs(), pinsBucket():
case plc.SeatHoldersBucket(), assign.PoliciesBucket(), assign.FindingsBucket(),
buckets.Puts(), engineReadsBucket(), retiredBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.Inputs():
return true
}
return slices.Contains(seatBuckets(), name) || name == buckets.Grants
@@ -291,9 +278,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// The inputs of the composition steps it asks a build seat to run: the seat mesh-wants' verb input is answered here in chunks
// of the core connection, of any size (design 47 G §354, §712, A-237).
inputs := inputsOver(kvs[buckets.Inputs()], js, account.Module, logger.Printf)
// The seat list (seatlist.go), followed in follow over the connection and the hub: the verb input's callers are checked
// against the seat's definition it holds (A-186).
gate := seatlist.NewPart("assigner", seatListAgainCallers)
// The seat list (seatlist.go), followed in follow over the connection and the hub.
gate := seatlist.NewPart("assigner")
if inputs != nil {
inputs.dropAtStart() // the objects of keys another part deleted (A-465)
stopInput, err := said.AnswerSeat(meshwants.Seat, seatlist.Definition(gate.Reader, meshwants.Seat), map[string]process.SeatHandler{meshwants.VerbInput: inputs.verb(conn, buscalls.SeatAckWithin)})
@@ -376,7 +362,6 @@ func watched() []string {
buckets.Differences(),
buckets.Wants(),
assign.PoliciesBucket(),
buckets.Provisions,
plc.SeatHoldersBucket(),
engineReadsBucket(),
writtenBucket(),
@@ -389,8 +374,6 @@ func watched() []string {
// each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends.
func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, kvs kvSet, say func(oc.Condition) error, findings assign.Findings, bus caller, iv *catalogViews, inputs *inputKeeper, gate *seatlist.Part, logger *log.Logger, getenv func(string) string) error {
w := newWorld()
// The callers' seat entries are resolved from the assigner's own seat-holders as it last read them (A-298).
gate.Holders(seatlist.HoldersIn(func() (map[string][]byte, error) { return w.read(plc.SeatHoldersBucket()), nil }))
dirty := make(chan struct{}, 1)
wake := func() {
select {
@@ -572,7 +555,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
}
}
// And the two that choose where grants are read from, with the same delivery as the grants part that writes one.
for _, name := range []string{buckets.Grants, assign.PoliciesBucket(), buckets.Provisions, plc.SeatHoldersBucket()} {
for _, name := range []string{buckets.Grants, assign.PoliciesBucket(), plc.SeatHoldersBucket()} {
if kvs[name] == nil && js != nil {
name := name
process.AwaitBucket(ctx, js, nc, name, "until then the bucket is not read",
@@ -622,7 +605,6 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
logger.Printf("%s is not heard, so a wait on an outside cause is judged again only when its check changes: %v", subject, err)
}
}
p.outside.readStart()
}
if js != nil {
p.publishWants = wantsPublisher(ctx, js, thisNode)
@@ -636,20 +618,10 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
// A held revert's running entry and its want's, asked of the seat mesh-catalog's verb entry (design 47 G §710, A-217).
p.entries = meshcatalogclient.NewEntries(iv.h.call)
p.entries.Changed = func(meshcatalogclient.EntryRow) { wake() }
// The pins of the revisions (pins.go, design 47 G §579): the gate's pinned, freed, heads-replaced and pins-stated
// on the hub, registered before it starts.
if kv := kvs[pinsBucket()]; kv != nil {
k := &pinKeeper{ctx: ctx, st: kvEnds{ctx, kv}, objects: objects, inUse: inUseOf(w), findings: findings, now: time.Now, logf: logger.Printf}
if err := k.made(); err != nil {
logger.Printf("the state %s is not marked made, so no revision is removed until it is: %v", pinsState, err)
}
if err := hearPins(hub, k); err != nil {
logger.Printf("a pin is not heard, so no revision is removed until the assigner starts again: %v", err)
} else {
p.graph.pins = k
k.watchUnpins(kv)
}
}
// The revisions that are not current are removed when one ceases to be current and once now (revisions.go,
// design 47 H §764, H-A-86).
p.graph.revisions = &revisionRemover{ctx: ctx, objects: objects, inUse: inUseOf(w), logf: logger.Printf}
p.graph.revisions.judge()
}
}
// The node-engine seat's manifest-reads-changed, for one node (ADR 0618 §4): that node is asked again, with no timer.
@@ -688,8 +660,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
if bus != nil && js != nil && inputs != nil && p.publishWants != nil && runs != nil {
run := &compositionRun{runs: runs, inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept,
trunk: trunkOf(iv, w.trunkName), running: w.runningOf, away: func(machine string) bool { return w.awayNow()[machine] }, put: runAskPublisher(js), ask: bus.CallFile, now: time.Now, logf: logger.Printf}
steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, resume: run.resume, ended: runs.stepEnded, now: time.Now, logf: logger.Printf,
asker: stepAskerOf(seatlist.Definition(gate.Reader, meshwants.Seat))}
steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, resume: run.resume, ended: runs.stepEnded, now: time.Now, logf: logger.Printf}
move := steps.move(ctx) // the readers a rerun carried (§727, A-470d)
runs.moved.Store(&move)
// The state's watch wakes the steps waiting on a run and hears a bound's expiry; the runs standing at the start are taken up
@@ -125,7 +125,7 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
}
kvs[buckets.Puts()].(*watchKV).closed = true
kvs[buckets.Wants()].(*watchKV).wrote = wrote
go follow(ctx, nil, nil, kvs, func(oc.Condition) error { return nil }, assign.Findings{}, nil, nil, nil, seatlist.NewPart("assigner", seatListAgainCallers), log.New(io.Discard, "", 0), func(string) string { return "" })
go follow(ctx, nil, nil, kvs, func(oc.Condition) error { return nil }, assign.Findings{}, nil, nil, nil, seatlist.NewPart("assigner"), log.New(io.Discard, "", 0), func(string) string { return "" })
select {
case key := <-wrote:
if key != "want.ace.node" {
@@ -76,9 +76,6 @@ type outsideActs struct {
cleared map[string]bool
// sigs is each check's last signature, by node.module.
sigs map[string]string
// own is the machines whose own network is open; ownRead says the read at the start was answered.
own map[string]bool
ownRead bool
// causes is, by condition key, the nodes whose waits stand on it, as the last pass kept them.
causes map[string][]string
// started says the first pass judged every wait again.
@@ -93,7 +90,7 @@ func newOutsideActs(call func(string, any) (json.RawMessage, error), wake func()
settle = outside.DefaultSettle
}
return &outsideActs{call: call, wake: wake, logf: logf, bound: bound, settle: settle, reading: map[string]bool{}, answered: map[string][]outside.Cond{},
dirty: map[string]bool{}, cleared: map[string]bool{}, sigs: map[string]string{}, own: map[string]bool{}, causes: map[string][]string{}}
dirty: map[string]bool{}, cleared: map[string]bool{}, sigs: map[string]string{}, causes: map[string][]string{}}
}
// outsideSettings are the bounds of the wait, the settings outside-wait-minutes and outside-settle-seconds (design 47 G §633, §706),
@@ -135,31 +132,8 @@ func settingFile(path string) (int, error) {
return n, nil
}
// readStart reads the open conditions of every machine once at the start, for the machines whose own network is held.
func (o *outsideActs) readStart() {
go func() {
answer, err := o.call(conditionsVerb, map[string]any{"scope": "machine"})
if err != nil {
o.logf("the open conditions are not read at the start (%v): no machine is held for its own network until its event is heard", err)
return
}
conds, err := outside.ParseConditions(answer)
if err != nil {
o.logf("%v", err)
return
}
o.mu.Lock()
for m := range outside.OwnNetwork(conds) {
o.own[m] = true
}
o.ownRead = true
o.mu.Unlock()
o.wake()
}()
}
// Heard is one of the seat's events condition-raised, -changed or -cleared: it says which machines to judge again and keeps the
// machines whose own network is open; nothing is kept of the condition.
// Heard is one of the seat's events condition-raised, -changed or -cleared: it says which machines to judge again; nothing is kept
// of the condition (the network hold ends, design 47 H §750, H-A-55).
func (o *outsideActs) Heard(subject string, body []byte) {
var e mcseat.ConditionEvent
if err := json.Unmarshal(body, &e); err != nil || e.Key == "" {
@@ -168,9 +142,6 @@ func (o *outsideActs) Heard(subject string, body []byte) {
}
cleared := strings.HasSuffix(subject, "."+mcseat.EventConditionCleared)
o.mu.Lock()
if p := strings.Split(e.Key, "."); len(p) == 3 && p[0] == "machine" && p[2] == "network" {
o.own[p[1]] = !cleared
}
if cleared {
for _, node := range o.causes[e.Key] {
o.dirty[node] = true
@@ -186,16 +157,6 @@ func (o *outsideActs) Heard(subject string, body []byte) {
o.wake()
}
// Held says whether machine's own network is open: no want moves to it.
func (o *outsideActs) Held(machine string) bool {
if o == nil {
return false
}
o.mu.Lock()
defer o.mu.Unlock()
return o.own[machine]
}
// take is the conditions read for machine, consumed (one answer is one judging), or false while the kept call is unanswered,
// which it starts when none is in flight.
func (o *outsideActs) take(machine string) ([]outside.Cond, bool) {
@@ -237,23 +237,6 @@ func TestTheWaitOnAnOutsideCause(t *testing.T) {
})
}
func TestAMachineWhoseOwnNetworkIsOpenGetsNoWant(t *testing.T) {
m := assign.Module{Name: "nats", FP: fpTest, Built: true, Nodes: []string{"ace"}, MachineHeld: func(node string) bool { return node == "ace" }}
ok, rule, why := assign.Holds(m, "ace", assign.Nodes{Online: map[string]bool{"ace": true}})
if ok || rule == 0 || !strings.Contains(why, "machine.ace.network") {
t.Errorf("holds = %v %d %q", ok, rule, why)
}
o := newOutsideActs(nil, func() {}, func(string, ...any) {}, 0, 0)
o.Heard("mesh.seat.mesh-conditions.event.condition-raised", []byte(`{"key":"machine.ace.network","subject":{"id":"ace","scope":"machine","machine":"ace"}}`))
if !o.Held("ace") || o.Held("g14") {
t.Error("the own network condition's event does not hold the machine")
}
o.Heard("mesh.seat.mesh-conditions.event.condition-cleared", []byte(`{"key":"machine.ace.network","subject":{"id":"ace","scope":"machine","machine":"ace"}}`))
if o.Held("ace") {
t.Error("the machine is held after its condition cleared")
}
}
func TestTheToolsReadAndReleaseAnOutsideWait(t *testing.T) {
r := newOutsideRig(t)
r.cond("machine.hub.unreachable", "ace")
@@ -5,7 +5,6 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"mesh-delivery-assigner/internal/buckets"
"slices"
@@ -18,16 +17,10 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
"git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay"
"git.novox.be/novox/mesh-sdk/go/motion"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"sync"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient"
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
"git.novox.be/novox/mesh-sdk/go/catalogue"
@@ -64,19 +57,12 @@ type passer struct {
findings assign.Findings
// calls is the wait of the kept manifest-reads calls (enginereads.ReadsCalls, ADR 0668 §3), made at the first pass.
calls *enginereads.ReadsCalls
// failedProvisions are the failed provisions whose finding this process wrote (ADR 0570 §4).
failedProvisions map[string]bool
// missing are the required provisions with no key this process pulsed, and missingSaid those whose finding it wrote.
missing, missingSaid map[string]bool
// unplaced are the nodes whose unplaced finding this process wrote.
unplaced map[string]bool
// answeredAt is, by the id of a backup ask, how many values the have had when its answer was written into the want.
answeredAt map[string]uint64
// now is the clock; nil is time.Now.
now func() time.Time
// neither is, by node and module, each module on neither list whose condition this process raised (issue 1945). Nil until the
// first pass after a start, which clears what a previous process may have raised (startClears).
neither map[string]neitherOne
// ends are the pipeline ends the wants follow (pipelines.go), once the first registrations are heard; until then, and nil, the trunk-only want.
ends *pipelineEnds
// acts are the puts' acts (ADR 0546); nil: the puts are not taken further (no bus to call, or no puts bucket).
@@ -254,14 +240,13 @@ func (p *passer) nodeWants(ctx context.Context) {
}
}
// wants writes, for every module the trunk holds a fingerprint of, each node's want by the six rules (ADR 0429 §6), and
// the hold of a fingerprint whose health check failed. The assigner compares its wants with nothing the controller
// sends: the differences bucket is kept read-only with its history (ADR 0539 §8).
// wants writes, for every module assigned or wanted, each node's want naming the newest version released on the branch its
// placement names whose pipeline passed, written tier by tier (design 47 H §750, §751). The want is held for nothing else: the
// rules of ADR 0429 §6, the network hold and A-428's exceptions end (H-A-55); a put owns its node's want, and the operator's yes
// on a module of policy record is the node-engine's have and the asker's (yeses).
func (p *passer) wants(ctx context.Context) {
assigned := p.w.assigned()
assignedBranches := p.w.placedBranches()
// Every module of every branch's catalog half, for the parts that ask which exist; a node's entries come from the half of the
// branch its placement names (design 47 G §14, §184).
trunk := p.w.trunk()
views := map[string]*branchView{}
viewOf := func(branch string) *branchView {
@@ -282,9 +267,6 @@ func (p *passer) wants(ctx context.Context) {
}
wanted := p.w.wantsOf()
chosen, _ := assign.DecodePolicies(p.w.read(assign.PoliciesBucket()))
provisionsOf := p.provisionsOf()
missing := map[string]bool{}
defer func() { p.provisionsMissing(ctx, missing) }()
defer p.unplacedFindings()
defer func() { p.seatHolders(ctx, viewOf(p.w.trunkName()).manifests) }()
p.clearStaleCalls()
@@ -298,397 +280,32 @@ func (p *passer) wants(ctx context.Context) {
for name := range wanted {
names[name] = true
}
neither := map[string]neitherOne{}
// The first pass after a start remembers nothing a previous process said: a neither or held condition it raised would stay
// raised for ever once the module has its want or is no longer held. That pass clears, once, the neither condition of every
// assigned module not on neither list and the held condition of every module not held; a clear of a condition that is not open is
// quiet at the router.
first := p.neither == nil
stale := map[string]neitherOne{}
left := map[string]string{}
defer func() { p.sayNeither(neither, stale, left) }()
// Where grants are read from, once for the pass: the grants part's (issue 774, ADR 0571 clause 9).
grants := p.w.grantsBucket()
for name := range names {
groups := p.groups(name, nodesOf[name], assignedBranches[name])
for gi, g := range groups {
func() {
bv := viewOf(g.branch)
trunk, manifests, nodes, moduleGraph, served, offering := bv.trunk, bv.manifests, bv.nodes, bv.graph, bv.served, bv.offering
fp := trunk[name].Fp2
name := name
nodesOf := map[string][]string{name: g.nodes}
wantedOf := map[string]map[string]mwc.Want{name: wantedHere(wanted[name], g.nodes, groups, gi == 0)}
// A want follows a passed pipeline: a module whose fingerprint no pipeline passed at on the branch its placements
// chose, and whose wants do not name it already, gets none (design 47 G §29, §35).
var branches []string
if p.ends.following() {
branches = []string{g.branch}
}
if fp != "" && !p.ends.mayWant(name, fp, branches, wantedOf[name]) {
if !runnerFirstStart(name, manifests, p.w.runningNow()) && !manifests[name].ClaimsSeat(meshcatalog.Seat) {
none := assign.Outcome{Rules: noPipelineRules(g.nodes, name, fp, branches)}
p.sayNoWant(name, g.nodes, wantedOf[name], none)
p.sayWhy(ctx, name, g.nodes, wantWhy{Branch: g.branch, Fp: fp, Step: whyNoPipeline}, wantedOf[name], &none, nil)
p.nameHeldBuilds(ctx, name, fp, g.nodes, nodes, wantedOf[name], p.namedBuild(name, fp, trunk[name].FpSrc))
return
}
// The first start of the pipeline runner (design 47 G A-428): no runner runs to pass its pipeline, so its want is
// written from the trunk's entry by the other rules, and said once. So is the holder of mesh-catalog's: every
// pipeline runs on a revision it took in, and a revision naming a newer one is taken in only by that one (§16,
// G-5's whole test of indexer and assigner: the new indexer is wanted and granted, starts, and takes it in).
if key := "no-pipeline-start " + name + "@" + fp; !p.skipped[key] {
if p.skipped == nil {
p.skipped = map[string]bool{}
}
p.skipped[key] = true
if manifests[name].ClaimsSeat(meshcatalog.Seat) {
p.logf("%s at %s is wanted with no passed pipeline: it claims the seat %s, every pipeline runs on a revision it took in, and a revision naming it is taken in only by it (design 47 G §16)", name, shortHex(fp), meshcatalog.Seat)
} else {
p.logf("%s at %s is wanted with no passed pipeline: it claims the seat %s and no module holding that seat runs on any node, so none can pass one (design 47 G A-428)", name, shortHex(fp), mpr.Seat)
}
}
}
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Needs: trunk[name].Needs, Chosen: chosenOf(chosen, name), Provisions: provisionsOf(name), Answering: answering(manifests, nodes), Offered: offered(manifests), Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name], FormNeeds: p.w.buildNeeds(name, fp),
Self: func(node string) bool { return placed[node] },
Composed: func(node, fp string) (assign.ComposedDigest, bool) {
return p.w.composed(node, name, fp) // a placed node's own: the digest its own have names (ADR 0571 §6)
},
GrantedAt: func(fp string) bool {
key, err := grantsstate.PartGrantKey(name, fp)
return err == nil && p.w.has(grants, key)
},
Restored: func(node string) bool {
key, err := asks.RestoreAskKey(node, name, fp)
return err == nil && p.answeredWith(key, assign.OptionRestore)
},
Allowed: func(node, digest string) bool {
key, err := asks.RecordAskKey(node, name, digest)
return err == nil && p.answeredWith(key, assign.OptionYes)
}}
if fp != "" {
held, _ := mwc.HeldKey(name, fp)
grant, _ := grantsstate.PartGrantKey(name, fp)
hold, _ := asks.HoldAskKey(name, fp)
noPipeline := runnerFirstStart(name, manifests, p.w.runningNow()) || manifests[name].ClaimsSeat(meshcatalog.Seat)
m.KeptBuilds = p.namedBuild(name, m.FP, trunk[name].FpSrc, noPipeline)
m.Released = p.answeredWith(hold, "release")
m.Kept = p.answeredWith(hold, "keep")
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(buckets.Wants(), held) && !m.Released, p.w.has(grants, grant)
m.BuildFailed, m.BuildError = p.w.buildStanding(name, fp)
// A build whose version is released counts as done for rule 1; its build.ended counts only for a build not
// released (design 47 G A-429), so a restart that lost the end loses nothing of a released build.
if releasedBuild(m.KeptBuilds, fp) {
m.Built, m.BuildFailed, m.BuildError = true, false, ""
}
if raw := p.w.value(grants, grant); raw != nil {
if g, err := grantsstate.DecodeGrant(raw); err == nil {
m.GrantedOn = func(node string) bool { return assign.GrantNamesNode(g, node, name) }
}
}
if m.Released {
p.clearHeld(name, fp)
}
}
if served != nil {
m.Invokes, m.Served = keptInvokes(moduleGraph, name), served
}
m.Offering = offering
last := map[string]string{}
for _, node := range m.Nodes {
last[node] = p.w.lastHealthy(node, name, fp)
}
if p.outside != nil {
m.MachineHeld = p.outside.Held
nodes = p.outsideReview(ctx, name, fp, m.Nodes, nodes, manifests[name].ClaimsSeat(mcseat.Seat), p.movedTo(wanted, nodes))
}
out := assign.Wants(m, nodes, wantedOf[name], last)
p.sayWhy(ctx, name, m.Nodes, wantWhy{Branch: g.branch, Fp: fp, Step: whyRules}, wantedOf[name], &out, &m)
for _, node := range m.Nodes {
for _, prov := range m.MissingProvisions(node) {
missing[node+"."+name+"."+prov] = true
}
}
p.yeses(ctx, name, m, wantedOf[name], &out)
gatherNeither(neither, name, out)
leftNeither(left, m, wantedOf[name], out)
p.sayNoWant(name, m.Nodes, wantedOf[name], out)
p.sayUnreached(name, m.Nodes, out)
p.sayNodeFailed(name, fp, m.Nodes, nodes, out, first)
if first {
startClears(stale, m, out)
if fp != "" && !m.Held && out.Held == nil && !out.Lifted {
p.clearHeld(name, fp)
}
}
if out.Held != nil {
key, err := mwc.HeldKey(name, fp)
if err == nil {
held := *out.Held
p.put(ctx, buckets.Wants(), key, func() ([]byte, error) { return mwc.EncodeHeld(held) })
}
}
if out.Lifted {
p.lift(ctx, name, fp)
}
if out.Held != nil || (m.Held && !out.Lifted) {
p.holdAsk(ctx, name, fp, out.Held)
p.heldCondition(ctx, name, fp, out.Held, len(out.Restore) > 0)
}
for _, node := range out.Restore {
p.restoreAsk(ctx, name, node, fp)
}
for _, need := range out.Record {
p.recordAsk(ctx, name, fp, need)
}
for node, want := range out.Wants {
p.putWant(ctx, node, name, want)
}
if len(out.Wants) > 0 {
p.ends.wrote(name, fp, branches)
}
}()
}
}
}
// movedTo says whether the delivery moved owner to node: owner's want there names a fingerprint its have there does not yet run
// healthy (design 47 G §706, *What counts as an outside cause*: a rewritten file of a module the delivery moved is that module's).
func (p *passer) movedTo(wanted map[string]map[string]mwc.Want, n assign.Nodes) func(owner, node string) bool {
return func(owner, node string) bool {
w, ok := wanted[owner][node]
return ok && !w.Removed && !n.RunsHealthy(node, owner, w.Fp)
}
}
// sayNoWant says in the journal why a node of module gets no want after this pass, from the rule the outcome names for it
// (issue 1945: a new assignment whose trunk build was not done was held back with nothing said), once per reason, and says
// when the node has a want again. A node with a live want, written before and kept or written in this pass, has one.
func (p *passer) sayNoWant(module string, nodes []string, before map[string]mwc.Want, out assign.Outcome) {
if p.noWant == nil {
p.noWant = map[string]string{}
}
assigned := map[string]bool{}
for _, node := range nodes {
assigned[node] = true
key := node + "." + module
w, written := out.Wants[node]
if !written {
w, written = before[node]
}
reason := ""
if h, held := out.Rules[node]; held && (!written || w.Removed) {
reason = h.Reason()
}
last, said := p.noWant[key]
switch {
case reason != "" && reason != last:
p.logf("%s on %s gets no want: %s", module, node, reason)
p.noWant[key] = reason
case reason == "" && said && written && !w.Removed:
p.logf("%s on %s has a want", module, node)
delete(p.noWant, key)
}
}
for key := range p.noWant {
if node, m, _ := strings.Cut(key, "."); m == module && !assigned[node] {
delete(p.noWant, key) // no longer assigned there: its removed want says so
}
}
}
// sayUnreached says in the journal, once per words, that a module is wanted on a node and may not be reached there, since
// no provider of what it requires runs healthy (rule 5, which holds no want: issue 1960), and once when it is reached again.
func (p *passer) sayUnreached(module string, nodes []string, out assign.Outcome) {
if p.unreached == nil {
p.unreached = map[string]string{}
}
for _, node := range nodes {
key := node + "." + module
why, last := out.Unreached[node], p.unreached[key]
switch {
case why != "" && why != last:
p.logf("%s on %s is wanted and may not be reached: %s", module, node, why)
p.unreached[key] = why
case why == "" && last != "":
p.logf("%s on %s is reached: its providers run healthy", module, node)
delete(p.unreached, key)
}
}
}
// sayNodeFailed raises, once per node and words, the condition of a failed health check of a module's newest fingerprint fp
// that is the node's own, since fp runs healthy elsewhere and is not held (ADR 0557 §1, issue 1496), and says it in the journal;
// it clears the condition once when the node no longer fails alone, or is no longer one of the module's nodes. The first pass
// after a start remembers nothing a previous process raised: it clears, for every module, the condition of every node the
// presence bucket knows that does not fail alone, since a node that recovered while no process watched leaves no failed have to
// find it by (a clear of a condition not open is quiet at the router). A condition the router did not take is said again at the next pass.
func (p *passer) sayNodeFailed(module, fp string, nodes []string, known assign.Nodes, out assign.Outcome, first bool) {
if p.nodeFailed == nil {
p.nodeFailed = map[string]string{}
}
look := map[string]bool{}
for _, node := range nodes {
look[node] = true
}
for key := range p.nodeFailed {
if node, m, _ := strings.Cut(key, "."); m == module {
look[node] = true
}
}
if first {
for node := range known.Online {
look[node] = true
}
}
for _, node := range slices.Sorted(maps.Keys(look)) {
key := node + "." + module
f, failing := out.NodeFailed[node]
last, said := p.nodeFailed[key]
switch {
case failing && f.Words+"|"+strings.Join(f.Others, ",") != last:
p.logf("%s on %s: the health check at %s failed here (%s), while it runs healthy on %s: the failure is %s's, so nothing "+
"is held or set back", module, node, fp, f.Words, strings.Join(f.Others, ", "), node)
if p.say != nil {
if err := p.say(assign.NodeFailedCondition(node, module, fp, f, true)); err != nil {
p.logf("the condition of %s failing on %s alone could not be said, and is said at the next pass: %v", module, node, err)
continue
}
bv := viewOf(g.branch)
wantedOf := wantedHere(wanted[name], g.nodes, groups, gi == 0)
target, ok := p.ends.target(name, g.branch)
m := assign.Module{Name: name, Manifest: bv.manifests[name], Chosen: chosenOf(chosen, name), Nodes: g.nodes, Puts: puts[name],
Self: func(node string) bool { return placed[node] },
Target: func(string) (assign.Target, bool) { return target, ok },
Allowed: func(node, digest string) bool {
key, err := asks.RecordAskKey(node, name, digest)
return err == nil && p.answeredWith(key, assign.OptionYes)
}}
if p.outside != nil && ok {
p.outsideReview(ctx, name, target.Fp, m.Nodes, bv.nodes, bv.manifests[name].ClaimsSeat(mcseat.Seat), p.movedTo(wanted, bv.nodes))
}
p.nodeFailed[key] = f.Words + "|" + strings.Join(f.Others, ",")
case !failing && (said || first):
if said {
p.logf("%s on %s no longer fails its health check alone", module, node)
out := assign.Wants(m, wantedOf)
p.sayWhy(ctx, name, m.Nodes, wantWhy{Branch: g.branch, Fp: target.Fp, Version: target.Version, Pipeline: target.Pipeline, Step: whyStep(ok)}, wantedOf, &out)
if ok {
p.yeses(ctx, name, m, wantedOf, &out)
}
if p.say != nil {
if err := p.say(assign.NodeFailedCondition(node, module, fp, assign.NodeFailure{}, false)); err != nil && said {
p.logf("the condition of %s failing on %s alone could not be cleared, and is cleared at the next pass: %v", module, node, err)
continue
}
}
delete(p.nodeFailed, key)
}
}
}
// neitherOne is a module assigned to a node that holds no want for it after a pass, and the rule holding its first want back.
type neitherOne struct {
node, module string
holding assign.Holding
}
// gatherNeither adds the nodes on which module is on neither list in its outcome, keyed by node and module.
func gatherNeither(into map[string]neitherOne, module string, out assign.Outcome) {
for node, h := range out.Neither {
into[node+"."+module] = neitherOne{node: node, module: module, holding: h}
}
}
// leftNeither says, by node and module, for every node of m not on neither list after the pass, why: "" when the node has a live
// want for it (written before and kept, or written in this pass), else why it has none. sayNeither clears a neither condition in
// these words, so a clear never says a want the node does not hold (issue 1945: "nats on novox has a want again" while the bus bar
// left it with none).
func leftNeither(into map[string]string, m assign.Module, before map[string]mwc.Want, out assign.Outcome) {
for _, node := range m.Nodes {
if _, on := out.Neither[node]; on {
continue
}
w, written := out.Wants[node]
if !written {
w, written = before[node]
}
why := ""
h, ruled := out.Rules[node]
switch {
case written && !w.Removed:
case (m.Held && !out.Lifted) || out.Held != nil:
why = "it is held, and its held condition says so"
case slices.ContainsFunc(out.Record, func(r assign.RecordNeed) bool { return r.Node == node }):
why = "its move waits for the operator's yes on its record ask"
case putOn(m, node):
why = "its put waits on its own steps and the operator's answer"
case m.FP == "":
why = "the trunk has no build of it"
case ruled:
why = h.Reason()
default:
why = "the node does not run it"
}
into[node+"."+m.Name] = why
}
}
// putOn says whether an operator's put on node owns m's want there: one still on its steps (ADR 0546), so the rules write none
// meanwhile. An ended put (done, refused, released) stays in the bucket and owns nothing.
func putOn(m assign.Module, node string) bool {
p, put := m.Puts[node]
_, owns := assign.PutOwns(p)
return put && owns
}
// startClears adds, for the first pass after a start, every node of m that is not on neither list after the pass: a neither
// condition a previous process raised for it is cleared. That covers a node with a want and one with none that is not on neither
// list, such as a held module (issue 1945: the hold is said on its own, so a neither raised for it before the restart would stay up
// for ever) or a move waiting for its record ask. Clearing a condition that is not open is quiet at the router.
func startClears(into map[string]neitherOne, m assign.Module, out assign.Outcome) {
for _, node := range m.Nodes {
if _, on := out.Neither[node]; !on {
into[node+"."+m.Name] = neitherOne{node: node, module: m.Name}
}
}
}
// sayNeither raises, once, the condition of every module on neither list in this pass and writes it to the journal, and clears,
// once, the condition of each one said before that has a want again (issue 1945). A say that fails is made again at the next pass.
// stale is, on the first pass after a start, the modules whose condition a previous process may have raised and that have a want:
// each is cleared once.
// left is, by node and module, why each assigned node not on neither list after this pass has no want ("" when it has one); a key
// it does not hold is no longer assigned.
func (p *passer) sayNeither(now, stale map[string]neitherOne, left map[string]string) {
if p.neither == nil {
p.neither = map[string]neitherOne{}
for key, one := range stale {
if _, on := now[key]; !on {
p.neither[key] = one
for node, want := range out.Wants {
p.putWant(ctx, node, name, want)
}
}
}
for key, one := range now {
if _, said := p.neither[key]; said {
continue
}
c := assign.NeitherCondition(one.node, one.module, one.holding, true)
if p.say != nil {
if err := p.say(c); err != nil {
p.logf("the condition of %s on neither list on %s could not be said, and is said at the next pass: %v", one.module, one.node, err)
continue
}
}
p.logf("%s is assigned to %s and on neither list: no want, and not in the node's own composition; what holds its first want back: %s",
one.module, one.node, one.holding.Reason())
p.neither[key] = one
}
for key, one := range p.neither {
if _, still := now[key]; still {
continue
}
why, assigned := left[key]
if !assigned {
why = "it is no longer assigned there"
}
if p.say != nil {
if err := p.say(assign.NeitherLeftCondition(one.node, one.module, why)); err != nil {
p.logf("the condition of %s on neither list on %s could not be cleared, and is cleared at the next pass: %v", one.module, one.node, err)
continue
}
}
if why == "" {
p.logf("%s on %s has a want again", one.module, one.node)
} else {
p.logf("%s on %s is no longer on neither list and has no want: %s", one.module, one.node, why)
}
delete(p.neither, key)
}
}
// put writes one key of the assigner's own bucket; a write that fails is said and made again at the next pass.
@@ -736,59 +353,6 @@ func (p *passer) putWant(ctx context.Context, node, module string, want mwc.Want
}
}
// holdAsk writes, once, the ask about a held fingerprint (ADR 0528 clause 6, ADR 0437 clause 10): the first ask that needs
// no proof of who answered, which the view may answer and the teller fills. In release 5 an answer changes nothing on a
// node (wants are only compared, ADR 0429 §7), and the options say so.
func (p *passer) holdAsk(ctx context.Context, module, fp string, fresh *mwc.Held) {
key, err := asks.HoldAskKey(module, fp)
if err != nil || p.w.has(buckets.Asks(), key) {
return
}
// The asking is the hold's time in seconds: a hold of the same fingerprint written again after a lift is another asking, under
// an id of its own, also after a restart, since the router drops a second ask under an id it keeps (issue 1906).
now := p.clock()
asking := int(now.Unix())
h := p.heldHold(module, fp, fresh)
a := assign.HoldAsk(key, module, fp, h.Node, h.Words, asking, now)
p.put(ctx, buckets.Asks(), key, func() ([]byte, error) { return assign.EncodeAsk(key, a) })
}
// lift takes away a hold the assigner lifted by itself, since every node that runs the module runs the held fingerprint healthy
// (issue 1906): the held record and its ask (the teller then takes the ask back from the operator), and the held condition is
// cleared saying why, once. A hold of the same fingerprint written later is said and asked again.
func (p *passer) lift(ctx context.Context, module, fp string) {
held, err := mwc.HeldKey(module, fp)
if err != nil {
return
}
ask, err := asks.HoldAskKey(module, fp)
if err != nil {
return
}
c := assign.LiftedCondition(module, fp)
if !p.said[c.Key+".lifted"] {
if p.say != nil {
if err := p.say(c); err != nil {
// The hold stays written, so the next pass lifts it again and says so then.
p.logf("the condition of the lifted hold of %s could not be cleared, and is cleared at the next pass: %v", module, err)
return
}
}
p.logf("%s", assign.LiftedWords(module, fp))
p.said[c.Key+".lifted"] = true
delete(p.said, c.Key+".raised") // a hold written again is said again
}
p.remove(ctx, buckets.Wants(), held)
p.remove(ctx, buckets.Asks(), ask)
// The restore asks of that fingerprint, on every node, go with the hold: every node runs it healthy, so no node waits to be
// set back from it (issue 1906, note 28).
for key := range p.w.read(buckets.Asks()) {
if k, err := asks.ParseAnyAskKey(key); err == nil && k.Kind == asks.AskKindRestore && k.Module == module && k.FP == fp {
p.remove(ctx, buckets.Asks(), key)
}
}
}
// staleAsk says why the hold, restore or record ask at key (read as the SDK reads it) is about something the trunk has left
// behind, or "" when it is not: a module no longer on the trunk, a fingerprint that is no longer the module's trunk fingerprint,
// or, for a record ask, a composition the module's node no longer waits on. A backup ask, and a revert ask (design 47 G §710), is stale when the module's want on its
@@ -894,29 +458,6 @@ func (p *passer) yesWritten(k asks.AskKey) bool {
return err == nil && w.Yes == k.Composed
}
// fingerprintsOf are the fingerprints a node may still want of module: the trunk's, the one its want names and the one its have
// names, each once.
func (p *passer) fingerprintsOf(node, module, trunkFP string) []string {
var out []string
add := func(fp string) {
if fp != "" && !slices.Contains(out, fp) {
out = append(out, fp)
}
}
add(trunkFP)
if key, err := mwc.WantKey(node, module); err == nil {
if w, err := mwc.DecodeWant(p.w.value(buckets.Wants(), key)); err == nil {
add(w.Fp)
}
}
if key, err := nodeengineclient.HaveKey(node, module); err == nil {
if h, err := nodeengineclient.DecodeHave(p.w.value(buckets.Have, key)); err == nil {
add(h.Fp)
}
}
return out
}
// staleAsks takes away, at every pass, each hold, restore, record and backup ask that staleAsk names (issue 1906, note 28): an ask
// outlived its fingerprint, and the teller takes its router ask back from the operator when the key goes. With the trunk not
// read at all (no module), nothing is taken away.
@@ -929,10 +470,6 @@ func (p *passer) staleAsks(ctx context.Context, trunk map[string]meshcatalog.Mod
// The condition is cleared first and the ask goes only once that was said: the ask is what brings this key back at the
// next pass, so a clear that failed once is made again, with no state of its own (issue 1906, note 29).
k, kerr := asks.ParseAnyAskKey(key)
held := kerr == nil && (k.Kind == asks.AskKindHold || k.Kind == asks.AskKindRestore)
if held && !p.clearHeld(k.Module, k.FP) {
continue
}
// A backup ask's condition is cleared with it, raised again if a move to that fingerprint is backed up again.
if kerr == nil && k.Kind == asks.AskKindBackup && p.say != nil {
if err := p.say(assign.BackupCondition(k.Node, k.Module, k.FP, "", false)); err != nil {
@@ -947,12 +484,6 @@ func (p *passer) staleAsks(ctx context.Context, trunk map[string]meshcatalog.Mod
}
p.logf("the ask %s is about a fingerprint or composition the trunk has left behind and is taken away: %s", key, why)
p.remove(ctx, buckets.Asks(), key)
// The held record stays: it is the memory that this fingerprint failed (ADR 0408 §4), and a revert returns the trunk to
// exactly it. Only the condition, which would else ask for an answer that can no longer be given, is cleared, and it is
// raised again if the trunk returns to the fingerprint.
if held {
delete(p.said, assign.HeldCondition(k.Module, k.FP, "", "", false, false).Key+".raised")
}
}
}
}
@@ -970,48 +501,6 @@ func (p *passer) remove(ctx context.Context, bucket, key string) {
p.w.put(bucket, key, nil)
}
// heldHold is the hold of fp as written: the node whose health check failed and its words.
func (p *passer) heldHold(module, fp string, fresh *mwc.Held) mwc.Held {
if fresh != nil {
return *fresh
}
key, err := mwc.HeldKey(module, fp)
if err != nil {
return mwc.Held{}
}
h, _ := mwc.DecodeHeld(p.w.read(buckets.Wants())[key])
return h
}
// heldCondition raises, once, the condition for a fingerprint the assigner holds (ADR 0429 §6, ADR 0539 §19): the module, the
// node whose health check failed and the check's words. A module holding data needs the operator's answer to its restore ask.
func (p *passer) heldCondition(ctx context.Context, module, fp string, fresh *mwc.Held, data bool) {
h := p.heldHold(module, fp, fresh)
c := assign.HeldCondition(module, fp, h.Node, h.Words, true, data)
if p.say == nil || p.said[c.Key+".raised"] {
return
}
if err := p.say(c); err != nil {
p.logf("the condition of the held %s could not be said, and is said at the next pass: %v", module, err)
return
}
p.said[c.Key+".raised"] = true
delete(p.said, c.Key+".lifted") // a hold raised again and lifted again is said lifted again
delete(p.said, c.Key+".cleared") // and released again is cleared again
}
// restoreAsk writes, once, the ask about a module holding data whose held fingerprint failed its health check on node.
func (p *passer) restoreAsk(ctx context.Context, module, node, fp string) {
key, err := asks.RestoreAskKey(node, module, fp)
if err != nil || p.w.has(buckets.Asks(), key) {
return
}
words := p.heldHold(module, fp, nil).Words
now := p.clock()
a := assign.RestoreAsk(key, node, module, fp, words, assign.AskingSeconds(now), now)
p.put(ctx, buckets.Asks(), key, func() ([]byte, error) { return assign.EncodeAsk(key, a) })
}
// answeredWith says whether the teller wrote an answer to the ask at key as it stands now and the option chosen is option. The
// answer is the operator's, given in the view or on the phone, and stays under its key (ADR 0539 §14, §16, §19); it counts only
// for the asking it answered (its warrant names that asking's id), so an answer to an ask taken away, or to an earlier hold of
@@ -1029,21 +518,6 @@ func (p *passer) answeredWith(key, option string) bool {
return id == ask.ID
}
// clearHeld clears, once, the condition of a hold the operator released.
// It says whether nothing is left to say: the clear was said now or before, or there is no router to say it to.
func (p *passer) clearHeld(module, fp string) bool {
c := assign.HeldCondition(module, fp, "", "", false, false)
if p.say == nil || p.said[c.Key+".cleared"] {
return true
}
if err := p.say(c); err != nil {
p.logf("the condition of the released %s could not be cleared, and is cleared at the next pass: %v", module, err)
return false
}
p.said[c.Key+".cleared"] = true
return true
}
// recordAsk writes, once, the ask about a move of a module of policy record that waits for the operator's yes.
func (p *passer) recordAsk(ctx context.Context, module, fp string, need assign.RecordNeed) {
key, err := asks.RecordAskKey(need.Node, module, need.Digest)
@@ -1091,7 +565,7 @@ func (p *passer) backups(ctx context.Context, reverting map[string]bool) {
continue
}
current, currentAsk, there := assign.DecodeAsk(akey, p.w.read(buckets.Asks())[akey])
if there != nil || assign.BackupAskDigest(current) != assign.ShortDigest(want.Composed) {
if there != nil {
after := -1
if there == nil {
after = current.Asking
@@ -1126,7 +600,7 @@ func (p *passer) backups(ctx context.Context, reverting map[string]bool) {
// backupAsk writes the backup ask at its asking and raises the condition of the failed backup, once.
func (p *passer) backupAsk(ctx context.Context, key, node, module string, want mwc.Want, words string, asking int) {
a := assign.BackupAsk(key, node, module, want.Fp, want.Composed, words, asking, time.Now())
a := assign.BackupAsk(key, node, module, want.Fp, words, asking, time.Now())
p.put(ctx, buckets.Asks(), key, func() ([]byte, error) { return assign.EncodeAsk(key, a) })
c := assign.BackupCondition(node, module, want.Fp, words, true)
if p.say == nil || p.said[c.Key] {
@@ -1176,13 +650,11 @@ func (p *passer) yeses(ctx context.Context, module string, m assign.Module, befo
p.recordAsk(ctx, module, w.Fp, assign.RecordNeed{Node: node, Digest: h.Waits, Judgement: h.Judgement, Words: h.Detail})
}
}
// A want carried over from before the node composed its own may still name the controller's composition; a placed
// node's want names the build alone (ADR 0547 §2).
if yes == "" {
release = ""
}
if yes != w.Yes || release != w.Release || w.Composed != "" || !w.Self {
w.Yes, w.Release, w.Composed, w.Self = yes, release, "", true
if yes != w.Yes || release != w.Release || !w.Self {
w.Yes, w.Release, w.Self = yes, release, true
out.Wants[node] = w
}
}
@@ -1196,139 +668,6 @@ func chosenOf(chosen map[string]catalogue.PersonPolicy, module string) *catalogu
return nil
}
// provisionsOf answers, per module, the provisions it consumes on a node, by key (ADR 0570 §4), from the seat
// mesh-provisioner's state; nil while the state is not read (the seat not delivered yet), so no want waits on it. A failed
// provision is also said as the assigner's finding `provider.<key>.provision-failed` (a waiting one is the watchman's
// needs-operator), deleted once it is no longer failed.
func (p *passer) provisionsOf() func(module string) func(node string) map[string]meshprovisionerclient.Provision {
if p.kvs[buckets.Provisions] == nil {
return func(string) func(string) map[string]meshprovisionerclient.Provision { return nil }
}
by := map[string]map[string]map[string]meshprovisionerclient.Provision{} // module, node, key
failed := map[string]meshprovisionerclient.Provision{}
for key, raw := range p.w.read(buckets.Provisions) {
pr, err := meshprovisionerclient.Decode(raw)
if err != nil {
continue
}
node, module := pr.Consumer.Node, pr.Consumer.Module
if by[module] == nil {
by[module] = map[string]map[string]meshprovisionerclient.Provision{}
}
if by[module][node] == nil {
by[module][node] = map[string]meshprovisionerclient.Provision{}
}
by[module][node][key] = pr
if pr.State == meshprovisionerclient.Failed {
failed[key] = pr
}
}
p.provisionFindings(failed)
return func(module string) func(string) map[string]meshprovisionerclient.Provision {
return func(node string) map[string]meshprovisionerclient.Provision { return by[module][node] }
}
}
// provisionFindings writes the finding of each failed provision and deletes the ones no longer failed.
func (p *passer) provisionFindings(failed map[string]meshprovisionerclient.Provision) {
if p.findings.Put == nil {
return
}
now := time.Now
if p.now != nil {
now = p.now
}
for key, pr := range failed {
if p.failedProvisions[key] {
continue
}
f, err := assign.ProvisionFailedFinding(key, pr, now())
if err == nil {
err = p.findings.Put(f)
}
if err != nil {
p.logf("the failed provision %s is not said: %v", key, err)
continue
}
if p.failedProvisions == nil {
p.failedProvisions = map[string]bool{}
}
p.failedProvisions[key] = true
}
for key := range p.failedProvisions {
if _, still := failed[key]; still {
continue
}
if err := p.findings.Delete(assign.ProvisionFailedKey(key)); err != nil {
p.logf("the failed provision %s is not cleared: %v", key, err)
continue
}
delete(p.failedProvisions, key)
}
}
// MissingBound is how long a required provision may have no key before it is said: a module's delivery bound, the default
// when its manifest names none (ADR 0564 clause 13 (d)).
var MissingBound = motion.DefaultDeliveryBound
// missingPulse is the pulse key of a provision with no key, `provision-missing.<node>.<module>.<provision>`.
func missingPulse(key string) string { return "provision-missing." + key }
// provisionsMissing is the loud end of a required provision with no key (the lead, 2026-10-29, ADR 0570 §4): each one is
// pulsed with the bound when it is first seen; once its pulse expired and it still has no key, the finding
// `provider.<node>.<module>.<provision>.provision-missing` is written; one keyed again ends both. The pulse's expiry, heard on
// the watch, is the pass that writes it: no timer.
func (p *passer) provisionsMissing(ctx context.Context, missing map[string]bool) {
pulses := p.kvs[plc.PulsesBucket()]
if p.missing == nil {
p.missing = map[string]bool{}
}
for key := range missing {
switch {
case !p.missing[key]:
p.missing[key] = true
if pulses != nil {
p.pulse(ctx, missingPulse(key), MissingBound)
}
case pulses == nil || !p.pulsing(missingPulse(key)):
if p.missingSaid[key] || p.findings.Put == nil {
continue
}
f, err := assign.ProvisionMissingFinding(key, MissingBound, p.clock())
if err == nil {
err = p.findings.Put(f)
}
if err != nil {
p.logf("the missing provision %s is not said: %v", key, err)
continue
}
if p.missingSaid == nil {
p.missingSaid = map[string]bool{}
}
p.missingSaid[key] = true
}
}
for key := range p.missing {
if missing[key] {
continue
}
delete(p.missing, key)
if pulses != nil && p.pulsing(missingPulse(key)) {
if err := pulses.Delete(ctx, missingPulse(key)); err == nil {
p.w.put(plc.PulsesBucket(), missingPulse(key), nil)
}
}
if p.missingSaid[key] && p.findings.Delete != nil {
if err := p.findings.Delete(assign.ProvisionMissingKey(key)); err != nil {
p.logf("the missing provision %s is not cleared: %v", key, err)
p.missing[key] = true
continue
}
delete(p.missingSaid, key)
}
}
}
// unplacedFindings writes, from the lease write, the finding `machine.<node>.unplaced` for each node that runs but no
// assignments names, which gets no want (ADR 0571 §6), and deletes it once the node is placed.
func (p *passer) unplacedFindings() {
@@ -1423,40 +762,6 @@ func (p *passer) seatHolders(ctx context.Context, manifests map[string]catalogue
}
}
// offered says whether some manifest of the trunk, or a module the mesh provides itself, offers a provision under its name
// (a module offers its own name too, catalogue.Manifest.Offers): what a requirement nobody offers is told from.
func offered(manifests map[string]catalogue.Manifest) func(string) bool {
names := map[string]bool{}
add := func(m catalogue.Manifest) {
for _, o := range m.Offers() {
names[o] = true
}
}
for _, m := range manifests {
add(m)
}
for _, module := range overlay.ProvidedModules() {
if m, _, err := overlay.ProvidedManifest(module); err == nil {
add(m)
}
}
return func(provision string) bool { return names[provision] }
}
// answering is, by provision, the manifests of the modules placed on some node that provide it: what a want's
// provisions are judged by (catalogue.Awaited, issue 774). A provider in the catalogue placed nowhere answers nobody.
func answering(manifests map[string]catalogue.Manifest, n assign.Nodes) func(string) []catalogue.Manifest {
return func(provision string) []catalogue.Manifest {
var out []catalogue.Manifest
for _, name := range slices.Sorted(maps.Keys(n.Providers[provision])) {
if m, ok := manifests[name]; ok && len(n.Providers[provision][name]) > 0 {
out = append(out, m)
}
}
return out
}
}
// branchView is what one pass reads of the catalog half of one watched branch (design 47 G §184, §195): its modules, their
// manifests, the nodes the rules read through them, the module graph and the two readings of what runs.
type branchView struct {
@@ -1536,56 +841,11 @@ func wantedHere(wanted map[string]mwc.Want, own []string, groups []group, first
return out
}
// noPipelineRules is why each node of module gets no want while no pipeline passed at fp on its branches (design 47 G §29): said as
// any rule's reason (sayNoWant), once.
func noPipelineRules(nodes []string, module, fp string, branches []string) map[string]assign.Holding {
out := map[string]assign.Holding{}
for _, node := range nodes {
out[node] = assign.Holding{Rule: mwc.RuleNotHeld, Words: fmt.Sprintf("no pipeline of %s passed at %s on %s, and a want follows a passed pipeline (design 47 G §29)",
module, shortHex(fp), strings.Join(branches, ", "))}
}
return out
}
// runnerFirstStart says whether module claims the seat mesh-pipeline-runner while no module claiming it runs on any node (design 47
// G A-428): read from the node-engine seat's running statements the assigner hears, each module's claims from its manifest. With
// no running heard yet it says false: the gate stands until the assigner knows.
func runnerFirstStart(module string, manifests map[string]catalogue.Manifest, running map[string]map[string]nodeengine.Running) bool {
if !manifests[module].ClaimsSeat(mpr.Seat) || running == nil {
return false
}
for _, mods := range running {
for m, r := range mods {
if !r.Stopped && r.Fingerprint != "" && manifests[m].ClaimsSeat(mpr.Seat) {
return false
}
}
}
return true
}
// releasedBuild says whether the build at fp is a released version's, as a want of it names it (namedBuild): such a build counts as
// done for rule 1 whatever the builder's build.ended said or the assigner kept of it (design 47 G A-429).
func releasedBuild(named func(fp string) (assign.KeptBuild, bool), fp string) bool {
if named == nil {
return false
}
k, ok := named(fp)
return ok && k.Version != ""
}
// nameHeldBuilds rewrites, for a module the trunk's newer fingerprint of which passed no pipeline, each want held at an older build
// that names no version: with the release whose parts are exactly what that build made, everything else of the want as it stands (novox 2026-11-16: mesh-hand-acts held at 4eb3cf48, released 0.1.13, after the
// mesh-sdk merge moved its fingerprint). No want is written for the unpassed fingerprint, nor for a build with no release.
func (p *passer) nameHeldBuilds(ctx context.Context, name, trunkFP string, group []string, n assign.Nodes, held map[string]mwc.Want, named func(fp string) (assign.KeptBuild, bool)) {
for _, node := range group {
w, ok := held[node]
if !ok || w.Removed || w.Fp == "" || w.Fp == trunkFP || w.Version != "" {
continue
}
if k, found := named(w.Fp); !found || k.Version == "" {
continue
}
p.putWant(ctx, node, name, n.NameKept(assign.Module{KeptBuilds: named}, node, w))
// movedTo says whether the delivery moved owner to node: owner's want there names a fingerprint its have there does not yet run
// healthy (design 47 G §706, *What counts as an outside cause*: a rewritten file of a module the delivery moved is that module's).
func (p *passer) movedTo(wanted map[string]map[string]mwc.Want, n assign.Nodes) func(owner, node string) bool {
return func(owner, node string) bool {
w, ok := wanted[owner][node]
return ok && !w.Removed && !n.RunsHealthy(node, owner, w.Fp)
}
}
@@ -6,7 +6,7 @@ import (
"encoding/json"
"errors"
"fmt"
stdio "git.novox.be/novox/mesh-sdk/go"
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"maps"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/puts"
@@ -24,7 +24,6 @@ import (
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
mwseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents"
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
"github.com/nats-io/nats.go/jetstream"
@@ -39,7 +38,6 @@ import (
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
"mesh-delivery-assigner/internal/assign"
plc "mesh-delivery-assigner/internal/assignment"
@@ -106,166 +104,13 @@ func composes(t *testing.T, w *world, node, digest string) {
var fpTest = meshcatalogclient.SourcePrefix2 + strings.Repeat("b", 64)
// 1906/H2: given a hold lifted by itself, when the assigner is started again and the same fingerprint is held again, then the
// hold ask is written under another asking, with an id of its own, so the teller raises it (the router drops a second ask under
// an id it keeps).
func TestAHoldAskedAgainAfterALiftAndARestartHasANewID(t *testing.T) {
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
kvs := kvSet{buckets.Asks(): asksKV, buckets.Wants(): wantsKV}
key, _ := asks.HoldAskKey("web", fpTest)
held := &mwc.Held{Node: "ace", Words: "health failed"}
first := newPasser(kvs, time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC))
first.holdAsk(context.Background(), "web", fpTest, held)
a1, err := decodeAsk(key, asksKV.values[key])
if err != nil {
t.Fatal(err)
}
first.lift(context.Background(), "web", fpTest)
if _, there := asksKV.values[key]; there {
t.Fatal("the lift left the hold ask")
}
again := newPasser(kvs, time.Date(2026, 10, 25, 2, 0, 0, 0, time.UTC)) // started again: it remembers nothing
again.holdAsk(context.Background(), "web", fpTest, held)
a2, err := decodeAsk(key, asksKV.values[key])
if err != nil {
t.Fatal(err)
}
if a2.Ask.ID == a1.Ask.ID {
t.Errorf("the ask written again after a lift and a restart has the id %s of the first", a1.Ask.ID)
}
// Issue 1966, ADR 0668 §5: assigner_ask_again writes the hold again under a new asking and records the hand act; the
// asking the operator answered is refused, naming the answer; a proposal's authorising ask is refused by its rule.
answersKV := &memKV{values: map[string][]byte{}}
st := memStore{asksState: asksKV, answersState: answersKV}
var recorded []string
record := func(key string, body any) (json.RawMessage, error) {
raw, _ := json.Marshal(body)
recorded = append(recorded, key+" "+string(raw))
return json.RawMessage(`{}`), nil
}
now := time.Date(2026, 10, 25, 3, 0, 0, 0, time.UTC)
out, err := askAgainByHand(st, record, key, "the channel lost it", "lead@shanks", now)
a3, derr := decodeAsk(key, asksKV.values[key])
if err != nil || derr != nil || a3.Ask.ID == a2.Ask.ID || len(recorded) != 1 || !strings.Contains(recorded[0], askAgainTool+" "+key) {
t.Fatalf("ask again: %v, %v; asked %s (was %s), recorded %v", out, err, a3.Ask.ID, a2.Ask.ID, recorded)
}
answersKV.values[key], _ = mdc.EncodeAnswer(mdc.Answer{Option: "keep", Warrant: a3.Ask.ID + "@2026-10-25T03:01:00Z", Words: "the operator"})
if _, err := askAgainByHand(st, record, key, "again", "lead@shanks", now); err == nil || !strings.Contains(err.Error(), "keep") {
t.Errorf("an answered asking is asked again: %v", err)
}
// Issue 1966 note 165 (AC165-2, AC165-3): the teller's ask-ended for a backup ask whose want names another fingerprint writes
// nothing and the journal says it is stale; the same ask with its want at the same fingerprint in another composition, or no
// want yet, is written again.
backKey, _ := asks.BackupAskKey("ace", "web", fpTest)
fpOther := meshcatalogclient.SourcePrefix2 + strings.Repeat("c", 64)
for _, c := range []struct {
name string
want *mwc.Want
have *nodeengine.HaveByNodeModuleValue // A-453: the module's have on the node
stale bool
}{
{"its want names another fingerprint", &mwc.Want{Fp: fpOther, Self: true}, nil, true},
{"its want is removed", &mwc.Want{Removed: true}, nil, true},
{"its want names the fingerprint in another composition", &mwc.Want{Fp: fpTest, Composed: "sha256:" + strings.Repeat("e", 64)}, nil, false},
{"no want yet", nil, nil, false},
// A-453: only a held have with no wait is a failed backup; a hold that carries its wait, or a have no longer held, takes the ask away.
{"its have is held with no wait", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits}, false},
{"its have is held with a wait", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits,
Wait: &nodeengine.Wait{Step: nodeengine.WaitStepCompose, Reason: "entry-missing", Since: now}}, true},
{"its have runs", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}, true},
{"its have backs up again (A-453a)", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseBackingUp, Health: nodeengineclient.HealthWaits}, false},
{"its have is held, naming the build it keeps running", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpOther, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits}, false},
{"its have applies at another fingerprint", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpOther, Phase: nodeengineclient.PhaseApplying, Health: nodeengineclient.HealthWaits}, true},
} {
t.Run("ended backup ask: "+c.name, func(t *testing.T) {
bkv := &memKV{values: map[string][]byte{}}
pb := newPasser(kvSet{buckets.Asks(): bkv, buckets.Wants(): &memKV{values: map[string][]byte{}}, buckets.Have: &memKV{values: map[string][]byte{}}}, now)
ba := assign.BackupAsk(backKey, "ace", "web", fpTest, "sha256:"+strings.Repeat("d", 64), "w", 2, now)
bv, err := assign.EncodeAsk(backKey, ba)
if err != nil {
t.Fatal(err)
}
bkv.values[backKey] = bv
if c.want != nil {
wkey, _ := mwc.WantKey("ace", "web")
raw, _ := mwc.EncodeWant(*c.want)
pb.w.put(buckets.Wants(), wkey, raw)
}
if c.have != nil {
hkey, _ := nodeengineclient.HaveKey("ace", "web")
raw, err := nodeengineclient.EncodeHave(*c.have)
if err != nil {
t.Fatal(err)
}
pb.w.put(buckets.Have, hkey, raw)
}
_, bask, _ := assign.DecodeAsk(backKey, bv)
body, err := tellerevents.EncodeEnded(tellerevents.Ended{Key: backKey, Ask: bask.ID, Outcome: occ.OutcomeExpired})
if err != nil {
t.Fatal(err)
}
var said []string
stale := func(key string) string {
return pb.staleAsk(key, map[string]meshcatalog.ModulesByModuleValue{"web": {Fp2: fpOther}})
}
if err := askAgain(context.Background(), bkv, nil, stale, body, now.Add(time.Hour), func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }); err != nil {
t.Fatal(err)
}
written := !bytes.Equal(bkv.values[backKey], bv)
if written == c.stale || (c.stale && !strings.Contains(strings.Join(said, "\n"), "stale")) {
t.Errorf("written again = %v (stale %v), journal %q", written, c.stale, said)
}
})
}
for _, c := range []struct{ key, by, why string }{{key, stdio.Unproven, "a caller"}, {"hold.none", "lead@shanks", "no ask"}} {
if _, err := askAgainByHand(st, record, c.key, "w", c.by, now); err == nil {
t.Errorf("%s: asked again", c.why)
}
}
// ADR 0668 §3 (review round 1): the bind's read of the operator channel's asks keeps its wait through transport faults;
// past five minutes with the channel's machine heard it is the finding call-unanswered, cleared once answered; with
// the machine away none is written. A refusal is an answer and ends the wait.
for _, heard := range []bool{false, true} {
clock, findings := now, map[string][]byte{}
cw := callWait{heard: func() bool { return heard }, now: func() time.Time { return clock },
findings: assign.Findings{Put: putInto(findings),
Delete: func(k string) error { delete(findings, k); return nil }}}
tries, during := 0, 0
answer, err := cw.keep(context.Background(), asksCallID, channelAsksVerb, func() (json.RawMessage, error) {
tries++
clock = clock.Add(2 * time.Minute)
if tries == 5 {
during = len(findings)
return json.RawMessage(`{"asks":[]}`), nil
}
return nil, errors.New("nats: no responders available for request")
}, t.Logf)
if want := map[bool]int{false: 0, true: 1}[heard]; err != nil || string(answer) != `{"asks":[]}` || tries != 5 || during != want || len(findings) != 0 {
t.Errorf("heard %v: %s %v after %d tries; findings during %d, after %d", heard, answer, err, tries, during, len(findings))
}
}
if _, err := (callWait{}).keep(context.Background(), asksCallID, channelAsksVerb, func() (json.RawMessage, error) {
return nil, &busretry.Answered{Words: "refused"}
}, t.Logf); err == nil {
t.Error("a refusal was asked again")
}
}
// 1906/H4: an answer counts only for the asking it answered: an answer to an earlier hold of the same fingerprint, or to an ask
// taken away, is not taken for the hold that stands now.
// 1906/H4: an answer counts only for the asking it answered: an answer to an earlier asking of the same ask, or to an ask
// taken away, is not taken for the asking that stands now.
func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
// ADR 0603 §5: a provision is offered by a manifest's provides, by a module's own name, and by a module the mesh provides
// itself (mesh-addressing, mesh-wireguard's); nothing else.
off := offered(map[string]catalogue.Manifest{"node-tools": {Module: "node-tools"}, "db": {Module: "db", Provides: []catalogue.Offer{{Name: "database"}}}})
for provision, want := range map[string]bool{"node-tools": true, "database": true, "mesh-addressing": true, "queue": false} {
if off(provision) != want {
t.Errorf("offered(%s) = %v, want %v", provision, !want, want)
}
}
key, _ := asks.HoldAskKey("web", fpTest)
key, _ := asks.BackupAskKey("ace", "web", fpTest)
p := newPasser(kvSet{}, time.Now())
put := func(asking int) string {
a := assign.HoldAsk(key, "web", fpTest, "ace", "health failed", asking, time.Now())
a := assign.BackupAsk(key, "ace", "web", fpTest, "disk full", asking, time.Now())
v, err := assign.EncodeAsk(key, a)
if err != nil {
t.Fatal(err)
@@ -274,20 +119,20 @@ func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
return mustAskID(a)
}
first := put(100)
v, _ := mdc.EncodeAnswer(mdc.Answer{Option: "release", Warrant: first + "@2026-10-25T01:00:00Z", Words: "the operator"})
v, _ := mdc.EncodeAnswer(mdc.Answer{Option: assign.OptionBackupSkip, Warrant: first + "@2026-10-25T01:00:00Z", Words: "the operator"})
p.w.put(buckets.Answers, key, v)
if !p.answeredWith(key, "release") {
if !p.answeredWith(key, assign.OptionBackupSkip) {
t.Fatal("the answer to the ask that stands is not taken")
}
if p.answeredWith(key, "keep") {
if p.answeredWith(key, assign.OptionBackupAgain) {
t.Error("an answer was taken for an option not chosen")
}
put(200) // the same fingerprint held again, after a lift
if p.answeredWith(key, "release") {
t.Error("the answer to an earlier hold carried over to a later one")
put(200) // the same backup asked again
if p.answeredWith(key, assign.OptionBackupSkip) {
t.Error("the answer to an earlier asking carried over to a later one")
}
p.w.put(buckets.Asks(), key, nil) // the ask taken away
if p.answeredWith(key, "release") {
if p.answeredWith(key, assign.OptionBackupSkip) {
t.Error("an answer to an ask taken away was taken")
}
t.Run("a put of the bus acts on the operator's warrant alone", putOfTheBus)
@@ -678,7 +523,7 @@ func putOfTheBus(t *testing.T) {
<-woke
got = step()
w, err := mwc.DecodeWant(wantsKV.values["want.novox.nats"])
if got.Step != puts.Wanted || err != nil || w.Fp != fp || w.Composed != digest || !w.Put || !w.Hold {
if got.Step != puts.Wanted || err != nil || w.Fp != fp || !w.Put || !w.Hold {
t.Fatalf("snapshotted, the put stands at %s (%s); the want %+v, %v", got.Step, got.Words, w, err)
}
if call := <-bus.calls; !strings.HasPrefix(call, "seat:mesh-hand-acts.warranted ") || !strings.Contains(call, a.Ask.ID) { // ADR 0565 §11: into the seat
@@ -755,266 +600,6 @@ func putOfTheBus(t *testing.T) {
}
}
// 1945/AC21: given a module on neither list on a node, when passes follow, then the assigner raises its condition once, says it
// again after a say that failed, and clears it once when the module has a want again; a module on neither list is also written to
// the journal, so it is loud with no operator channel.
func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
// A pair only one source names is a condition of the same kind (issue 1126): folded here to hold the count.
aAssignmentsAloneNamesWhereModulesRun(t)
// The verb edges a want waits on come from the graph and what runs (ADR 0614 §2, §3): folded here to hold the count.
aWantWaitsOnTheVerbEdgesTheGraphKeeps(t)
t.Run("gathers", testThePassGathersTheModulesOnNeitherList)
p := newPasser(kvSet{}, time.Date(2026, 10, 25, 15, 0, 0, 0, time.UTC))
var said []oc.Condition
failing := true
p.say = func(c oc.Condition) error {
if failing {
failing = false
return errors.New("the channel is away")
}
said = append(said, c)
return nil
}
var logged []string
p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
one := map[string]neitherOne{"g14.screen-lock": {node: "g14", module: "screen-lock",
holding: assign.Holding{Rule: mwc.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}}}
p.sayNeither(one, nil, nil) // the say fails
p.sayNeither(one, nil, nil)
p.sayNeither(one, nil, nil)
if len(said) != 1 || said[0].State != occ.StateRaised || said[0].Subject.Machine != "g14" || said[0].Subject.ID != "screen-lock" {
t.Fatalf("said = %+v, want one raised condition of screen-lock on g14", said)
}
if !strings.Contains(strings.Join(logged, "\n"), "screen-lock") {
t.Errorf("the journal does not name the module on neither list: %q", logged)
}
wants := map[string]string{"g14.screen-lock": ""}
p.sayNeither(map[string]neitherOne{}, nil, wants)
p.sayNeither(map[string]neitherOne{}, nil, wants)
if len(said) != 2 || said[1].State != occ.StateCleared || said[1].Key != said[0].Key {
t.Errorf("said = %+v, want the condition cleared once under its key", said)
}
if last := logged[len(logged)-1]; last != "screen-lock on g14 has a want again" {
t.Errorf("the clear said %q", last)
}
// 1945: a module that leaves neither list with no want is cleared in words saying so, never "has a want again". The bus is
// on neither list by a rule as any module (ADR 0601 §1), nothing for the operator to do.
bus := map[string]neitherOne{"novox.nats": {node: "novox", module: "nats", holding: assign.Holding{Rule: mwc.RuleGrant, Words: "no credential"}}}
p.sayNeither(bus, nil, nil)
if c := said[len(said)-1]; c.State != occ.StateRaised || !strings.HasPrefix(c.Explanation, occ.NothingToDo) || !strings.Contains(c.Explanation, "rule 6 holds its first want back: no credential") {
t.Errorf("the bus's neither condition = %+v", c)
}
p.sayNeither(map[string]neitherOne{}, nil, map[string]string{"novox.nats": "it is held, and its held condition says so"})
if c := said[len(said)-1]; c.State != occ.StateCleared || strings.Contains(c.Resolved, "has a want") {
t.Errorf("a module that left neither list with no want was cleared as %+v", c)
}
if last := logged[len(logged)-1]; last != "nats on novox is no longer on neither list and has no want: it is held, and its held condition says so" {
t.Errorf("the clear without a want said %q", last)
}
}
// 1945/AC21: the pass gathers the modules on neither list of every module's outcome, keyed by node and module.
func testThePassGathersTheModulesOnNeitherList(t *testing.T) {
got := map[string]neitherOne{}
gatherNeither(got, "screen-lock", assign.Outcome{Neither: map[string]assign.Holding{"g14": {Rule: mwc.RuleGrant, Words: "w"}}})
gatherNeither(got, "rofi", assign.Outcome{})
if len(got) != 1 || got["g14.screen-lock"].node != "g14" || got["g14.screen-lock"].module != "screen-lock" || got["g14.screen-lock"].holding.Rule != mwc.RuleGrant {
t.Errorf("gathered = %+v", got)
}
// 1945 (the bus bar): a node off neither list is cleared in words that say why it has no want; an operator's put running
// through its steps owns the want, so it is named, never "the node does not run it".
for _, c := range []struct {
name string
puts map[string]puts.Put
before map[string]mwc.Want
why string
}{
{"a live want", nil, map[string]mwc.Want{"novox": {Fp: fpTest}}, ""},
{"a put on its steps", map[string]puts.Put{"novox": {FP: fpTest, Step: puts.Person}}, nil, "its put waits on its own steps and the operator's answer"},
{"nothing", nil, nil, "the node does not run it"},
{"an ended put", map[string]puts.Put{"novox": {FP: fpTest, Step: puts.Refused}}, nil, "the node does not run it"},
} {
left := map[string]string{}
leftNeither(left, assign.Module{Name: "nats", FP: fpTest, Nodes: []string{"novox"}, Puts: c.puts}, c.before, assign.Outcome{})
if why, there := left["novox.nats"]; !there || why != c.why {
t.Errorf("%s: left %q (%v), want %q", c.name, why, there, c.why)
}
}
// 1945 (a first want held back silently): the journal says why a node gets no want when the reason appears or changes, and
// when it has a want again: one line per change, not per pass.
p := newPasser(kvSet{}, time.Date(2026, 10, 25, 21, 30, 0, 0, time.UTC))
var logged []string
p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
notDone := assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: mwc.RuleBuildDone, Words: "the build of rofi at 4e7b118e is being built"}}}
wanted := assign.Outcome{Wants: map[string]mwc.Want{"shanks": {Fp: fpTest, Composed: "sha256:" + strings.Repeat("c", 64)}}}
for i, c := range []struct {
before map[string]mwc.Want
out assign.Outcome
line string // the line said in this pass, "" for none
}{
{nil, notDone, "rofi on shanks gets no want: rule 1: the build of rofi at 4e7b118e is being built"},
{nil, notDone, ""},
{map[string]mwc.Want{"shanks": {Removed: true}}, notDone, ""},
{nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: mwc.RuleGrant, Words: "no credential"}}}, "rofi on shanks gets no want: rule 6: no credential"},
{nil, wanted, "rofi on shanks has a want"},
{map[string]mwc.Want{"shanks": wanted.Wants["shanks"]}, assign.Outcome{}, ""},
} {
logged = nil
p.sayNoWant("rofi", []string{"shanks"}, c.before, c.out)
if got := strings.Join(logged, "\n"); got != c.line {
t.Errorf("pass %d said %q, want %q", i, got, c.line)
}
}
// 1496, ADR 0557 §1: a failure that is the node's own (its fingerprint healthy elsewhere, not held) raises the node's condition
// once, naming the node, the module, F and the have's words, and clears it once when it ends.
var conds []oc.Condition
p.say = func(c oc.Condition) error { conds = append(conds, c); return nil }
failing := map[string]assign.NodeFailure{"shanks": {Others: []string{"ace"}, Words: "health failed, phase running: w"}}
for i, c := range []struct {
failed map[string]assign.NodeFailure
state string // the condition said in this pass, "" for none
}{
{failing, occ.StateRaised},
{failing, ""},
{nil, occ.StateCleared},
{nil, ""},
} {
conds = nil
p.sayNodeFailed("rofi", fpTest, []string{"shanks"}, assign.Nodes{}, assign.Outcome{NodeFailed: c.failed}, false)
switch {
case c.state == "" && len(conds) != 0, c.state != "" && (len(conds) != 1 || conds[0].State != c.state):
t.Errorf("node failed, pass %d said %+v, want %q", i, conds, c.state)
case c.state == occ.StateRaised && (conds[0].Key != "mesh-delivery-assigner.failed-alone.shanks.rofi" ||
!strings.Contains(conds[0].Explanation, "failed on shanks (health failed, phase running: w), while it runs healthy on ace")):
t.Errorf("node failed, raised %+v", conds[0])
}
}
// D2: a node that leaves the module's nodes while it failed alone is cleared once; the first pass after a start clears every
// node the presence bucket knows, for every module, failed have or not (a node that recovered unwatched leaves none).
conds = nil
p.sayNodeFailed("rofi", fpTest, []string{"shanks"}, assign.Nodes{}, assign.Outcome{NodeFailed: failing}, false)
p.sayNodeFailed("rofi", fpTest, nil, assign.Nodes{}, assign.Outcome{}, false)
p.sayNodeFailed("rofi", fpTest, nil, assign.Nodes{}, assign.Outcome{}, false)
if len(conds) != 2 || conds[1].State != occ.StateCleared || conds[1].Key != "mesh-delivery-assigner.failed-alone.shanks.rofi" {
t.Errorf("a node leaving the module said %+v, want one raise and one clear", conds)
}
known := assign.Nodes{Online: map[string]bool{"ace": true, "g14": false},
Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"g14": {"rofi": {Fp: fpTest, Health: nodeengineclient.HealthFailed}}}}
for _, c := range []struct {
name string
known assign.Nodes
keys []string
}{
{"a failed have", known, []string{"mesh-delivery-assigner.failed-alone.ace.rofi", "mesh-delivery-assigner.failed-alone.g14.rofi"}},
{"no failed have", assign.Nodes{Online: known.Online}, []string{"mesh-delivery-assigner.failed-alone.ace.rofi", "mesh-delivery-assigner.failed-alone.g14.rofi"}},
} {
conds = nil
fresh := newPasser(kvSet{}, time.Date(2026, 10, 26, 22, 0, 0, 0, time.UTC))
fresh.logf = func(string, ...any) {}
fresh.say = func(c oc.Condition) error { conds = append(conds, c); return nil }
fresh.sayNodeFailed("rofi", fpTest, nil, c.known, assign.Outcome{}, true)
var keys []string
for _, cond := range conds {
if cond.State == occ.StateCleared {
keys = append(keys, cond.Key)
}
}
if !slices.Equal(keys, c.keys) || len(conds) != len(c.keys) {
t.Errorf("first pass, %s: said %+v, want clears of %q", c.name, conds, c.keys)
}
}
}
// A neither or held condition a previous process raised is cleared by the first pass after a start: a fresh pass remembers
// nothing it said, so it clears, once, the neither condition of every assigned module not on neither list and the held condition of
// every module that is not held. Clearing a condition that is not open is quiet at the router.
func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) {
t.Run("a held module", testAHeldModuleRaisesNoNeitherAndClearsAStandingOne)
wantsKV := &memKV{values: map[string][]byte{}}
p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, time.Now())
var said []oc.Condition
p.say = func(c oc.Condition) error { said = append(said, c); return nil }
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
putModuleRaw(p.w, trunkBranch, "web", mod)
placeOn(t, p.w, "g14", "web")
on := true
p.w.heardPresence("g14", !on, time.Now())
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
p.w.put(buckets.Have, "g14.web", have)
key, _ := mwc.WantKey("g14", "web")
want, _ := mwc.EncodeWant(mwc.Want{Fp: fpTest, Composed: "sha256:" + strings.Repeat("c", 64)})
p.w.put(buckets.Wants(), key, want)
p.wants(context.Background())
cleared := map[string]bool{}
for _, c := range said {
if c.State != occ.StateCleared {
t.Errorf("the first pass raised %+v", c)
}
cleared[c.Key] = true
}
neither := assign.NeitherCondition("g14", "web", assign.Holding{}, false).Key
held := assign.HeldCondition("web", fpTest, "", "", false, false).Key
if !cleared[neither] || !cleared[held] {
t.Errorf("cleared = %v, want %s and %s", cleared, neither, held)
}
n := len(said)
p.wants(context.Background())
if len(said) != n {
t.Errorf("a second pass said again: %+v", said[n:])
}
}
// 1945/AC24: given a module held at F under rule 2 on a node that runs it with no want, when the assigner passes, then it raises no
// neither condition for it, a neither condition this process raised for it before is cleared, and so is one a previous process
// raised (the first pass after a start), each once.
func testAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
setup := func() (*passer, *[]oc.Condition) {
p := newPasser(kvSet{buckets.Wants(): &memKV{values: map[string][]byte{}}, buckets.Asks(): &memKV{values: map[string][]byte{}}}, time.Now())
said := &[]oc.Condition{}
p.say = func(c oc.Condition) error { *said = append(*said, c); return nil }
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fpTest, Repository: "novox/resolved", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
putModuleRaw(p.w, trunkBranch, "systemd-resolved", mod)
placeOn(t, p.w, "shanks", "systemd-resolved")
on := true
p.w.heardPresence("shanks", !on, time.Now())
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed})
p.w.put(buckets.Have, "shanks.systemd-resolved", have)
hkey, _ := mwc.HeldKey("systemd-resolved", fpTest)
held, _ := mwc.EncodeHeld(mwc.Held{Node: "shanks", Words: "health failed"})
p.w.put(buckets.Wants(), hkey, held)
return p, said
}
key := assign.NeitherCondition("shanks", "systemd-resolved", assign.Holding{}, false).Key
neithers := func(said []oc.Condition) (raised, cleared int) {
for _, c := range said {
if c.Key != key {
continue
}
if c.State == occ.StateRaised {
raised++
} else {
cleared++
}
}
return
}
// a standing neither this process raised
p, said := setup()
p.neither = map[string]neitherOne{"shanks.systemd-resolved": {node: "shanks", module: "systemd-resolved", holding: assign.Holding{Rule: mwc.RuleNotHeld}}}
p.wants(context.Background())
p.wants(context.Background())
if r, c := neithers(*said); r != 0 || c != 1 {
t.Errorf("standing in this process: %d raised, %d cleared, want 0 and 1: %+v", r, c, *said)
}
// one a previous process raised: the first pass after a start clears it
p, said = setup()
p.wants(context.Background())
p.wants(context.Background())
if r, c := neithers(*said); r != 0 || c != 1 {
t.Errorf("after a start: %d raised, %d cleared, want 0 and 1: %+v", r, c, *said)
}
}
// 1906 note 28: an ask outlives its fingerprint. Each pass takes away the hold, restore and record asks of a module whose
// fingerprint (or composition) is not the trunk's now, or whose module left the trunk; a trunk not read takes nothing away; a
// backup ask is taken away when its module's want on the node names another fingerprint or is removed, and left when the want
@@ -1026,15 +611,10 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
return map[string]meshcatalog.ModulesByModuleValue{"web": {Fp2: fp, Manifest: "sha256:" + strings.Repeat("1", 64), Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}}
}
type ask struct{ name, key string }
holdOld, _ := asks.HoldAskKey("web", fpOld)
holdNew, _ := asks.HoldAskKey("web", fpNew)
holdGone, _ := asks.HoldAskKey("gone", fpOld)
restOld, _ := asks.RestoreAskKey("novox", "web", fpOld)
restNew, _ := asks.RestoreAskKey("novox", "web", fpNew)
recOld, _ := asks.RecordAskKey("novox", "web", digOld)
recNew, _ := asks.RecordAskKey("novox", "web", digNew)
backOld, _ := asks.BackupAskKey("novox", "web", fpOld)
all := []string{holdOld, holdNew, holdGone, restOld, restNew, recOld, recNew, backOld}
all := []string{recOld, recNew, backOld}
for _, c := range []struct {
name string
trunk map[string]meshcatalog.ModulesByModuleValue
@@ -1042,13 +622,12 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
noDigit bool // the node's own composition is not read
want *mwc.Want // the want of web on novox, nil when none is written
}{
{"the trunk moved on", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false, nil},
{"the backup's want names another fingerprint", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld, backOld}, false, &mwc.Want{Fp: fpNew, Self: true}},
{"the backup's want is removed", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld, backOld}, false, &mwc.Want{Removed: true}},
{"the backup's want names the fingerprint in another composition (its have waits for a yes: a mid-move have is not past the hold, A-453a)", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false, &mwc.Want{Fp: fpOld, Composed: digNew}},
{"the trunk is at the old fingerprint", trunkOf(fpOld), []string{holdNew, holdGone, restNew, recOld}, false, nil},
{"the trunk's fingerprint is not read yet", trunkOf(""), []string{holdGone}, false, nil},
{"the composition is not read yet", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, true, nil},
{"the trunk moved on", trunkOf(fpNew), []string{recOld}, false, nil},
{"the backup's want names another fingerprint", trunkOf(fpNew), []string{recOld, backOld}, false, &mwc.Want{Fp: fpNew, Self: true}},
{"the backup's want is removed", trunkOf(fpNew), []string{recOld, backOld}, false, &mwc.Want{Removed: true}},
{"the trunk is at the old fingerprint", trunkOf(fpOld), []string{recOld}, false, nil},
{"the trunk's fingerprint is not read yet", trunkOf(""), nil, false, nil},
{"the composition is not read yet", trunkOf(fpNew), nil, true, nil},
{"the trunk is not read at all", nil, nil, false, nil},
{"the trunk is not read at all, the backup's want moved", nil, nil, false, &mwc.Want{Fp: fpNew, Self: true}},
} {
@@ -1061,14 +640,10 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
k, _ := asks.ParseAnyAskKey(key)
var a asks.Ask
switch k.Kind {
case asks.AskKindHold:
a = assign.HoldAsk(key, k.Module, k.FP, "novox", "w", 1, time.Now())
case asks.AskKindRestore:
a = assign.RestoreAsk(key, k.Node, k.Module, k.FP, "w", 2, time.Now())
case asks.AskKindRecord:
a = assign.RecordAsk(key, k.Node, k.Module, fpNew, assign.RecordNeed{Node: k.Node, Digest: k.Composed}, 2, time.Now())
default:
a = assign.BackupAsk(key, k.Node, k.Module, k.FP, digOld, "w", 2, time.Now())
a = assign.BackupAsk(key, k.Node, k.Module, k.FP, "w", 2, time.Now())
}
v, err := assign.EncodeAsk(key, a)
if err != nil {
@@ -1095,20 +670,7 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
}
p.w.put(buckets.Wants(), wkey, raw)
}
heldOld, _ := mwc.HeldKey("web", fpOld)
wantsKV.values[heldOld] = []byte("x")
p.w.put(buckets.Wants(), heldOld, []byte("x"))
firstAsk, _ := decodeAsk(holdOld, asksKV.values[holdOld])
p.staleAsks(context.Background(), c.trunk)
// the held record and condition of a fingerprint whose hold ask went are taken away with it
heldOld, _ = mwc.HeldKey("web", fpOld)
_, heldThere := wantsKV.values[heldOld]
cleared := slices.ContainsFunc(said, func(c oc.Condition) bool {
return c.Key == assign.HeldCondition("web", fpOld, "", "", false, false).Key && c.State == occ.StateCleared
})
if gone := slices.Contains(c.gone, holdOld); !heldThere || cleared != gone {
t.Errorf("hold ask of the old fingerprint gone = %v: its held record stays = %v (must), its condition cleared = %v", gone, heldThere, cleared)
}
// the failed backup's condition is cleared with its ask, and with no other (AC165-1)
backCleared := slices.ContainsFunc(said, func(c oc.Condition) bool {
return c.Key == assign.BackupCondition("novox", "web", fpOld, "", false).Key && c.State == occ.StateCleared
@@ -1123,66 +685,9 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
t.Errorf("%s: left in the bucket = %v, want %v", key, there, want)
}
}
if slices.Contains(c.gone, holdOld) {
// the trunk returns to the fingerprint (a revert): the held record still holds it, and it is asked and said again
p.now = func() time.Time { return time.Now().Add(time.Hour) }
p.holdAsk(context.Background(), "web", fpOld, nil)
p.heldCondition(context.Background(), "web", fpOld, nil, false)
again, err := decodeAsk(holdOld, asksKV.values[holdOld])
raised := slices.ContainsFunc(said, func(c oc.Condition) bool { return c.State == occ.StateRaised })
if err != nil || again.Ask.ID == firstAsk.Ask.ID || !raised {
t.Errorf("the trunk returned to the held fingerprint: ask %q (first %q, %v), condition raised again = %v", again.Ask.ID, firstAsk.Ask.ID, err, raised)
}
}
})
}
t.Run("a clear that failed to be said is said at the next pass, and the ask goes after it", func(t *testing.T) {
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
p := newPasser(kvSet{buckets.Asks(): asksKV, buckets.Wants(): wantsKV}, time.Now())
var said []oc.Condition
fail := true
p.say = func(c oc.Condition) error {
if fail {
fail = false
return errors.New("router unreachable")
}
said = append(said, c)
return nil
}
a := assign.HoldAsk(holdOld, "web", fpOld, "novox", "w", 1, time.Now())
v, err := assign.EncodeAsk(holdOld, a)
if err != nil {
t.Fatal(err)
}
asksKV.values[holdOld] = v
p.w.put(buckets.Asks(), holdOld, v)
p.staleAsks(context.Background(), trunkOf(fpNew))
if _, there := asksKV.values[holdOld]; !there || len(said) != 0 {
t.Fatalf("after a failed clear: ask left = %v, conditions said = %d; want the ask kept and nothing said", there, len(said))
}
p.staleAsks(context.Background(), trunkOf(fpNew))
if _, there := asksKV.values[holdOld]; there || len(said) != 1 || said[0].State != occ.StateCleared {
t.Errorf("at the next pass: ask left = %v, conditions said = %v; want the ask gone and the clear said", there, said)
}
})
t.Run("a lift takes the restore asks of its fingerprint on every node", func(t *testing.T) {
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
p := newPasser(kvSet{buckets.Asks(): asksKV, buckets.Wants(): wantsKV}, time.Now())
keep, _ := asks.RestoreAskKey("ace", "web", fpNew)
keepOther, _ := asks.RestoreAskKey("ace", "other", fpOld)
for _, key := range []string{restOld, mustKey(asks.RestoreAskKey("ace", "web", fpOld)), keep, keepOther} {
asksKV.values[key] = []byte("x")
p.w.put(buckets.Asks(), key, []byte("x"))
}
p.lift(context.Background(), "web", fpOld)
for key, gone := range map[string]bool{restOld: true, mustKey(asks.RestoreAskKey("ace", "web", fpOld)): true, keep: false, keepOther: false} {
if _, there := asksKV.values[key]; there == gone {
t.Errorf("%s: left in the bucket = %v after the lift", key, there)
}
}
})
}
func mustKey(key string, err error) string {
@@ -1216,7 +721,7 @@ func aReleasesApproveAnswersTheMoveOfItsBuildAlone(t *testing.T) {
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fpOld, Composed: was, Phase: nodeengineclient.PhaseWaitingYes, Health: nodeengineclient.HealthOK,
Judgement: assign.JudgedRecreates, Waits: waiting, Detail: "it recreates web.server", BuildAlone: true})
p.w.put(buckets.Have, "g14.web", have)
m := assign.Module{Name: "web", FP: fpTest, Nodes: []string{"g14"}, Self: func(string) bool { return true }}
m := assign.Module{Name: "web", Nodes: []string{"g14"}, Self: func(string) bool { return true }}
out := assign.Outcome{Wants: map[string]mwc.Want{"g14": {Fp: fpTest, Self: true}}}
p.yeses(context.Background(), "web", m, map[string]mwc.Want{"g14": {Fp: fpOld, Self: true}}, &out)
if w := out.Wants["g14"]; w.Yes != "" || w.Release != "" {
@@ -1254,7 +759,11 @@ func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) {
p.logf = t.Logf
put := func(bucket, key string, v []byte) { p.w.put(bucket, key, v) }
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
putModuleRaw(p.w, trunkBranch, "web", mod) // the trunk's build is not done: rule 1 holds g14 at fpOld
putModuleRaw(p.w, trunkBranch, "web", mod)
// The newest version passed is fpOld's (design 47 H §750): the trunk's newer build passed no pipeline.
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
p.ends.trunk = p.w.trunkName
_, _ = p.ends.take(ended(trunkBranch, 1, mpr.PipelineEndedOutcomePassed, mover("web", fpOld, "0.1.1", 1, mpr.StepResultOutcomePassed)))
placeOn(t, p.w, "g14", "web")
pl, _ := mmac.EncodeAssignments(mma.Assignments{Node: "g14", Modules: []mma.PlacedModule{{Module: "web"}}})
put(plc.AssignmentsBucket(), "assignments.g14", pl)
@@ -1263,7 +772,7 @@ func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) {
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fpOld, Composed: was, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
put(buckets.Have, "g14.web", have)
wkey, _ := mwc.WantKey("g14", "web")
want, _ := mwc.EncodeWant(mwc.Want{Fp: fpOld, Composed: was})
want, _ := mwc.EncodeWant(mwc.Want{Fp: fpOld, Version: "0.1.1", Pipeline: trunkBranch + ".1", Rank: 1})
wantsKV.values[wkey] = want
put(buckets.Wants(), wkey, want)
sync := func() {
@@ -1276,7 +785,7 @@ func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) {
p.nodeWants(context.Background())
sync()
got, err := mwc.DecodeWant(wantsKV.values[wkey])
if err != nil || got.Fp != fpOld || got.Composed != "" || !got.Self {
if err != nil || got.Fp != fpOld || !got.Self {
t.Fatalf("the placed node's want is %+v (%v), want it at %s, the node composing it", got, err, fpOld)
}
nkey, _ := mwc.WantNodeKey("g14")
@@ -1392,92 +901,6 @@ func TestAnApprovedAssignmentsCallsNoControllerVerb(t *testing.T) {
}
}
// ADR 0614 §2, §3: of what a module invokes, the pass keeps as edges the verbs the graph does not leave out and another
// module serves; a verb is served once a running healthy build's manifest (the builder's build, the indexer's manifest)
// serves it, and a build whose manifest cannot be read counts what the graph says its module serves. No graph read: no wait.
func aWantWaitsOnTheVerbEdgesTheGraphKeeps(t *testing.T) {
p := newPasser(kvSet{}, time.Now())
fp := func(c string) string { return meshcatalogclient.SourcePrefix2 + strings.Repeat(c, 64) }
if p.served(graph.Graph{}, assign.Nodes{}) != nil {
t.Error("with no graph read, a want waits on a verb")
}
for module, n := range map[string]graph.Node{
"caller": {FP: fp("c"), Repository: "novox/caller", Invokes: []string{"seat:x.v", "loop.l", "none.t", "caller.own"}, LeftOut: []string{"loop.l"},
Serves: []string{"caller.own"}},
"server": {FP: fp("e"), Repository: "novox/server", Serves: []string{"seat:x.v", "seat:x.w"}},
"loop": {FP: fp("1"), Repository: "novox/loop", Serves: []string{"loop.l"}},
"late": {FP: fp("2"), Repository: "novox/late", Serves: []string{"late.t"}},
} {
raw, err := graph.Encode(n)
if err != nil {
t.Fatal(err)
}
_ = raw
p.w.putModule(trunkBranch, module, meshcatalog.ModulesByModuleValue{}, n, nil)
}
g := p.moduleGraph(trunkBranch)
if got := keptInvokes(g, "caller"); !slices.Equal(got, []string{"seat:x.v"}) || keptInvokes(g, "unknown") != nil {
t.Errorf("the kept verb edges of caller = %v", got)
}
digest := strings.Repeat("d", 64)
parsedServer, err := catalogue.Decode([]byte(`{"module":"server","claims":[{"name":"x","scope":"mesh","serves":["w"]}]}`))
if err != nil {
t.Fatal(err)
}
p.w.manifests[digest] = parsed{module: "server", manifest: parsedServer}
endDone(t, p.w, "server", fp("f"), digest)
running := nodeengine.HaveByNodeModuleValue{Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
at := func(f string) nodeengine.HaveByNodeModuleValue { h := running; h.Fp = f; return h }
served := p.served(g, assign.Nodes{Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{
"g14": {"server": at(fp("f")), "late": at(fp("2"))}}})
for v, want := range map[string]bool{"seat:x.w": true, "seat:x.v": false, "late.t": true} {
if served(v) != want {
t.Errorf("served(%s) = %v, want %v: the running server's manifest serves w alone, late's is not read", v, served(v), want)
}
}
// The modules whose running build's manifest is not read are said once per change of their set (G-23).
var said []string
p.logf = func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }
lateOnly := assign.Nodes{Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"g14": {"server": at(fp("f")), "late": at(fp("2"))}}}
p.served(g, lateOnly)
p.served(g, lateOnly)
p.served(g, assign.Nodes{Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"g14": {"late": at(fp("2")), "loop": at(fp("1"))}}})
p.served(g, lateOnly)
if len(said) != 2 || !strings.Contains(said[0], "[loop late]") && !strings.Contains(said[0], "[late loop]") || !strings.Contains(said[1], "[late]") {
t.Errorf("the unread manifests were said %q, want once per change of the set", said)
}
// A-426a: a healthy build held (no catalog entry of it) or waiting for a yes serves what it serves, said once; one held unhealthy,
// or whose running statement says it stopped, does not.
held := at(fp("2"))
held.Phase = nodeengineclient.PhaseHeld
served = p.served(g, assign.Nodes{Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"novox": {"late": held}}})
if !served("late.t") {
t.Error("a healthy held build of late does not serve late.t")
}
stopped := assign.Nodes{Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"novox": {"late": held}},
Running: map[string]map[string]nodeengine.Running{"novox": {"late": {Stopped: true}}}}
sick := held
sick.Health = "failed"
if p.served(g, stopped)("late.t") || p.served(g, assign.Nodes{Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"novox": {"late": sick}}})("late.t") {
t.Error("a held build stopped, or not healthy, serves late.t")
}
// ADR 0614 §5: bus-state counts as offered only from a healthy build on a node present; the bus module's build on an
// offline node offers nothing, and one whose manifest is not read counts its trunk manifest's offer. A-352: a have waiting
// for a person's yes on a newer build names the build that still runs, healthy, and it offers bus-state too.
bus := map[string]catalogue.Manifest{"bus": {Module: "bus", Provides: []catalogue.Offer{{Name: graph.BusState, Scope: catalogue.ScopeMesh}}}}
for _, online := range []bool{false, true} {
for _, phase := range []string{nodeengineclient.PhaseRunning, nodeengineclient.PhaseWaitingYes} {
h := at(fp("2"))
h.Phase = phase
offering := p.offering(bus, assign.Nodes{Online: map[string]bool{"n2": online},
Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"n2": {"bus": h}}})
if offering(graph.BusState) != online {
t.Errorf("with the bus module's node online %v in phase %s, bus-state offered %v", online, phase, offering(graph.BusState))
}
}
}
}
// decodedAsk is an asks-state value with the router's ask it carries decoded (mesh-sdk keeps it as raw JSON).
type decodedAsk struct {
Asking int
@@ -1567,20 +990,10 @@ func aRecordAskIsNotTakenAwayBeforeItsYesIsWrittenAndIsAskedAgainWhenWrittenAgai
p.recordAsk(context.Background(), "web", fp, assign.RecordNeed{Node: "novox", Digest: dig})
return readAskID(t, p, rkey)
}
rest, _ := asks.RestoreAskKey("novox", "web", fp)
restore := func(p *passer) string {
p.restoreAsk(context.Background(), "web", "novox", fp)
return readAskID(t, p, rest)
}
hold, _ := asks.HoldAskKey("web", fp)
holdIt := func(p *passer) string {
p.holdAsk(context.Background(), "web", fp, &mwc.Held{Node: "novox", Words: "w"})
return readAskID(t, p, hold)
}
for name, c := range map[string]struct {
write func(*passer) string
key string
}{"record": {rec, rkey}, "restore": {restore, rest}, "hold": {holdIt, hold}} {
}{"record": {rec, rkey}} {
at := time.Now()
asksKV := &memKV{values: map[string][]byte{}}
p := newPasser(kvSet{buckets.Asks(): asksKV, buckets.Wants(): &memKV{values: map[string][]byte{}}}, at)
@@ -1,539 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
mdseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/assignment"
)
// The pins of the assignment graph revisions (design 47 G §153, §178, §216, §579; A-169): the gate says on the seat
// mesh-delivery which revisions a delivery record starts (`pinned`) and stops (`freed`) pointing at, and once at each of its
// starts `pins-stated`. The assigner hears them on its durable consumer of the events stream and keeps, in its old `state`
// section `pins` (4 MiB), one key `trunk.<n>` per assignment graph revision whose value is the list of the records pinning
// it, added or removed by compare-and-set before the event is acknowledged; a record heard twice pins once. Two more keys:
// `made`, written once when the state is first used, and `pins-stated`, the `at` of the last pins-stated applied. A revision
// is removed, its object and its key and the entry objects no kept revision lists and no want or running build names
// (§179), when its list is empty and it is not current (the newest the assigner wrote), only while the consumer is caught up
// and once a pins-stated said after `made` was applied; judged on a freed, on a revision ceasing to be current, on being
// caught up, and at start once both hold. Never on a timer, and never by reading the gate's state.
const (
pinsState = "pins"
pinsTool = "assigner_pins"
unpinTool = "assigner_unpin"
pinsMadeKey = "made"
pinsStatedKey = "pins-stated"
pinsCapBytes = 4 << 20
pinMissingKind = "pin-missing"
)
// PinsValue is one value of the state pins: the records that pin the revision, and whether the revision was no longer held
// when a record pinned it.
type PinsValue struct {
Records []string `json:"records"`
Missing bool `json:"missing,omitempty"`
}
// pinKey is the key of an assignment graph revision, its object's own name (A-169).
func pinKey(revision int64) string { return assigngraph.TrunkObject(revision) }
// pinRevision is the revision a key of the state names, and whether it names one.
func pinRevision(key string) (int64, bool) {
n, ok := strings.CutPrefix(key, "trunk.")
if !ok {
return 0, false
}
r, err := strconv.ParseInt(n, 10, 64)
return r, err == nil && r > 0
}
// pinObjects is the object bucket of the revisions as the pins remove from it.
type pinObjects interface {
assigngraph.Objects
Delete(ctx context.Context, name string) error
}
// pinKeeper keeps the pins and removes what nothing pins.
type pinKeeper struct {
ctx context.Context
st endStore
objects pinObjects
// inUse says a want or a running build names the module's fingerprint (§179).
inUse func(module, fp string) bool
findings assign.Findings
now func() time.Time
logf func(string, ...any)
mu sync.Mutex
// caught is whether the last event applied left none of the consumer's messages pending.
caught bool
}
// edit applies change to the value at key by compare-and-set, again while another write comes between.
func (k *pinKeeper) edit(key string, change func(v PinsValue, found bool) (PinsValue, bool)) error {
for {
raw, rev, err := k.st.get(key)
if err != nil {
return err
}
var v PinsValue
if raw != nil {
if err := json.Unmarshal(raw, &v); err != nil {
return fmt.Errorf("the pins of %s do not read: %w", key, err)
}
}
next, write := change(v, raw != nil)
if !write {
return nil
}
value, _ := json.Marshal(next)
if err := k.st.swap(key, value, rev); errors.Is(err, errSwapped) {
continue
} else {
return err
}
}
}
// held says the revision's object is kept.
func (k *pinKeeper) held(revision int64) (bool, error) {
_, err := k.objects.Get(k.ctx, assigngraph.TrunkObject(revision))
if errors.Is(err, assigngraph.ErrNoObject) {
return false, nil
}
return err == nil, err
}
// pin adds record to each assignment graph revision of p; a revision no longer held is written missing, with the finding
// pin-missing naming the record.
func (k *pinKeeper) pin(p mdseat.Pins) error {
for _, r := range p.Assignment {
held, err := k.held(r)
if err != nil {
return err
}
err = k.edit(pinKey(r), func(v PinsValue, _ bool) (PinsValue, bool) {
if slices.Contains(v.Records, p.Record) && (held || v.Missing) {
return v, false
}
if !slices.Contains(v.Records, p.Record) {
v.Records = append(v.Records, p.Record)
slices.Sort(v.Records)
}
v.Missing = v.Missing || !held
return v, true
})
if err != nil {
return err
}
if !held {
k.missing(r, p.Record)
}
}
return nil
}
// free removes record from each assignment graph revision of p; a freed applied twice changes nothing.
func (k *pinKeeper) free(p mdseat.Pins) error {
for _, r := range p.Assignment {
err := k.edit(pinKey(r), func(v PinsValue, found bool) (PinsValue, bool) {
i := slices.Index(v.Records, p.Record)
if !found || i < 0 {
return v, false
}
v.Records = slices.Delete(v.Records, i, i+1)
return v, true
})
if err != nil {
return err
}
}
return nil
}
// missing writes the finding pin-missing, scope the assigner, id the revision.
func (k *pinKeeper) missing(revision int64, record string) {
if k.findings.Put == nil {
k.logf("the revision %d the record %s pins is not held, and no finding writer is wired to say so", revision, record)
return
}
o := assign.Observed{Scope: assignment.Module(), ID: strconv.FormatInt(revision, 10), Kind: pinMissingKind,
Summary: fmt.Sprintf("the delivery record %s pins the assignment graph revision %d, which the assigner no longer holds", record, revision),
Headline: "A delivery record pins a revision the assigner no longer holds",
Explanation: fmt.Sprintf("The record %s pinned revision %d after it was removed; %s clears it.", record, revision, unpinTool),
Source: assignment.Module()}
if err := k.findings.Put(o.Finding(k.now())); err != nil {
k.logf("the finding %s of revision %d is not written: %v", pinMissingKind, revision, err)
}
}
// made writes the key made once, create-only; one there already stays.
func (k *pinKeeper) made() error {
raw, _, err := k.st.get(pinsMadeKey)
if err != nil || raw != nil {
return err
}
value, _ := json.Marshal(k.now().UTC())
if err := k.st.swap(pinsMadeKey, value, 0); err != nil && !errors.Is(err, errSwapped) {
return err
}
return nil
}
// timeAt is the time a key holds; false when it holds none.
func (k *pinKeeper) timeAt(key string) (time.Time, bool, error) {
raw, _, err := k.st.get(key)
if err != nil || raw == nil {
return time.Time{}, false, err
}
var t time.Time
if err := json.Unmarshal(raw, &t); err != nil {
return t, false, fmt.Errorf("the key %s does not read: %w", key, err)
}
return t, true, nil
}
// stated keeps the at of a pins-stated applied.
func (k *pinKeeper) stated(at time.Time) error {
for {
raw, rev, err := k.st.get(pinsStatedKey)
if err != nil {
return err
}
var was time.Time
if raw != nil && json.Unmarshal(raw, &was) == nil && !at.After(was) {
return nil
}
value, _ := json.Marshal(at.UTC())
if err := k.st.swap(pinsStatedKey, value, rev); errors.Is(err, errSwapped) {
continue
} else {
return err
}
}
}
// statedSinceMade says a pins-stated said after the state was made was applied.
func (k *pinKeeper) statedSinceMade() (bool, error) {
made, ok, err := k.timeAt(pinsMadeKey)
if err != nil || !ok {
return false, err
}
at, ok, err := k.timeAt(pinsStatedKey)
if err != nil || !ok {
return false, err
}
return at.After(made), nil
}
// hear applies one of the seat mesh-delivery's events and acknowledges it only once applied.
func (k *pinKeeper) hear(m jetstream.Msg) {
err := k.apply(m.Subject(), m.Data())
if err != nil {
k.logf("%s: %v; it is heard again", m.Subject(), err)
_ = m.Nak()
return
}
_ = m.Ack()
caught := false
if meta, merr := m.Metadata(); merr == nil {
caught = meta.NumPending == 0
}
k.mu.Lock()
was := k.caught
k.caught = caught
k.mu.Unlock()
if caught && (!was || m.Subject() != mdseat.SubjectPinned) {
k.judge() // on a freed, on a candidate's heads replaced, on pins-stated, and on being caught up
}
}
// apply takes one event's body.
func (k *pinKeeper) apply(subject string, data []byte) error {
switch subject {
case mdseat.SubjectPinned, mdseat.SubjectFreed:
var p mdseat.Pins
if err := json.Unmarshal(data, &p); err != nil || p.Record == "" {
k.logf("%s holds no pins naming a record, and is not taken: %v", subject, err)
return nil
}
if subject == mdseat.SubjectPinned {
return k.pin(p)
}
return k.free(p)
case mdseat.SubjectPinsStated:
var s mdseat.PinsStatedEvent
if err := json.Unmarshal(data, &s); err != nil || s.At.IsZero() {
k.logf("%s holds no at, and is not taken: %v", subject, err)
return nil
}
return k.stated(s.At)
}
return nil // heads-replaced: a candidate's revision stops being current, judged after
}
// judge removes every revision neither pinned nor current, when the rule allows a removal at all.
func (k *pinKeeper) judge() {
k.mu.Lock()
caught := k.caught
k.mu.Unlock()
if !caught {
return
}
ok, err := k.statedSinceMade()
if err != nil {
k.logf("the pins are not judged: %v", err)
return
}
if !ok {
return // no pins-stated applied since the state was made: a pin said before may not be heard yet
}
if err := k.remove(); err != nil {
k.logf("the revisions nothing pins are not removed: %v", err)
}
}
// remove removes each kept revision that is not current and that no record pins, and then the entries left unlisted.
func (k *pinKeeper) remove() error {
names, err := k.objects.Names(k.ctx)
if err != nil {
return err
}
var kept []int64
for _, n := range names {
if r, ok := pinRevision(n); ok {
kept = append(kept, r)
}
}
if len(kept) == 0 {
return nil
}
current := slices.Max(kept)
listed := map[string]bool{}
removed := 0
for _, r := range kept {
raw, _, err := k.st.get(pinKey(r))
if err != nil {
return err
}
var v PinsValue
if raw != nil {
if err := json.Unmarshal(raw, &v); err != nil {
return fmt.Errorf("the pins of %d do not read: %w", r, err)
}
}
if r == current || len(v.Records) > 0 {
rev, err := assigngraph.ReadRevision(k.ctx, k.objects, r)
if err != nil {
return err
}
for _, e := range rev.Entries {
listed[e.Digest] = true
}
continue
}
if err := k.objects.Delete(k.ctx, assigngraph.TrunkObject(r)); err != nil {
return fmt.Errorf("revision %d: %w", r, err)
}
if raw != nil {
if err := k.st.drop(pinKey(r)); err != nil {
return fmt.Errorf("the pins of %d: %w", r, err)
}
}
removed++
}
if removed == 0 {
return nil
}
// A run's candidate kept now lists its entries too, until its group-ended drops it (A-201, A-235).
for _, n := range names {
group, heads, ok := meshwants.ParseAssignmentRevisionsByCandidateGroupHeadsKey(n)
if !ok {
continue
}
rev, err := assigngraph.ReadCandidate(k.ctx, k.objects, group, heads)
if err != nil {
return err
}
for _, e := range rev.Entries {
listed[e.Digest] = true
}
}
for _, n := range names {
digest, ok := strings.CutPrefix(n, "entry.")
if !ok || listed[digest] {
continue
}
e, err := assigngraph.ReadEntry(k.ctx, k.objects, digest)
if err != nil {
return err
}
if k.inUse != nil && k.inUse(e.Module, e.Fp) {
continue
}
if err := k.objects.Delete(k.ctx, n); err != nil {
return fmt.Errorf("the entry %s: %w", digest, err)
}
}
k.logf("%d assignment graph revisions no record pins and not current are removed", removed)
return nil
}
// hearPins registers the seat mesh-delivery's four events on the assigner's hub.
func hearPins(hub *process.EventHub, k *pinKeeper) error {
for _, s := range []string{mdseat.SubjectPinned, mdseat.SubjectFreed, mdseat.SubjectHeadsReplaced, mdseat.SubjectPinsStated} {
if err := hub.Handle(s, k.hear); err != nil {
return fmt.Errorf("%s: %w", s, err)
}
}
return nil
}
// pinsTools are assigner_pins and assigner_unpin over the runtime's states.
func pinsTools(st store, ask func(string, any) (json.RawMessage, error)) []stdio.Tool {
return []stdio.Tool{{Name: pinsTool, Description: "the assignment graph revisions the delivery records pin (design 47 G §579): per revision " +
"the records that pin it and whether it was missing when pinned, the newest revision kept (current, never removed), when the state " +
"was made, the last pins-stated applied, and the state's fill against its cap of 4 MiB.",
Run: func(map[string]any) (any, error) { return readPins(st) }},
{Name: unpinTool, Description: "Remove one record from a revision's pins, for a freed that was lost (design 47 G §579); written in the " +
"hand-act log first. The removal rule then applies.",
Input: schema(map[string]string{"revision": "the assignment graph revision's number", "record": "the delivery record's id",
"why": "why it is removed by hand"}, []string{"revision", "record", "why"}),
Run: func(a map[string]any) (any, error) {
record, _ := a["record"].(string)
why, _ := a["why"].(string)
return unpin(st, ask, fmt.Sprint(a["revision"]), record, why, stdio.CallerOf(a))
}}}
}
// readPins is what assigner_pins answers.
func readPins(st store) (any, error) {
ks, err := st.keys(pinsState)
if err != nil {
return nil, err
}
sort.Strings(ks)
revisions := map[string]PinsValue{}
out := map[string]any{"revisions": revisions}
var newest int64
bytes := 0
for _, key := range ks {
raw, _, found, err := st.get(pinsState, key)
if err != nil || !found {
continue
}
bytes += len(key) + len(raw)
if r, ok := pinRevision(key); ok {
var v PinsValue
if json.Unmarshal(raw, &v) == nil {
revisions[key] = v
}
newest = max(newest, r)
continue
}
out[key] = json.RawMessage(raw)
}
out["fill"] = meshwants.Fill{Store: pinsState, Bytes: int64(bytes), Cap: pinsCapBytes}
if newest > 0 {
out["newest-pinned"] = newest
}
return out, nil
}
// unpin removes one record from one revision's list, the hand act recorded first.
func unpin(st store, ask func(string, any) (json.RawMessage, error), revision, record, why, by string) (any, error) {
r, err := strconv.ParseInt(strings.TrimPrefix(revision, "trunk."), 10, 64)
switch {
case err != nil || r <= 0:
return nil, fmt.Errorf("revision %q is no assignment graph revision's number", revision)
case record == "" || why == "":
return nil, errors.New("name the record and say why")
case by == "" || by == stdio.Unproven:
return nil, errors.New("an unpin names who asks it, and the runtime proved no caller")
}
key := pinKey(r)
raw, _, found, err := st.get(pinsState, key)
if err != nil {
return nil, err
}
var v PinsValue
if found {
if err := json.Unmarshal(raw, &v); err != nil {
return nil, err
}
}
i := slices.Index(v.Records, record)
if i < 0 {
return nil, fmt.Errorf("the record %s does not pin revision %d: %s lists its pins", record, r, pinsTool)
}
taken, err := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: unpinTool + " " + key + " " + record,
Why: why + " (" + by + ")", Cause: "unpin"})
if err != nil {
return nil, fmt.Errorf("the hand act is not recorded, so nothing is removed: %w", err)
}
v.Records = slices.Delete(v.Records, i, i+1)
value, _ := json.Marshal(v)
if err := st.put(pinsState, key, value); err != nil {
return nil, err
}
return map[string]any{"revision": r, "unpinned": record, "records": v.Records, "hand-act": taken.Log}, nil
}
// watchUnpins judges again when a revision's pins change in the state itself, as assigner_unpin writes them from the tool
// process: the assigner's own state, heard as it changes, never on a timer.
func (k *pinKeeper) watchUnpins(kv jetstream.KeyValue) {
w, err := kv.WatchAll(k.ctx, jetstream.UpdatesOnly())
if err != nil {
k.logf("the state %s is not watched, so a hand unpin is judged at the next freed or revision: %v", pinsState, err)
return
}
go func() {
defer func() { _ = w.Stop() }()
for e := range w.Updates() {
if e == nil {
continue
}
if _, ok := pinRevision(e.Key()); ok && e.Operation() == jetstream.KeyValuePut {
k.judge()
}
}
}()
}
// inUseOf says a want or a running build names the module's fingerprint, as the world holds them now.
func inUseOf(w *world) func(module, fp string) bool {
return func(module, fp string) bool {
for _, want := range w.wantsOf()[module] {
if want.Fp == fp {
return true
}
}
for _, mods := range w.runningNow() {
if r, ok := mods[module]; ok && r.Fingerprint == fp {
return true
}
}
return false
}
}
func pinsBucket() string { return assignment.Module() + "_" + pinsState }
@@ -1,119 +0,0 @@
package main
import (
"context"
"encoding/json"
"testing"
"time"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
mdseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assigngraph"
)
func (m memObjects) Delete(_ context.Context, name string) error { delete(m, name); return nil }
// The pins of the assignment graph revisions (design 47 G §153, §178, §579; A-169): a revision two records pin stays until
// both free it; a record pinning twice pins once; the current revision is never removed; nothing is removed before a
// pins-stated said after the state was made, nor while the consumer is behind; a revision no longer held is pinned missing,
// with the finding pin-missing; an entry another kept revision lists, or a want or a running build names, stays.
func TestPinsKeepARevisionWhileARecordPinsIt(t *testing.T) {
objects := memObjects{}
entry := func(module, fp string) string {
e := meshwants.AssignmentEntry{Node: "n1", Module: module, Fp: fp}
d := assigngraph.EntryDigest(e)
raw, _ := json.Marshal(e)
objects[assigngraph.EntryObject(d)] = raw
return d
}
shared, old, running := entry("a", "fp-a"), entry("b", "fp-b1"), entry("c", "fp-c1")
revision := func(n int64, digests ...string) {
rev := meshwants.AssignmentRevision{Revision: n}
for _, d := range digests {
rev.Entries = append(rev.Entries, meshwants.NodeEntryRef{Node: "n1", Digest: d})
}
raw, _ := json.Marshal(rev)
objects[assigngraph.TrunkObject(n)] = raw
}
// A run's candidate lists an entry no trunk revision lists (A-201): it stays while the candidate is kept.
cand := entry("d", "fp-d2")
candRaw, _ := json.Marshal(meshwants.AssignmentRevision{Revision: 1, Kind: "candidate", Group: "run-7", Heads: "h1",
Entries: []meshwants.NodeEntryRef{{Node: "n1", Module: "d", Digest: cand}}})
objects[assigngraph.CandidateObject("run-7", "h1")] = candRaw
revision(1, shared, old, running)
revision(2, shared, old)
revision(3, shared)
var found []mcseat.Finding
clock := time.Date(2026, 11, 12, 10, 0, 0, 0, time.UTC)
k := &pinKeeper{ctx: context.Background(), st: newMemEnds(), objects: objects, now: func() time.Time { return clock },
logf: t.Logf,
inUse: func(module, fp string) bool { return module == "c" && fp == "fp-c1" },
findings: assign.Findings{Put: func(f mcseat.Finding) error { found = append(found, f); return nil }}}
if err := k.made(); err != nil {
t.Fatal(err)
}
say := func(subject string, body any) {
t.Helper()
raw, _ := json.Marshal(body)
if err := k.apply(subject, raw); err != nil {
t.Fatal(err)
}
k.judge()
}
pins := func(record string, revs ...int64) mdseat.Pins { return mdseat.Pins{Record: record, Assignment: revs} }
has := func(n int64) bool { _, ok := objects[assigngraph.TrunkObject(n)]; return ok }
k.caught = true
say(mdseat.SubjectPinned, pins("r1", 1))
say(mdseat.SubjectPinned, pins("r1", 1))
say(mdseat.SubjectPinned, pins("r2", 1))
say(mdseat.SubjectFreed, pins("r9", 2)) // nothing pins 2, but no pins-stated since made: nothing removed
if !has(2) {
t.Fatal("revision 2 was removed before a pins-stated said after the state was made")
}
k.caught = false
say(mdseat.SubjectPinsStated, mdseat.PinsStatedEvent{At: clock.Add(time.Minute)})
if !has(2) {
t.Fatal("revision 2 was removed while the consumer was behind")
}
k.caught = true
say(mdseat.SubjectFreed, pins("r1", 1))
if has(2) {
t.Error("revision 2, neither pinned nor current, is kept")
}
if !has(1) || !has(3) {
t.Error("revision 1 (pinned by r2) or 3 (current) was removed")
}
raw, _, _ := k.st.get(pinKey(1))
if string(raw) != `{"records":["r2"]}` {
t.Errorf("the pins of 1 = %s, want r2 alone (r1 pinned twice pins once, and freed once)", raw)
}
say(mdseat.SubjectFreed, pins("r2", 1))
if has(1) {
t.Error("revision 1 is kept once both records freed it")
}
if _, ok := objects[assigngraph.EntryObject(old)]; ok {
t.Error("the entry no kept revision lists and nothing names is kept")
}
if _, ok := objects[assigngraph.EntryObject(running)]; !ok {
t.Error("the entry of a running build is removed")
}
if _, ok := objects[assigngraph.EntryObject(shared)]; !ok {
t.Error("the entry the current revision lists is removed")
}
if _, ok := objects[assigngraph.EntryObject(cand)]; !ok {
t.Error("the entry a run's kept candidate lists is removed")
}
say(mdseat.SubjectHeadsReplaced, mdseat.HeadsReplacedEvent{})
if _, ok := objects[assigngraph.CandidateObject("run-7", "h1")]; !ok {
t.Error("the gate's heads-replaced dropped a run's candidate")
}
say(mdseat.SubjectPinned, pins("r3", 2))
raw, _, _ = k.st.get(pinKey(2))
if string(raw) != `{"records":["r3"],"missing":true}` || len(found) != 1 || found[0].Kind != pinMissingKind || found[0].ID != "2" {
t.Errorf("a pin of a revision no longer held = %s, findings %+v", raw, found)
}
}
@@ -16,18 +16,20 @@ import (
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mprc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
mpcat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-assigner/internal/assign"
)
// The wants follow a passed pipeline (design 47 G §29, §35, §325, §352): the runner says each pipeline's end as
// pipeline-ended, and a group's as group-ended. The assigner keeps, in its old state `written`, one key per module and
// watched branch: the source revision and end count of the newest end it heard for it, and whether the wants were written
// from it. A newer end replaces it by compare-and-set; an older one, or the same one again, writes nothing.
// The wants follow a passed pipeline (design 47 H §750): the runner says each pipeline's end once, per source revision, as
// pipeline-ended on `pipeline-ended.<branch>.<source>`, naming per mover its fingerprint, its released version and its tier
// (§746). The assigner keeps, in its state `written` (G §352), one key per module and watched branch: the newest released
// version of the module on that branch whose pipeline passed, with the pipeline and the mover's rank in it. A newer version
// replaces it by compare-and-set; an older one, the same one again, a failed pipeline and a run on request write nothing.
// writtenState is the assigner's state of the pipeline ends its wants follow; writtenTool reads it.
const (
@@ -35,23 +37,17 @@ const (
writtenTool = "assigner_written"
)
// pipelineEndedSubject and groupEndedSubject are the runner's events the assigner consumes.
const (
pipelineEndedSubject = "mesh.seat." + mpr.Seat + ".event." + mpr.EventPipelineEnded + ".>"
groupEndedSubject = "mesh.seat." + mpr.Seat + ".event." + mpr.EventGroupEnded + ".>"
)
// pipelineEndedSubject is the runner's event the assigner consumes.
const pipelineEndedSubject = "mesh.seat." + mpr.Seat + ".event." + mpr.EventPipelineEnded + ".>"
// End is one value of the state written: the newest end of a module's pipeline on a watched branch.
// End is one value of the state written: the newest version of a module released on a watched branch whose pipeline passed.
type End struct {
Pipeline string `json:"pipeline"`
Source int64 `json:"source"`
EndCount int64 `json:"end-count"`
Fp string `json:"fp"`
Commit string `json:"commit,omitempty"`
// Passed: the pipeline ended passed, every required step passed (and, for a group, the group ended passed).
Passed bool `json:"passed"`
// Written: the wants were written from this end.
Written bool `json:"written,omitempty"`
Why string `json:"why,omitempty"`
Version string `json:"version"`
// Rank is the mover's tier in the pipeline's delivery order, from 1 (§746, §750).
Rank int64 `json:"rank"`
}
// endKey is the key of a module on a watched branch, each name escaped as a key's name is.
@@ -59,15 +55,10 @@ func endKey(module, branch string) string {
return keys.EscapeName(module) + "." + keys.EscapeName(branch)
}
// newer says whether an end of (source, count) replaces the one kept: a later source revision, or the same one ended again.
func (e End) newer(source, count int64) bool {
return source > e.Source || (source == e.Source && count > e.EndCount)
}
// requiredPassed says whether no required step of a pipeline stands failed, unasked or unended (design 47 G §30: a failed
// optional step holds nothing).
func requiredPassed(p mpr.PipelineEnded) bool {
for _, s := range p.Steps {
// moverPassed says whether no required step of a mover stands failed, unasked or unended: a required step ending "nothing to
// run" counts as passed, as a bypassed one does (design 47 H §746, H-A-3); a failed optional step holds nothing (G §30).
func moverPassed(m mpr.MoverEnded) bool {
for _, s := range m.Steps {
if s.Status != mpr.StepResultStatusRequired {
continue
}
@@ -78,7 +69,7 @@ func requiredPassed(p mpr.PipelineEnded) bool {
return false
}
}
return p.Outcome == mpr.PipelineEndedOutcomePassed
return true
}
// endStore is the kept state written: the bus bucket, or a test's.
@@ -139,19 +130,19 @@ func isWrongSequence(err error) bool {
return errors.As(err, &ae) && ae.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence
}
// pipelineEnds follows the runner's ends. A nil *pipelineEnds is the trunk-only want, kept until the first registrations
// (design 47 G §95, §353).
// pipelineEnds follows the runner's ends; a nil *pipelineEnds keeps none, and no want is written from it.
type pipelineEnds struct {
st endStore
wake func()
logf func(string, ...any)
// on is whether the wants follow the ends: set by the first registration heard, never before (design 47 G §95, §353).
// on says the first registration was heard (design 47 G §353), said once in the journal.
on atomic.Bool
mu sync.Mutex
// firsts is, by module, the first watched branch of its registration, the branch of a placement that names none (design
// 47 G §14).
firsts map[string]string
// runEnded is told a run on request's group-ended, its group `run-<id>`, to drop the run's candidate (A-235); nil drops nothing.
// runEnded is told a run on request's pipeline-ended, its group `run-<id>`, to drop the run's candidate (A-235, H-A-58); nil
// drops nothing.
runEnded func(group string) error
// left is told the key of a pipeline ended superseded or cancelled, whose composition steps leave the runs they read
// (design 47 G §723, A-469n); none tells nothing. Set once the composition steps are served, after the hub starts.
@@ -160,110 +151,121 @@ type pipelineEnds struct {
onBranches func([]string)
// trunk names the first watched branch, the trunk's (design 47 G §507), the branch of a module registered with none.
trunk func() string
// runner is the module holding the seat mesh-pipeline-runner, as the last pass read it from the acting edges (A-334); nil
// until a pass read the graph.
runner atomic.Pointer[string]
// held are the group members' pipeline-ended heard before a pass read the graph, by their body: unacknowledged, so the
// bus keeps them, and taken once the runner is read (A-334), never dropped for the runner not being known yet.
held map[string]jetstream.Msg
}
// setRunner keeps the module holding the seat mesh-pipeline-runner, read by the pass from the acting edges (A-334); the
// first one takes the group members' ends held until the graph was read.
func (e *pipelineEnds) setRunner(module string) {
e.mu.Lock()
first := e.runner.Swap(&module) == nil
held := e.held
e.held = nil
e.mu.Unlock()
if !first {
return
}
for _, m := range held {
e.hearPipeline(m)
}
}
// holdUntilRead keeps m, a group member's pipeline-ended, until a pass reads the runner from the graph; false when the
// runner is read already, so m is taken now.
func (e *pipelineEnds) holdUntilRead(m jetstream.Msg) bool {
e.mu.Lock()
defer e.mu.Unlock()
if e.runner.Load() != nil {
return false
}
if e.held == nil {
e.held = map[string]jetstream.Msg{}
}
e.held[string(m.Data())] = m
e.logf("%s is a group member's end heard before the assignment graph is read: held until a pass reads which module is the runner (A-334)", m.Subject())
return true
}
// isRunner says whether module holds the seat mesh-pipeline-runner, as the last pass read it.
func (e *pipelineEnds) isRunner(module string) bool {
r := e.runner.Load()
return r != nil && *r != "" && *r == module
}
func newPipelineEnds(st endStore, wake func(), logf func(string, ...any)) *pipelineEnds {
return &pipelineEnds{st: st, wake: wake, logf: logf, firsts: map[string]string{}}
}
// take keeps one pipeline's end if it is newer than the one kept for its module and branch, by compare-and-set.
func (e *pipelineEnds) take(p mpr.PipelineEnded, passed bool) error {
key := endKey(p.Module, p.Branch)
for {
raw, rev, err := e.st.get(key)
if err != nil {
return err
// take keeps, for each mover of a passed pipeline that released a version, that version as the newest of its module on the
// pipeline's branch, by compare-and-set on a newer version (design 47 H §750): an older version, or the same one again, writes
// nothing. It says whether any key changed.
func (e *pipelineEnds) take(p mpr.PipelineEnded) (bool, error) {
if p.Outcome != mpr.PipelineEndedOutcomePassed || p.Run != "" {
return false, nil
}
changed := false
for _, m := range p.Movers {
if m.Version == "" || m.Fingerprint == "" || !moverPassed(m) {
continue
}
if raw != nil {
var kept End
if err := json.Unmarshal(raw, &kept); err == nil && !kept.newer(p.Source, p.EndCount) {
return nil // an older end, or this one again
key := endKey(m.Module, p.Branch)
for {
raw, rev, err := e.st.get(key)
if err != nil {
return changed, err
}
if raw != nil {
var kept End
if err := json.Unmarshal(raw, &kept); err == nil && compareVersion(m.Version, kept.Version) <= 0 {
break // an older version, or this one again
}
}
value, err := json.Marshal(End{Pipeline: p.Pipeline, Source: p.Source, Fp: m.Fingerprint, Version: m.Version, Rank: max(m.Tier, 1)})
if err != nil {
return changed, err
}
if err := e.st.swap(key, value, rev); errors.Is(err, errSwapped) {
continue
} else if err != nil {
return changed, err
}
changed = true
break
}
end := End{Source: p.Source, EndCount: p.EndCount, Fp: p.Fingerprint, Commit: p.Commit, Passed: passed, Why: p.Why}
value, err := json.Marshal(end)
if err != nil {
return err
}
if err := e.st.swap(key, value, rev); errors.Is(err, errSwapped) {
continue
} else if err != nil {
return err
}
e.wake()
return nil
}
return changed, nil
}
// markWritten says the wants of module on branch were written from the end it keeps, if that is still the end of fp.
func (e *pipelineEnds) markWritten(module, branch, fp string) {
key := endKey(module, branch)
for {
raw, rev, err := e.st.get(key)
var kept End
if err != nil || raw == nil || json.Unmarshal(raw, &kept) != nil || kept.Fp != fp || kept.Written {
return
}
kept.Written = true
value, _ := json.Marshal(kept)
if err := e.st.swap(key, value, rev); errors.Is(err, errSwapped) {
continue
} else if err != nil {
e.logf("the end of %s on %s is not marked written: %v", module, branch, err)
}
return
// endOf is the end kept for module on branch: the newest version released there whose pipeline passed.
func (e *pipelineEnds) endOf(module, branch string) (End, bool) {
if e == nil {
return End{}, false
}
}
// passedAt says whether the newest end kept for module on branch passed at fp.
func (e *pipelineEnds) passedAt(module, branch, fp string) bool {
raw, _, err := e.st.get(endKey(module, branch))
var kept End
return err == nil && raw != nil && json.Unmarshal(raw, &kept) == nil && kept.Passed && kept.Fp == fp
if err != nil || raw == nil || json.Unmarshal(raw, &kept) != nil || kept.Fp == "" {
return End{}, false
}
return kept, true
}
// target is the build module's wants on branch name (design 47 H §750): the newest version released there whose pipeline
// passed, with the pipeline and the mover's rank; false when none is kept.
func (e *pipelineEnds) target(module, branch string) (assign.Target, bool) {
end, ok := e.endOf(module, e.branchOf(module, branch))
if !ok {
return assign.Target{}, false
}
return assign.Target{Fp: end.Fp, Version: end.Version, Pipeline: end.Pipeline, Rank: end.Rank}, true
}
// hearPipeline takes a pipeline-ended: a passed one's released versions are kept (take), and a run on request's drops the
// run's candidate assignment graph (A-235, H-A-58); a run delivers nothing (design 47 G §29, §64). A pipeline ended superseded or
// cancelled has its composition steps leave the runs they read (G §723).
func (e *pipelineEnds) hearPipeline(m jetstream.Msg) {
p, err := mprc.PipelineEnded(m.Data())
if err != nil {
e.logf("%s is not taken: %v", m.Subject(), err)
_ = m.Ack()
return
}
if left := e.left.Load(); left != nil && (p.Outcome == mpr.PipelineEndedOutcomeSuperseded || p.Outcome == mpr.PipelineEndedOutcomeCancelled) {
if err := (*left)(p.Pipeline); err != nil {
e.logf("%s: the composition steps of %s do not all leave their runs: %v; it is heard again", m.Subject(), p.Pipeline, err)
_ = m.Nak()
return
}
}
if p.Run != "" {
if e.runEnded != nil {
if err := e.runEnded(runGroupPrefix + p.Run); err != nil {
e.logf("%s: %v; it is heard again", m.Subject(), err)
_ = m.Nak()
return
}
}
_ = m.Ack()
return
}
changed, err := e.take(p)
if err != nil {
e.logf("%s: %v; it is heard again", m.Subject(), err)
_ = m.Nak()
return
}
if changed {
e.wake()
}
_ = m.Ack()
}
// hearEnds registers pipeline-ended on the assigner's hub.
func hearEnds(hub *process.EventHub, e *pipelineEnds) error {
if err := hub.Handle(pipelineEndedSubject, e.hearPipeline); err != nil {
return fmt.Errorf("%s: %w", pipelineEndedSubject, err)
}
return nil
}
// branchOf is the watched branch a placement names, else the first of the module's registration, else the trunk's.
@@ -282,76 +284,6 @@ func (e *pipelineEnds) branchOf(module, placed string) string {
return ""
}
// hearPipeline takes a pipeline-ended. A pipeline of a group writes nothing on its own end, but the runner's: in a group
// take-in that changes the runner it is the acting part the other members wait on, so it goes first, wanted on its own end
// once that passed, ahead of the group (A-334, as the node-engine does, §268). A run on request delivers nothing (design 47 G
// §29, §64).
func (e *pipelineEnds) hearPipeline(m jetstream.Msg) {
p, err := mprc.PipelineEnded(m.Data())
if left := e.left.Load(); err == nil && left != nil && (p.Outcome == mpr.PipelineEndedOutcomeSuperseded || p.Outcome == mpr.PipelineEndedOutcomeCancelled) {
if err := (*left)(p.Pipeline); err != nil {
e.logf("%s: the composition steps of %s do not all leave their runs: %v; it is heard again", m.Subject(), p.Pipeline, err)
_ = m.Nak()
return
}
}
switch {
case err != nil:
e.logf("%s is not taken: %v", m.Subject(), err)
case p.Run == "" && p.Group != "" && e.holdUntilRead(m):
return
case p.Run != "" || (p.Group != "" && !e.isRunner(p.Module)):
default:
if err := e.take(p, requiredPassed(p)); err != nil {
e.logf("%s: %v; it is heard again", m.Subject(), err)
_ = m.Nak()
return
}
}
_ = m.Ack()
}
// hearGroup takes a group-ended: each member's end, passed only if the group passed. A run on request's (group `run-<id>`) delivers
// nothing: it drops the run's candidate assignment graph (A-235).
func (e *pipelineEnds) hearGroup(m jetstream.Msg) {
g, err := mprc.GroupEnded(m.Data())
if err != nil {
e.logf("%s is not taken: %v", m.Subject(), err)
_ = m.Ack()
return
}
if strings.HasPrefix(g.Group, runGroupPrefix) {
if e.runEnded != nil {
if err := e.runEnded(g.Group); err != nil {
e.logf("%s: %v; it is heard again", m.Subject(), err)
_ = m.Nak()
return
}
}
_ = m.Ack()
return
}
for _, p := range g.Pipelines {
if err := e.take(p, g.Outcome == mpr.GroupEndedOutcomePassed && requiredPassed(p)); err != nil {
e.logf("%s: %v; it is heard again", m.Subject(), err)
_ = m.Nak()
return
}
}
_ = m.Ack()
}
// hearEnds registers the two events on the assigner's hub.
func hearEnds(hub *process.EventHub, e *pipelineEnds) error {
if err := hub.Handle(pipelineEndedSubject, e.hearPipeline); err != nil {
return fmt.Errorf("%s: %w", pipelineEndedSubject, err)
}
if err := hub.Handle(groupEndedSubject, e.hearGroup); err != nil {
return fmt.Errorf("%s: %w", groupEndedSubject, err)
}
return nil
}
// writtenTools is the tool assigner_written: every key of the state written, with its end.
func writtenTools(st store) []stdio.Tool {
return []stdio.Tool{{Name: writtenTool, Description: "the pipeline ends the assigner's wants follow: per module and watched branch, the source revision and end count of the newest end heard, whether it passed and whether the wants were written from it; with why, why each want is what it is (as assigner_why).",
@@ -383,56 +315,6 @@ func writtenTools(st store) []stdio.Tool {
}}}
}
// branchesOf is the watched branches the placements of module chose, each once; the default branch when none names one.
func (e *pipelineEnds) branchesOf(module string, placed map[string]map[string]string) []string {
if !e.following() {
return nil
}
seen := map[string]bool{}
var out []string
for _, b := range placed[module] {
b = e.branchOf(module, b)
if !seen[b] {
seen[b] = true
out = append(out, b)
}
}
if len(out) == 0 {
out = append(out, e.branchOf(module, ""))
}
sort.Strings(out)
return out
}
// mayWant says whether module's wants may name fp: a pipeline passed at fp on every branch, or the wants name fp already
// (a node placed later, a module taken away).
func (e *pipelineEnds) mayWant(module, fp string, branches []string, wanted map[string]mwc.Want) bool {
if !e.following() {
return true
}
for _, w := range wanted {
if w.Fp == fp {
return true
}
}
for _, b := range branches {
if !e.passedAt(module, b, fp) {
return false
}
}
return true
}
// wrote marks the end the wants of module were written from.
func (e *pipelineEnds) wrote(module, fp string, branches []string) {
if !e.following() {
return
}
for _, b := range branches {
e.markWritten(module, b, fp)
}
}
// following says whether the wants follow the pipeline ends yet; a nil follower never does.
func (e *pipelineEnds) following() bool { return e != nil && e.on.Load() }
@@ -476,7 +358,7 @@ func (e *pipelineEnds) registered(reg mpcat.Registration, gone bool) {
// followRegistrations follows the registrations through the seat mesh-catalog (design 47 G §353): a first view of its verb
// and each registration-changed on the hub. It returns when the hub's handler is registered; the first view is asked on its own
// goroutine, the wants keep the trunk-only form until a registration is heard.
// goroutine.
func followRegistrations(ctx context.Context, hub *process.EventHub, call mcc.Call, e *pipelineEnds) error {
regs := mcc.NewRegistrations(call)
regs.Follower.Logf = e.logf
@@ -4,35 +4,24 @@ import (
"context"
"encoding/json"
"fmt"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
plc "mesh-delivery-assigner/internal/assignment"
"slices"
"strings"
"testing"
"time"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
mpcat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"mesh-delivery-assigner/internal/assigngraph"
plc "mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
mws "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// memEnds is endStore over a map, with revisions.
@@ -72,250 +61,148 @@ func (m *heardMsg) Subject() string { return m.subject }
func (m *heardMsg) Ack() error { m.acked = true; return nil }
func (m *heardMsg) Nak() error { m.naked = true; return nil }
func pipe(module, branch string, source, count int64, fp, outcome string, steps ...mpr.StepResult) mpr.PipelineEnded {
return mpr.PipelineEnded{Module: module, Branch: branch, Source: source, EndCount: count, Fingerprint: fp, Outcome: outcome, Steps: steps,
Pipeline: branch + "." + module, Commit: "c"}
// mover is one mover at its pipeline's end, its required unit step ending outcome.
func mover(module, fp, version string, tier int64, outcome string) mpr.MoverEnded {
return mpr.MoverEnded{Module: module, Fingerprint: fp, Version: version, Tier: tier,
Steps: []mpr.StepResult{{Step: "unit", Status: mpr.StepResultStatusRequired, Outcome: outcome, Module: module}}}
}
func step(status, outcome string) mpr.StepResult {
return mpr.StepResult{Step: "unit", Status: status, Outcome: outcome}
// ended is the pipeline-ended of the take-in of source on branch, its movers given.
func ended(branch string, source int64, outcome string, movers ...mpr.MoverEnded) mpr.PipelineEnded {
return mpr.PipelineEnded{V: 1, Branch: branch, Source: source, EndCount: 1, Outcome: outcome, Movers: movers,
Pipeline: fmt.Sprintf("%s.%d", branch, source)}
}
// Design 47 G §29, §35, §325, §352 (tests 66, 67, 89): a want follows a passed pipeline, by compare-and-set per module and branch.
func TestAWantFollowsAPassedPipeline(t *testing.T) {
const fp1, fp2 = "fp1", "fp2"
// A6/AC1 to AC3 (design 47 H §750, H-A-3): a passed pipeline keeps each mover's released version as the newest of its module on
// the branch, with the pipeline and the mover's tier; a failed pipeline, a mover whose required step failed, one that released
// no version, an older version, the same one again and a run on request keep nothing.
func TestThePassedVersionIsKept(t *testing.T) {
pass, fail := mpr.PipelineEndedOutcomePassed, mpr.PipelineEndedOutcomeFailed
req, opt := mpr.StepResultStatusRequired, mpr.StepResultStatusOptional
ok, failed, nothing := mpr.StepResultOutcomePassed, mpr.StepResultOutcomeFailed, mpr.StepResultOutcomeNothingToRun
for _, c := range []struct {
name string
ends []mpr.PipelineEnded
group *mpr.GroupEnded
module string
branch string
fp string
may bool
name string
ends []mpr.PipelineEnded
want *End // nil: none kept for web on main
}{
{"a failed required step writes no want", []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, fail, step(req, "failed"))}, nil, "web", "main", fp1, false},
{"a pipeline passed writes", []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, pass, step(req, "passed"))}, nil, "web", "main", fp1, true},
{"a failed optional step holds nothing", []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, pass, step(req, "passed"), step(opt, "failed"))}, nil, "web", "main", fp1, true},
{"a pipeline passed at another fingerprint", []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, pass)}, nil, "web", "main", fp2, false},
{"an older revision ending after a newer one writes nothing", []mpr.PipelineEnded{pipe("web", "main", 6, 1, fp2, fail), pipe("web", "main", 5, 1, fp1, pass)}, nil, "web", "main", fp1, false},
{"another branch writes its own", []mpr.PipelineEnded{pipe("web", "main", 6, 1, fp2, fail), pipe("web", "beta", 5, 1, fp1, pass)}, nil, "web", "beta", fp1, true},
{"a later end of the same revision writes", []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, fail, step(req, "failed")), pipe("web", "main", 5, 2, fp1, pass, step(req, "passed"))}, nil, "web", "main", fp1, true},
{"the same end again after a later one keeps the later", []mpr.PipelineEnded{pipe("web", "main", 5, 2, fp1, pass), pipe("web", "main", 5, 1, fp1, fail)}, nil, "web", "main", fp1, true},
{"a member's own end writes nothing", []mpr.PipelineEnded{func() mpr.PipelineEnded { p := pipe("web", "main", 5, 1, fp1, pass); p.Group = "g"; return p }()}, nil, "web", "main", fp1, false},
{"a group ended passed writes its members", nil, &mpr.GroupEnded{Branch: "main", Group: "g", Outcome: mpr.GroupEndedOutcomePassed, Pipelines: []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, pass)}}, "web", "main", fp1, true},
{"a group ended failed writes none", nil, &mpr.GroupEnded{Branch: "main", Group: "g", Outcome: mpr.GroupEndedOutcomeFailed, Pipelines: []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, pass)}}, "web", "main", fp1, false},
{"a passed pipeline keeps its version", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "0.1.2", 2, ok))},
&End{Pipeline: "main.5", Source: 5, Fp: "fp1", Version: "0.1.2", Rank: 2}},
{"nothing to run counts as passed", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, nothing))},
&End{Pipeline: "main.5", Source: 5, Fp: "fp1", Version: "0.1.2", Rank: 1}},
{"a failed pipeline keeps nothing, a passed mover in it neither", []mpr.PipelineEnded{ended("main", 5, fail, mover("web", "fp1", "0.1.2", 1, ok),
mover("db", "fp9", "0.3.0", 1, failed))}, nil},
{"a mover whose required step failed keeps nothing", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, failed))}, nil},
{"no version released keeps nothing", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "", 1, ok))}, nil},
{"an older end heard after a newer one keeps the newer", []mpr.PipelineEnded{ended("main", 6, pass, mover("web", "fp2", "0.1.3", 1, ok)),
ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok))}, &End{Pipeline: "main.6", Source: 6, Fp: "fp2", Version: "0.1.3", Rank: 1}},
{"a newer version replaces", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok)),
ended("main", 6, pass, mover("web", "fp2", "0.1.10", 1, ok))}, &End{Pipeline: "main.6", Source: 6, Fp: "fp2", Version: "0.1.10", Rank: 1}},
{"a run on request keeps nothing", []mpr.PipelineEnded{func() mpr.PipelineEnded {
p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok))
p.Run = "r1"
return p
}()}, nil},
} {
woken := 0
e := newPipelineEnds(newMemEnds(), func() { woken++ }, t.Logf)
e.on.Store(true)
e := newPipelineEnds(newMemEnds(), func() {}, t.Logf)
for _, p := range c.ends {
if p.Group != "" {
continue // heard and ignored, as hearPipeline does
}
if err := e.take(p, requiredPassed(p)); err != nil {
t.Fatal(err)
if _, err := e.take(p); err != nil {
t.Fatalf("%s: %v", c.name, err)
}
}
if g := c.group; g != nil {
for _, p := range g.Pipelines {
if err := e.take(p, g.Outcome == mpr.GroupEndedOutcomePassed && requiredPassed(p)); err != nil {
t.Fatal(err)
}
}
}
if got := e.mayWant(c.module, c.fp, []string{c.branch}, nil); got != c.may {
t.Errorf("%s: may want = %v, want %v", c.name, got, c.may)
got, kept := e.endOf("web", "main")
switch {
case c.want == nil && kept:
t.Errorf("%s: kept %+v", c.name, got)
case c.want != nil && (!kept || got != *c.want):
t.Errorf("%s: kept %+v (%v), want %+v", c.name, got, kept, *c.want)
}
}
// A run on request's group-ended (group run-<id>, A-235) writes no want and drops the run's candidate; a drop that fails is
// heard again.
{
e := newPipelineEnds(newMemEnds(), func() {}, t.Logf)
e.on.Store(true)
var dropped []string
fail := false
e.runEnded = func(group string) error {
dropped = append(dropped, group)
if fail {
return fmt.Errorf("the bucket is not open")
}
return nil
}
raw, _ := json.Marshal(mpr.GroupEnded{V: 1, Branch: "main", Group: "run-7", Outcome: mpr.GroupEndedOutcomePassed,
Pipelines: []mpr.PipelineEnded{func() mpr.PipelineEnded { p := pipe("web", "main", 5, 1, fp1, pass); p.Run = "7"; return p }()}})
m := &heardMsg{data: raw, subject: "mesh.seat.mesh-pipeline-runner.event.group-ended.x"}
e.hearGroup(m)
if e.mayWant("web", fp1, []string{"main"}, nil) || !slices.Equal(dropped, []string{"run-7"}) || !m.acked {
t.Errorf("a run's group-ended: may want %v, dropped %v, acked %v; want no want, run-7 dropped, acked",
e.mayWant("web", fp1, []string{"main"}, nil), dropped, m.acked)
}
fail = true
m = &heardMsg{data: raw, subject: m.subject}
e.hearGroup(m)
if !m.naked {
t.Error("a run's candidate not dropped is not heard again")
// The same end heard again changes nothing and wakes nothing; a run on request's end drops its candidate.
woken := 0
e := newPipelineEnds(newMemEnds(), func() { woken++ }, t.Logf)
var dropped []string
e.runEnded = func(group string) error { dropped = append(dropped, group); return nil }
p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok))
for range 2 {
raw, _ := json.Marshal(p)
m := &heardMsg{data: raw, subject: "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.main.5"}
e.hearPipeline(m)
if !m.acked {
t.Error("the end was not acknowledged")
}
}
// A-334 (§29, §65, §268): in a group take-in that changes the runner, the runner's member (the holder of the seat
// mesh-pipeline-runner, read from the acting edges) is wanted on its own pipeline-ended once it passed, ahead of the group;
// the other members wait for group-ended.
{
e := newPipelineEnds(newMemEnds(), func() {}, t.Logf)
e.on.Store(true)
heard := func(p mpr.PipelineEnded) *heardMsg {
p.Group = "g"
raw, _ := json.Marshal(p)
m := &heardMsg{data: raw, subject: "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.x"}
e.hearPipeline(m)
return m
}
hear := func(p mpr.PipelineEnded) {
if m := heard(p); !m.acked {
t.Errorf("the pipeline-ended of %s is not acknowledged", p.Module)
}
}
// The runner's end heard before a pass read the graph is held, unacknowledged, and taken once the runner is read.
early := heard(pipe("runner", "main", 5, 1, "fp-runner", pass, step(req, "passed")))
if early.acked || early.naked || e.mayWant("runner", "fp-runner", []string{"main"}, nil) {
t.Fatalf("the runner's end before the graph is read: acked %v, naked %v, wanted %v; want it held", early.acked, early.naked,
e.mayWant("runner", "fp-runner", []string{"main"}, nil))
}
e.setRunner(assigngraph.Runner([]mws.AssignmentEntry{{Node: "ace", Module: "web", Edges: []mws.Edge{
{Kind: assigngraph.EdgeActing, Node: "ace", Module: "runner", Seat: mpr.Seat, Step: "pipeline"}}}}))
if !early.acked {
t.Error("the runner's end held until the graph was read is not taken once it is")
}
hear(pipe("web", "main", 5, 1, fp1, pass, step(req, "passed")))
if !e.mayWant("runner", "fp-runner", []string{"main"}, nil) || e.mayWant("web", fp1, []string{"main"}, nil) {
t.Errorf("a group changing the runner: may want the runner %v, web %v; want the runner ahead of the group and web not yet",
e.mayWant("runner", "fp-runner", []string{"main"}, nil), e.mayWant("web", fp1, []string{"main"}, nil))
}
hear(pipe("runner", "main", 6, 1, "fp-runner-2", fail, step(req, "failed")))
if e.mayWant("runner", "fp-runner-2", []string{"main"}, nil) {
t.Error("the runner's member that failed is wanted")
}
}
// Design 47 G A-428: the runner's first start skips the gate while no module claiming the seat runs; another module never
// does, and a runner running anywhere (stopped ones aside) closes it; with no running heard the gate stands.
manifests := map[string]catalogue.Manifest{"runner": {Claims: []catalogue.Claim{{Name: mpr.Seat}}}, "web": {}}
none := map[string]map[string]nodeengine.Running{"ace": {"web": {Fingerprint: fp1}}}
if !runnerFirstStart("runner", manifests, none) || runnerFirstStart("web", manifests, none) || runnerFirstStart("runner", manifests, nil) {
t.Error("the runner's first start: the runner skips the gate, another module or an unknown world does not")
}
if runnerFirstStart("runner", manifests, map[string]map[string]nodeengine.Running{"novox": {"runner": {Fingerprint: "fp-runner"}}}) ||
!runnerFirstStart("runner", manifests, map[string]map[string]nodeengine.Running{"novox": {"runner": {Stopped: true}}}) {
t.Error("a runner running closes the gate's opening; one stopped does not")
}
// A want that names the fingerprint already is not held by a later failure (a node placed afterwards gets it).
e := newPipelineEnds(newMemEnds(), func() {}, t.Logf)
e.on.Store(true)
if !e.mayWant("web", fp1, []string{"main"}, map[string]mwc.Want{"g14": {Fp: fp1}}) {
t.Error("a want naming the fingerprint was held")
}
// markWritten keeps the end and says it was written from; a newer end replaces it unwritten.
st := newMemEnds()
e = newPipelineEnds(st, func() {}, t.Logf)
e.on.Store(true)
_ = e.take(pipe("web", "main", 5, 1, fp1, pass), true)
e.wrote("web", fp1, []string{"main"})
var end End
_ = json.Unmarshal(st.vals[endKey("web", "main")], &end)
if !end.Written || end.Source != 5 {
t.Errorf("end = %+v, want written at source 5", end)
}
_ = e.take(pipe("web", "main", 6, 1, fp2, pass), true)
end = End{}
_ = json.Unmarshal(st.vals[endKey("web", "main")], &end)
if end.Written || end.Source != 6 {
t.Errorf("a newer end = %+v, want unwritten at source 6", end)
run := p
run.Run = "r1"
raw, _ := json.Marshal(run)
e.hearPipeline(&heardMsg{data: raw, subject: "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.main.5"})
if woken != 1 || !slices.Equal(dropped, []string{runGroupPrefix + "r1"}) {
t.Errorf("woken %d (want 1), dropped %v (want the run's candidate)", woken, dropped)
}
}
// Design 47 G §29: the wants of a pass follow a pipeline: with the follower on, a module whose fingerprint no pipeline passed at
// gets no want and one a pipeline passed at gets it; with it off (nil, the trunk-only want) the pass is as before.
func TestAPassWritesNoWantWithoutAPassedPipeline(t *testing.T) {
// A6/AC1, A5/AC1 (design 47 H §750): a pass writes, on every node the module is assigned to, the want naming the newest version
// released on its branch whose pipeline passed, with the pipeline and the rank; with none passed it writes none. No grant,
// canary, provider or composition is asked for.
func TestAPassWritesTheWantOfThePassedVersion(t *testing.T) {
fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64)
for _, c := range []struct {
name string
follow bool
passed bool
want bool
}{{"the trunk-only want", false, false, true}, {"no pipeline passed", true, false, false}, {"a pipeline passed", true, true, true}} {
for _, passed := range []bool{false, true} {
wantsKV := &memKV{values: map[string][]byte{}}
p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, time.Now())
p.say = func(oc.Condition) error { return nil }
mdigest := strings.Repeat("e", 64)
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: mdigest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
putModuleRaw(p.w, trunkBranch, "web", mod)
p.w.manifests[mdigest] = parsed{module: "web", manifest: catalogue.Manifest{Module: "web"}}
endDone(t, p.w, "web", fp, mdigest)
gkey, _ := grantsstate.PartGrantKey("web", fp)
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{})
p.w.put(buckets.Grants, gkey, grant)
digest := "sha256:" + strings.Repeat("c", 64)
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Composed: digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
p.w.put(buckets.Have, "g14.web", have)
composes(t, p.w, "g14", digest)
p.w.manifests[mdigest] = parsed{module: "web", manifest: catalogue.Manifest{Module: "web", Requires: []string{"database"}}}
placeOn(t, p.w, "g14", "web")
on := true
p.w.heardPresence("g14", !on, time.Now())
if c.follow {
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
p.ends.trunk = p.w.trunkName
p.ends.on.Store(true)
if c.passed {
_ = p.ends.take(pipe("web", trunkBranch, 5, 1, fp, mpr.PipelineEndedOutcomePassed), true)
}
placeOn(t, p.w, "ace", "web")
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
p.ends.trunk = p.w.trunkName
if passed {
_, _ = p.ends.take(ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("web", fp, "0.1.2", 2, mpr.StepResultOutcomePassed)))
}
p.wants(context.Background())
key, _ := mwc.WantKey("g14", "web")
if _, got := wantsKV.values[key]; got != c.want {
t.Errorf("%s: a want written = %v, want %v", c.name, got, c.want)
}
if c.follow && c.passed {
raw, _, _ := p.ends.st.get(endKey("web", trunkBranch))
var end End
_ = json.Unmarshal(raw, &end)
if !end.Written {
t.Errorf("%s: the end was not marked written", c.name)
for _, node := range []string{"g14", "ace"} {
key, _ := mwc.WantKey(node, "web")
raw, got := wantsKV.values[key]
if got != passed {
t.Errorf("passed %v: a want on %s written = %v", passed, node, got)
continue
}
if !passed {
continue
}
if w, err := mwc.DecodeWant(raw); err != nil || w.Fp != fp || w.Version != "0.1.2" || w.Pipeline != trunkBranch+".5" || w.Rank != 2 {
t.Errorf("the want on %s = %+v (%v)", node, w, err)
}
}
}
}
// Design 47 G §95, §130, §353, §352 (test 100): the wants keep the trunk-only form until the first registration is heard and
// switch on it; a branch a registration no longer watches loses its end, and its first branch is a placement's default.
func TestTheWantsSwitchOnTheFirstRegistration(t *testing.T) {
// Design 47 G §353, §352: a registration's first branch is a placement's default, and a branch it no longer watches loses its
// end; a registration gone loses them all.
func TestARegistrationNamesTheDefaultBranch(t *testing.T) {
st := newMemEnds()
woken := 0
e := newPipelineEnds(st, func() { woken++ }, t.Logf)
e.trunk = func() string { return trunkBranch }
_ = e.take(pipe("web", "main", 5, 1, "fp1", mpr.PipelineEndedOutcomeFailed), false)
_ = e.take(pipe("web", "beta", 5, 1, "fp1", mpr.PipelineEndedOutcomePassed), true)
if e.following() || !e.mayWant("web", "fp1", []string{"main"}, nil) {
t.Fatal("the follower held a want before any registration")
}
pass := mpr.PipelineEndedOutcomePassed
_, _ = e.take(ended("main", 5, pass, mover("web", "fp1", "0.1.1", 1, mpr.StepResultOutcomePassed)))
_, _ = e.take(ended("beta", 5, pass, mover("web", "fp2", "0.1.2", 1, mpr.StepResultOutcomePassed)))
e.registered(mpcat.Registration{Module: "web", Branches: []string{"beta"}}, false)
if !e.following() || woken == 0 {
t.Error("the first registration did not switch the wants on")
}
if e.mayWant("web", "fp1", []string{"main"}, nil) || !e.mayWant("web", "fp1", []string{"beta"}, nil) {
t.Error("after the switch the wants did not follow the ends")
}
if len(st.vals) != 1 || e.branchOf("web", "") != "beta" || e.branchOf("other", "") != trunkBranch {
if woken == 0 || len(st.vals) != 1 || e.branchOf("web", "") != "beta" || e.branchOf("other", "") != trunkBranch {
t.Errorf("ends kept %d, default branch %q: an unwatched branch's end stayed, or the default is wrong", len(st.vals), e.branchOf("web", ""))
}
if tg, ok := e.target("web", ""); !ok || tg.Fp != "fp2" {
t.Errorf("the default branch's target = %+v %v", tg, ok)
}
e.registered(mpcat.Registration{Module: "web"}, true)
if len(st.vals) != 0 {
t.Errorf("a registration gone left %d ends", len(st.vals))
}
}
// Design 47 G §14, §184 (test 68): a node's entry is computed from the catalog half of the branch its placement names, so a
// change on `beta` moves only the want of the node whose placement chose `beta`.
func TestACommitOnBetaMovesOnlyTheNodeThatChoseBeta(t *testing.T) {
// Design 47 G §14, §184 (test 68): a node's want follows the branch its placement names, so a version passed on `beta` moves only
// the want of the node whose placement chose `beta`.
func TestAVersionOnBetaMovesOnlyTheNodeThatChoseBeta(t *testing.T) {
fp := func(c string) string { return meshcatalogclient.SourcePrefix2 + strings.Repeat(c, 64) }
wantsKV := &memKV{values: map[string][]byte{}}
p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, time.Now())
@@ -323,15 +210,8 @@ func TestACommitOnBetaMovesOnlyTheNodeThatChoseBeta(t *testing.T) {
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
p.ends.trunk = p.w.trunkName
p.ends.registered(mpcat.Registration{Module: "web", Branches: []string{"main", "beta"}}, false)
digest := "sha256:" + strings.Repeat("c", 64)
on := true
for _, node := range []string{"g14", "ace"} {
composes(t, p.w, node, digest)
p.w.heardPresence(node, !on, time.Now())
}
for node, branch := range map[string]string{"g14": "", "ace": "beta"} {
pl := mma.Assignments{Node: node, Modules: []mma.PlacedModule{{Module: "web", Branch: branch}}}
values, err := mmac.AssignmentsValues(pl)
values, err := mmac.AssignmentsValues(mma.Assignments{Node: node, Modules: []mma.PlacedModule{{Module: "web", Branch: branch}}})
if err != nil {
t.Fatal(err)
}
@@ -339,24 +219,10 @@ func TestACommitOnBetaMovesOnlyTheNodeThatChoseBeta(t *testing.T) {
p.w.put(plc.AssignmentsBucket(), key, raw)
}
}
// publish makes the half of branch name web at fingerprint f: built, granted and passed by its pipeline.
publish := func(branch, f string, source int64) {
mdigest := strings.Repeat("e", 62) + fmt.Sprintf("%02d", source)
if branch == "beta" {
mdigest = strings.Repeat("f", 62) + fmt.Sprintf("%02d", source)
}
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: f, Manifest: mdigest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
putModuleRaw(p.w, branch, "web", mod)
p.w.manifests[mdigest] = parsed{module: "web", manifest: catalogue.Manifest{Module: "web"}}
endDone(t, p.w, "web", f, mdigest)
gkey, _ := grantsstate.PartGrantKey("web", f)
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{})
p.w.put(buckets.Grants, gkey, grant)
_ = p.ends.take(pipe("web", branch, source, 1, f, mpr.PipelineEndedOutcomePassed), true)
}
run := func(node, f string) {
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: f, Composed: digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
p.w.put(buckets.Have, node+".web", have)
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp("a"), Manifest: strings.Repeat("e", 64), Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
putModuleRaw(p.w, "main", "web", mod)
passed := func(branch, f, version string, source int64) {
_, _ = p.ends.take(ended(branch, source, mpr.PipelineEndedOutcomePassed, mover("web", f, version, 1, mpr.StepResultOutcomePassed)))
}
fpWant := func(node string) string {
key, _ := mwc.WantKey(node, "web")
@@ -366,18 +232,16 @@ func TestACommitOnBetaMovesOnlyTheNodeThatChoseBeta(t *testing.T) {
}
return w.Fp
}
publish("main", fp("a"), 5)
publish("beta", fp("b"), 5)
run("g14", fp("a"))
run("ace", fp("b"))
passed("main", fp("a"), "0.1.1", 5)
passed("beta", fp("b"), "0.2.1", 5)
p.wants(context.Background())
if fpWant("g14") != fp("a") || fpWant("ace") != fp("b") {
t.Fatalf("each node is wanted at its branch's build: g14 %s, ace %s", fpWant("g14"), fpWant("ace"))
}
publish("beta", fp("d"), 6)
passed("beta", fp("d"), "0.2.2", 6)
p.wants(context.Background())
if fpWant("g14") != fp("a") || fpWant("ace") != fp("d") {
t.Errorf("a commit on beta moves only ace: g14 %s (want main's), ace %s (want beta's new)", fpWant("g14"), fpWant("ace"))
t.Errorf("a version on beta moves only ace: g14 %s (want main's), ace %s (want beta's new)", fpWant("g14"), fpWant("ace"))
}
}
@@ -411,40 +275,3 @@ func TestTheCatalogHalfIsTakenInPerBranch(t *testing.T) {
t.Error("a module taken out of beta's half leaves main's")
}
}
// Design 47 G §379, §526: a want held at an older build, the trunk's newer fingerprint having passed no pipeline, is rewritten
// with that build's released version (and parts for an engine that reads them), the rest of the want kept; no want names the
// unpassed fingerprint (novox 2026-11-16, mesh-hand-acts 0.1.13 at 4eb3cf48).
func TestAWantHeldAtAnOlderBuildNamesItsReleaseWithoutAPassedPipeline(t *testing.T) {
old := meshcatalogclient.SourcePrefix2 + strings.Repeat("4", 64)
fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("9", 64)
wantsKV := &memKV{values: map[string][]byte{}}
p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, time.Now())
p.say = func(oc.Condition) error { return nil }
mdigest := strings.Repeat("e", 64)
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: mdigest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
putModuleRaw(p.w, trunkBranch, "web", mod)
p.w.manifests[mdigest] = parsed{module: "web", manifest: catalogue.Manifest{Module: "web"}}
endRaw, _ := json.Marshal(builds.BuildEnded{Module: "web", Fingerprint: old, Outcome: builds.BuildDone, Manifest: mdigest, Made: map[string]string{"tools": "sha256:1c"}})
if err := p.w.ends.put(endRaw); err != nil {
t.Fatal(err)
}
rel, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: "web", Source: 962, Version: "0.1.13", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "tools", Digest: "sha256:1c"}}})
if err := p.w.released.put(rel); err != nil {
t.Fatal(err)
}
placeOn(t, p.w, "g14", "web")
key, _ := mwc.WantKey("g14", "web")
held, _ := mwc.EncodeWant(mwc.Want{Fp: old, Self: true})
wantsKV.values[key] = held
p.w.put(buckets.Wants(), key, held)
p.w.heardPresence("g14", false, time.Now())
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
p.ends.trunk = p.w.trunkName
p.ends.on.Store(true)
p.wants(context.Background())
got, err := mwc.DecodeWant(wantsKV.values[key])
if err != nil || got.Fp != old || got.Version != "0.1.13" || !got.Self {
t.Errorf("the held want = %+v, %v; want %s at 0.1.13, self kept", got, err, old)
}
}
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"strings"
"testing"
"time"
@@ -67,6 +68,8 @@ func newOrderFixture(t *testing.T, mods []orderMod, history ...map[string]string
return nil
}
digest := "sha256:" + strings.Repeat("c", 64)
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
p.ends.trunk = p.w.trunkName
pl := mma.Assignments{Node: "ace"}
forms := map[string]int{} // by fingerprint: each module's own
modules := []string{}
@@ -92,6 +95,8 @@ func newOrderFixture(t *testing.T, mods []orderMod, history ...map[string]string
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{})
p.w.put(buckets.Grants, gkey, grant)
f.runs(m.name, m.run)
// The trunk's build is released with a passed pipeline (design 47 H §750).
_, _ = p.ends.take(ended(trunkBranch, 1, mpr.PipelineEndedOutcomePassed, mover(m.name, m.trunk, "0.1.1", 1, mpr.StepResultOutcomePassed)))
wkey, _ := mwc.WantKey("ace", m.name)
old, _ := mwc.EncodeWant(mwc.Want{Fp: m.run, Self: true})
p.w.put(buckets.Wants(), wkey, old)
@@ -136,14 +136,6 @@ func (p *passer) puts(ctx context.Context) {
world := p.putWorld(node, module, put, manifest, assigned[node])
world.Snapshot, world.Snapshotting, world.SnapshotFailed = p.acts.state(id)
next := assign.StepPut(world)
// A held bus whose user list the rules moved on (ADR 0539 §7): the put follows the composition its want names.
if w, ok := wanted[module][node]; ok && w.Put && w.Fp == next.Put.FP && w.Composed != "" && w.Composed != next.Put.Composed &&
(next.Put.Step == puts.Wanted || next.Put.Step == puts.Holding) {
if d, _ := p.composedFor(node, module, w.Fp); d != nil && d.UserListOnly && d.Digest == w.Composed {
next.Put = next.Put.Said(world.Now, next.Put.Step, fmt.Sprintf("the bus's user list changed: the want names composition %s", shortHex(w.Composed)))
next.Put.Composed = w.Composed
}
}
// The bus's want is written only for the act the operator's yes bound (ADR 0546 clause 8).
if next.Put.Step == puts.Wanted && put.Step != puts.Wanted && world.Bus && next.Put.Ask != "" {
if err := p.performs(next.Put.Ask, assign.PutAct(node, module, next.Put.FP, next.Put.Composed)); err != nil {
@@ -0,0 +1,140 @@
package main
import (
"context"
"fmt"
"slices"
"strconv"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/assigngraph"
)
// The assignment graph revisions are removed when they are no longer current (design 47 H §764, H-A-86): the gate's pins of
// them went with its pinned, freed, heads-replaced and pins-stated (H-A-48), and with them the assigner's state pins and its
// tools assigner_pins and assigner_unpin. A revision that is not current (the newest the assigner wrote) is removed, its object
// and the entry objects no kept revision or candidate lists and no want or running build names (G §179), when a revision ceases
// to be current and once at the start. Never on a timer.
// revisionObjects is the object bucket of the revisions as the remover removes from it.
type revisionObjects interface {
assigngraph.Objects
Delete(ctx context.Context, name string) error
}
// revisionRemover removes the revisions that are not current.
type revisionRemover struct {
ctx context.Context
objects revisionObjects
// inUse says a want or a running build names the module's fingerprint (§179).
inUse func(module, fp string) bool
logf func(string, ...any)
}
// judge removes every revision that is not current; a removal that fails is said and made at the next judging.
func (k *revisionRemover) judge() {
if err := k.remove(); err != nil {
k.logf("the revisions that are not current are not removed: %v", err)
}
}
// trunkRevision is the revision an object of a trunk revision names, and whether it names one.
func trunkRevision(name string) (int64, bool) {
n, ok := strings.CutPrefix(name, "trunk.")
if !ok {
return 0, false
}
r, err := strconv.ParseInt(n, 10, 64)
return r, err == nil && r > 0
}
// remove removes each kept revision that is not current, and then the entries no kept revision or candidate lists and no want or
// running build names (design 47 G §179).
func (k *revisionRemover) remove() error {
names, err := k.objects.Names(k.ctx)
if err != nil {
return err
}
var kept []int64
for _, n := range names {
if r, ok := trunkRevision(n); ok {
kept = append(kept, r)
}
}
if len(kept) == 0 {
return nil
}
current := slices.Max(kept)
listed := map[string]bool{}
removed := 0
for _, r := range kept {
if r == current {
rev, err := assigngraph.ReadRevision(k.ctx, k.objects, r)
if err != nil {
return err
}
for _, e := range rev.Entries {
listed[e.Digest] = true
}
continue
}
if err := k.objects.Delete(k.ctx, assigngraph.TrunkObject(r)); err != nil {
return fmt.Errorf("revision %d: %w", r, err)
}
removed++
}
if removed == 0 {
return nil
}
// A run's candidate kept now lists its entries too, until its run's pipeline-ended drops it (A-201, A-235).
for _, n := range names {
group, heads, ok := meshwants.ParseAssignmentRevisionsByCandidateGroupHeadsKey(n)
if !ok {
continue
}
rev, err := assigngraph.ReadCandidate(k.ctx, k.objects, group, heads)
if err != nil {
return err
}
for _, e := range rev.Entries {
listed[e.Digest] = true
}
}
for _, n := range names {
digest, ok := strings.CutPrefix(n, "entry.")
if !ok || listed[digest] {
continue
}
e, err := assigngraph.ReadEntry(k.ctx, k.objects, digest)
if err != nil {
return err
}
if k.inUse != nil && k.inUse(e.Module, e.Fp) {
continue
}
if err := k.objects.Delete(k.ctx, n); err != nil {
return fmt.Errorf("the entry %s: %w", digest, err)
}
}
k.logf("%d assignment graph revisions not current are removed", removed)
return nil
}
// inUseOf says a want or a running build names the module's fingerprint, as the world holds them now.
func inUseOf(w *world) func(module, fp string) bool {
return func(module, fp string) bool {
for _, want := range w.wantsOf()[module] {
if want.Fp == fp {
return true
}
}
for _, mods := range w.runningNow() {
if r, ok := mods[module]; ok && r.Fingerprint == fp {
return true
}
}
return false
}
}
@@ -0,0 +1,53 @@
package main
import (
"context"
"encoding/json"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/assigngraph"
)
func (m memObjects) Delete(_ context.Context, name string) error { delete(m, name); return nil }
// Design 47 H §764, H-A-86: with the gate's pins gone, an assignment graph revision that is not current is removed, with the
// entries no kept revision or candidate lists and no want or running build names; the current revision, the entries it lists,
// a running build's entry and a run's kept candidate with its entries stay.
func TestARevisionNotCurrentIsRemoved(t *testing.T) {
objects := memObjects{}
entry := func(module, fp string) string {
e := meshwants.AssignmentEntry{Node: "n1", Module: module, Fp: fp}
d := assigngraph.EntryDigest(e)
raw, _ := json.Marshal(e)
objects[assigngraph.EntryObject(d)] = raw
return d
}
shared, old, running, cand := entry("a", "fp-a"), entry("b", "fp-b1"), entry("c", "fp-c1"), entry("d", "fp-d2")
revision := func(n int64, digests ...string) {
rev := meshwants.AssignmentRevision{Revision: n}
for _, d := range digests {
rev.Entries = append(rev.Entries, meshwants.NodeEntryRef{Node: "n1", Digest: d})
}
raw, _ := json.Marshal(rev)
objects[assigngraph.TrunkObject(n)] = raw
}
candRaw, _ := json.Marshal(meshwants.AssignmentRevision{Revision: 1, Kind: "candidate", Group: "run-7", Heads: "h1",
Entries: []meshwants.NodeEntryRef{{Node: "n1", Module: "d", Digest: cand}}})
objects[assigngraph.CandidateObject("run-7", "h1")] = candRaw
revision(1, shared, old, running)
revision(2, shared, old)
revision(3, shared)
k := &revisionRemover{ctx: context.Background(), objects: objects, logf: t.Logf,
inUse: func(module, fp string) bool { return module == "c" && fp == "fp-c1" }}
k.judge()
has := func(name string) bool { _, ok := objects[name]; return ok }
for name, want := range map[string]bool{assigngraph.TrunkObject(1): false, assigngraph.TrunkObject(2): false, assigngraph.TrunkObject(3): true,
assigngraph.EntryObject(old): false, assigngraph.EntryObject(running): true, assigngraph.EntryObject(shared): true,
assigngraph.EntryObject(cand): true, assigngraph.CandidateObject("run-7", "h1"): true} {
if has(name) != want {
t.Errorf("%s kept = %v, want %v", name, has(name), want)
}
}
}
@@ -312,10 +312,10 @@ func warrantOf(v any) (*mma.Warrant, error) {
// mesh-node-enroller when the proven caller is that module's (its bus user `<node>.<module>`, never a person's
// `person.<module>`), else the caller as proven, which the refusal names.
func moduleOfCaller(by string, enrollers []string) string {
for _, module := range enrollers {
if (stdio.Callers{module}).Allows(by) {
return module
}
// The enrolment's caller stays (design 47 H-A-41): a module's bus user is `<node>.<module>`, a person's `person.<name>`.
_, module, ok := strings.Cut(by, ".")
if ok && !strings.HasPrefix(by, "person.") && slices.Contains(enrollers, module) {
return module
}
return by
}
@@ -20,11 +20,9 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/seatlist"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/views"
@@ -270,19 +268,6 @@ func TestTheSeatsVerbs(t *testing.T) {
})
}
s.seatList = nil
t.Run("1966: the seat list's again tool answers an operator's session through the mesh MCP server", func(t *testing.T) {
// Such a session is proved as its machine's tool runner (issue 627); the healers too, no other module. The list names
// the seats, resolved from the assigner's own seat-holders (A-295, A-298): a module holding neither is refused.
held := map[string][]byte{"node-tool-runner.shanks": []byte(`{"holders":[{"module":"node-tools","node":"shanks"}]}`),
"mesh-healers": []byte(`{"holders":[{"module":"mesh-delivery-watchman","node":"ace"}]}`)}
holders := seatlist.HoldersIn(func() (map[string][]byte, error) { return held, nil })
for caller, refused := range map[string]bool{"person.jochen": false, "shanks.node-tools": false, "ace.mesh-delivery-watchman": false,
"ace.mesh-healers": true, "ace.messenger": true} {
if err := seatlist.RefusalWith("assigner_seat_list_again", seatListAgainCallers, caller, holders); (err != nil) != refused {
t.Errorf("%s: refused=%v, err %v", caller, refused, err)
}
}
})
t.Run("1953/D4-D6: the seat mesh-wants' upgrade and bus", wantsVerbs)
t.Run("1955/L7 (ADR 0571 §6): rule 7 reads the node's own composition; an unplaced node is said", func(t *testing.T) {
w := newWorld()
@@ -533,51 +518,6 @@ func TestTheSeatsVerbs(t *testing.T) {
})
}
})
t.Run("1954/D1: a failed provision is the assigner's finding until it is provided", func(t *testing.T) {
findings := map[string][]byte{}
p := &passer{w: newWorld(), logf: func(string, ...any) {}, kvs: kvSet{buckets.Provisions: nil}, findings: assign.Findings{
Put: putInto(findings),
Delete: func(k string) error { delete(findings, k); return nil }}}
failed, _ := meshprovisionerclient.Encode(meshprovisionerclient.Provision{Provision: "db", Consumer: meshprovisionerclient.End{Module: "photos", Node: "ace"},
Provider: meshprovisionerclient.End{Module: "postgres", Node: "novox"}, State: meshprovisionerclient.Failed, Why: "the role exists already"})
p.w.putEntry(buckets.Provisions, "ace.photos.db", failed, 1, time.Now())
p.kvs[buckets.Provisions] = &memKV{}
of := p.provisionsOf()
if got := of("photos")("ace"); got["ace.photos.db"].State != meshprovisionerclient.Failed {
t.Fatalf("provisions = %+v", got)
}
if _, ok := findings[assign.ProvisionFailedKey("ace.photos.db")]; !ok || len(findings) != 1 {
t.Fatalf("findings = %v", findings)
}
provided, _ := meshprovisionerclient.Encode(meshprovisionerclient.Provision{Provision: "db", Consumer: meshprovisionerclient.End{Module: "photos", Node: "ace"},
Provider: meshprovisionerclient.End{Module: "postgres", Node: "novox"}, State: meshprovisionerclient.Provided})
p.w.putEntry(buckets.Provisions, "ace.photos.db", provided, 2, time.Now())
p.provisionsOf()
if len(findings) != 0 {
t.Errorf("findings after provided = %v", findings)
}
// A required provision with no key: pulsed with the bound first, said once the pulse expired, ended once keyed.
ctx := context.Background()
p.kvs[plc.PulsesBucket()] = &memKV{values: map[string][]byte{}}
gone := map[string]bool{"ace.photos.s3": true}
p.provisionsMissing(ctx, gone)
if !p.pulsing("provision-missing.ace.photos.s3") || len(findings) != 0 {
t.Fatalf("first seen: pulsing %v, findings %v", p.pulsing("provision-missing.ace.photos.s3"), findings)
}
p.provisionsMissing(ctx, gone)
if len(findings) != 0 {
t.Fatal("said before its bound")
}
p.w.put(plc.PulsesBucket(), "provision-missing.ace.photos.s3", nil) // the server expired it
p.provisionsMissing(ctx, gone)
if _, ok := findings[assign.ProvisionMissingKey("ace.photos.s3")]; !ok {
t.Fatalf("past its bound: findings %v", findings)
}
p.provisionsMissing(ctx, map[string]bool{})
if len(findings) != 0 {
t.Errorf("keyed again: findings %v", findings)
}
})
// A whole state is read in pages where the store has them, never a key at a time (27 s for the assignments live,
// 2026-11-14): the paged read answers what the key-by-key read answers, and asks no key alone.
direct, err := readAll(st, haveState)
@@ -1,16 +1,7 @@
package main
import (
stdio "git.novox.be/novox/mesh-sdk/go"
)
// The assigner reads the seat list as every part does, through mesh-sdk's one wiring (go/seatlist.Part; design 47 G §204): its
// reader over the seat mesh-catalog's verb seat-definitions and its two events, its tools `assigner_seat_list` and
// `assigner_seat_list_again`. What the reader holds is given to catalogue at every change, which the check of a proposal and the
// seat-holders state read; until the list is read, a mesh seat the bootstrapped definitions do not name is never judged held once
// (plc.CheckAssignments), and seat-holders is not written.
// A call through the mesh MCP server is proved only as its machine's tool runner (issue 627), so the holders of the seat
// node-tool-runner stand for an operator's session until a session is proved (issue 1966). The list names seats, never a
// module, resolved from the assigner's own seat-holders (design 47 G A-210, A-295, A-298).
var seatListAgainCallers = []string{stdio.AnyPerson, stdio.CallerSeatPrefix + "node-tool-runner", stdio.CallerSeatPrefix + "mesh-healers"}
@@ -60,7 +60,7 @@ func gateOver(t *testing.T, silent bool) *seatlist.Part {
// that waits for it to stay unread keeps short.
t.Cleanup(busretry.Waits(20*time.Millisecond, time.Millisecond, 5*time.Millisecond))
}
g := seatlist.NewPart("assigner", seatListAgainCallers)
g := seatlist.NewPart("assigner")
r := g.Follow(standInCatalog{silent: silent}, nil, t.Logf)
if r == nil {
t.Fatal("the reader did not start")
@@ -98,7 +98,7 @@ func TestTheSeatListIsWiredByTheSDKAlone(t *testing.T) {
// A-179, A-186: the definition the verb input's callers are checked against is the seat list's, none before the reader starts
// or while the seat mesh-catalog has not answered, and none for a seat the list does not hold.
func TestTheAnsweredSeatsDefinitionIsTheSeatListsOnceRead(t *testing.T) {
if _, ok := seatlist.Definition(seatlist.NewPart("assigner", seatListAgainCallers).Reader, "mesh-wants")(); ok {
if _, ok := seatlist.Definition(seatlist.NewPart("assigner").Reader, "mesh-wants")(); ok {
t.Error("a definition is answered before the seat list's reader started")
}
if _, ok := seatlist.Definition(gateOver(t, true).Reader, "mesh-wants")(); ok {
@@ -1,7 +1,6 @@
package main
import (
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
plc "mesh-delivery-assigner/internal/assignment"
"encoding/json"
@@ -394,7 +393,7 @@ func serveTools() error {
}
}()
// The seat list (seatlist.go), read through the runtime once Serve has its channel: the seat's verbs judge by it.
gate := seatlist.NewPart("assigner", seatListAgainCallers)
gate := seatlist.NewPart("assigner")
go func() {
<-stdio.Served()
gate.Follow(seatlist.StdioBus(), nil, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: "+f+"\n", a...) })
@@ -408,8 +407,6 @@ func serveTools() error {
return stdio.SeatPublish(wantsEventSubject(e), b, id)
})
wants.watched = stdio.NewWatchedStates() // the large states read whole from one watched copy (ADR 0580 clause 3)
// The callers' seat entries are resolved from the assigner's own seat-holders, its watched copy (A-298).
gate.Holders(seatlist.HoldersIn(func() (map[string][]byte, error) { return wants.watched.Values(mma.StateSeatHolders) }))
// The verb assignments with seats answers from the same watched copy, so every caller of it is fast too (issue 1982).
seatVerbs.watched = wants.watched
wants.revisions = runtimeObjects{busstate.ObjectsOf(assigngraph.RevisionsObjects)}
@@ -422,7 +419,6 @@ func serveTools() error {
tools = append(tools, enginereads.Tool(engineReadsTool, "the assigner", enginereads.KeptState(stdio.State(engineReadsState)), false, time.Now))
tools = append(tools, writtenTools(runtimeStore{})...)
tools = append(tools, whyTools(runtimeStore{})...)
tools = append(tools, pinsTools(runtimeStore{}, stdio.Ask)...)
tools = append(tools, askAgainTools(runtimeStore{}, stdio.Ask, time.Now)...)
return stdio.Serve("", tools)
}
@@ -24,26 +24,6 @@ import (
// moduleGraph is the module graph of the catalog half of branch, from the entries' invokes and serves (design 47 G §184, §195).
func (p *passer) moduleGraph(branch string) graph.Graph { return p.w.graphOf(branch) }
// keptInvokes are the verbs of module the graph keeps as edges: each it invokes and does not leave out, that a module
// other than itself serves. A verb no module of the graph serves is no edge, so it holds nothing; nil when the graph holds
// no node of module.
func keptInvokes(g graph.Graph, module string) []string {
n, ok := g[module]
if !ok {
return nil
}
var out []string
for _, v := range n.Invokes {
if slices.Contains(n.LeftOut, v) {
continue
}
if slices.ContainsFunc(g.Servers(v), func(s string) bool { return s != module }) {
out = append(out, v)
}
}
return out
}
// served is, for this pass, whether a build serving a verb runs healthy on some node (graph.Served): every running healthy
// have's build (the builder's builds key of its module and fingerprint) names its manifest, and the verbs that manifest
// serves are served. A healthy build whose manifest cannot be read counts what the graph says its module serves, said in
@@ -73,7 +73,7 @@ func viewTools(getenv func(string) string, state func(string) views.Source, now
marks := state(views.MarksState)
var out []stdio.Tool
for _, t := range firstViewTools {
out = append(out, views.FirstViewTool(assignment.Module(), t.tool, t.state, t.words, views.ParseCallers(getenv(views.CallersEnv(t.tool))), state(t.state), marks, now))
out = append(out, views.FirstViewTool(assignment.Module(), t.tool, t.state, t.words, state(t.state), marks, now))
}
for _, t := range readTools {
out = append(out, views.ReadTool(assignment.Module(), t.tool, t.state, t.words, state(t.state), marks, now))
@@ -26,32 +26,36 @@ const whyTool = "assigner_why"
// The steps of the pass that decide a want.
const (
whyNoPipeline = "no-passed-pipeline" // the trunk's fingerprint passed no pipeline on the branch, and no want names it
whyRules = "rules" // the six rules judged it
whyNoPipeline = "no-passed-pipeline" // no version of the module released on the branch passed its pipeline: the want written stays
whyPassed = "passed-pipeline" // the want names the newest version released on the branch whose pipeline passed (design 47 H §750)
)
// whyStep is the step that decided a want: a passed pipeline's version, or none yet.
func whyStep(passed bool) string {
if passed {
return whyPassed
}
return whyNoPipeline
}
// wantWhy is one key's value.
type wantWhy struct {
At time.Time `json:"at"`
Branch string `json:"branch,omitempty"`
// Fp is the fingerprint the trunk's entry names, the one the pass judged.
Fp string `json:"fp,omitempty"`
Step string `json:"step"`
// Fp, Version and Pipeline are the build the want names and the pipeline whose passed end released it (design 47 H §750).
Fp string `json:"fp,omitempty"`
Version string `json:"version,omitempty"`
Pipeline string `json:"pipeline,omitempty"`
Step string `json:"step"`
// Before is the want that stood before the pass, After the one the pass wrote (none: it kept Before).
Before *mwc.Want `json:"before,omitempty"`
After *mwc.Want `json:"after,omitempty"`
// Rule is the rule holding the node back from Fp, with its words.
Rule string `json:"rule,omitempty"`
// What the rules read of the module.
Built, BuildFailed, Held, Granted, Released, Kept bool `json:",omitempty"`
BuildError string `json:"build-error,omitempty"`
Words string `json:"words,omitempty"`
}
func whyKey(node, module string) string { return whyPrefix + node + "." + module }
// sayWhy writes the why of module on each of nodes, once per change (the time aside): every put is a change its watchers hear.
func (p *passer) sayWhy(ctx context.Context, module string, nodes []string, w wantWhy, before map[string]mwc.Want, out *assign.Outcome, m *assign.Module) {
func (p *passer) sayWhy(ctx context.Context, module string, nodes []string, w wantWhy, before map[string]mwc.Want, out *assign.Outcome) {
kv := p.kvs[writtenBucket()]
if kv == nil {
return
@@ -69,12 +73,6 @@ func (p *passer) sayWhy(ctx context.Context, module string, nodes []string, w wa
if a, ok := out.Wants[node]; ok {
one.After = &a
}
if h, ok := out.Rules[node]; ok {
one.Rule = h.Reason()
}
}
if m != nil {
one.Built, one.BuildFailed, one.BuildError, one.Held, one.Granted, one.Released, one.Kept = m.Built, m.BuildFailed, m.BuildError, m.Held, m.Granted, m.Released, m.Kept
}
same := one
same.At = time.Time{}
@@ -97,7 +95,7 @@ func (p *passer) sayWhy(ctx context.Context, module string, nodes []string, w wa
// whyTools is the tool assigner_why: every want's why, or those of one module or one node.
func whyTools(st store) []stdio.Tool {
return []stdio.Tool{{Name: whyTool, Description: "why a want is what it is: per node and module, the step of the assigner's last pass that decided the want (no passed pipeline, or the six rules with the rule holding it back), the fingerprint the trunk names, the want before and the one written, and what the rules read (built, held, granted). A time that stands still says the pass no longer judges that module. Arguments: module and node narrow it.",
return []stdio.Tool{{Name: whyTool, Description: "why a want is what it is: per node and module, the step of the assigner's last pass that decided the want (no passed pipeline yet, or a passed pipeline's version), the build, version and pipeline it names, the want before and the one written. A time that stands still says the pass no longer judges that module. Arguments: module and node narrow it.",
Input: schema(map[string]string{"module": "a module's name", "node": "a node's name"}, nil),
Run: func(args map[string]any) (any, error) {
ks, err := st.keys(writtenState)
@@ -12,7 +12,6 @@ import (
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
mdc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
@@ -135,23 +134,6 @@ func (w *world) put(bucket, key string, value []byte) {
}
}
// lastHealthy is the newest fingerprint other than fp the node ran the module healthy at, or empty.
func (w *world) lastHealthy(node, module, fp string) string {
key, err := nodeengineclient.HaveKey(node, module)
if err != nil {
return ""
}
w.mu.Lock()
defer w.mu.Unlock()
list := w.healthy[key]
for i := len(list) - 1; i >= 0; i-- {
if list[i] != fp {
return list[i]
}
}
return ""
}
// read is a copy of what a bucket holds now.
func (w *world) read(bucket string) map[string][]byte {
w.mu.Lock()
@@ -460,30 +442,6 @@ func (w *world) has(bucket, key string) bool {
return ok
}
// built says whether the builder's build of a module at a fingerprint is done, from its `build.ended` heard (design 47 G A-181).
func (w *world) built(module, fp string) bool {
e, ok := w.ends.endOf(module, fp)
return ok && e.Outcome == builds.BuildDone
}
// buildStanding says whether the builder's `build.ended` of module at fp said the build failed, and its error whole (design 47 G
// §575, A-162): the words of a hold under rule 1.
func (w *world) buildStanding(module, fp string) (failed bool, words string) {
if e, ok := w.ends.endOf(module, fp); ok && e.Outcome == builds.BuildFailed {
return true, e.Error
}
return false, ""
}
// buildNeeds are the reference forms the builder's done build of module at fp is written in (the build's end says them, ADR
// 0618 §1, A-181): nil when it is not done or names none.
func (w *world) buildNeeds(module, fp string) []string {
if e, ok := w.ends.endOf(module, fp); ok && e.Outcome == builds.BuildDone {
return e.Needs
}
return nil
}
// recordReads is what node's record `<node>.node` states its engine reads (ADR 0614 §4), and whether the assigner holds
// a record of it: the fallback where the engine does not answer the verb manifest-reads (enginereads.FromRecord).
func (w *world) recordReads(node string) ([]string, bool) {
@@ -3,7 +3,6 @@ package assign
import (
"fmt"
"mesh-delivery-assigner/internal/assignment"
"regexp"
"strings"
"time"
@@ -16,56 +15,12 @@ import (
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
)
// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (asks.HoldAskKey): keep it held or
// release it. Releasing writes the held fingerprint's want again by the rules (ADR 0539 §19). node and words are the held
// record's: the machine whose health check failed and the hold's words (issue 1906). asking numbers the writing: 0 the first, a
// larger number an ask written again after a hold lifted by itself, so the router takes it as new.
func HoldAsk(key, module, fp, node, words string, asking int, now time.Time) asks.Ask {
return assignerAsk(asking, oc.Ask{
ID: meshdeliveryclient.AskingID(key, asking),
Headline: fmt.Sprintf("%s is held at %s", module, short(fp)),
Explanation: fmt.Sprintf("The health check of %s failed on %s at %s (%s), so the assigner holds that version and wants "+
"no node to move to it. It lifts the hold by itself once every node that runs %s runs it healthy; releasing it "+
"lets the wants move to it now.", module, node, short(fp), words, module),
Options: []oc.Option{
{ID: "keep", Label: "Keep it held", Does: "the version stays held", Level: occ.Acknowledge},
{ID: "release", Label: "Release it", Does: "the version is released and the wants move to it by the rules again", Level: occ.Acknowledge},
},
Who: occ.Operator,
Expires: occ.FormatTime(now.Add(AskLasts)),
OnExpiry: "the version stays held",
About: key,
})
}
// The options of the restore ask (ADR 0537 §3, ADR 0539 §19), and what each does on the node.
const (
OptionRestore = "restore"
OptionLeaveStopped = "leave-stopped"
)
// RestoreAsk is the ask about a module holding data whose fingerprint failed its health check on a node (ADR 0429 §6): it is
// not set back by itself, since a set-back would lose the writes made since. words is the check's words. asking is the writing's
// number (AskingSeconds): an ask taken away as stale and written again is another asking under an id of its own, since the
// router drops a second ask under an id it keeps and the teller took the first back (issue 1966). Restore sets the
// node back to the last healthy fingerprint and loses the newer writes; leaving it stopped keeps the data as it is.
func RestoreAsk(key, node, module, fp, words string, asking int, now time.Time) asks.Ask {
return assignerAsk(asking, oc.Ask{
ID: meshdeliveryclient.AskingID(key, asking),
Headline: fmt.Sprintf("%s failed its health check on %s", module, node),
Explanation: fmt.Sprintf("%s holds data, and its health check failed on %s at %s: %s. Setting it back to the version that ran healthy "+
"would lose the writes made since; leaving it stopped keeps the data as it is.", module, node, short(fp), words),
Options: []oc.Option{
{ID: OptionRestore, Label: "Restore it", Does: "the node is set back to the last healthy version and the newer writes are lost", Level: occ.Acknowledge},
{ID: OptionLeaveStopped, Label: "Leave it stopped", Does: "the module stays as it is on the node and the data is kept", Level: occ.Acknowledge},
},
Who: occ.Operator,
Expires: occ.FormatTime(now.Add(AskLasts)),
OnExpiry: "the module stays as it is on the node",
About: key,
})
}
// The options of the outside ask (design 47 G §706).
const (
OptionGoOn = "go-on"
@@ -106,92 +61,6 @@ func WaitsOutsideFinding(node, module, causeKey, on string, at time.Time) Observ
Needs: "Nothing for you to do.", Source: "assigner.delivery-waits-outside"}
}
// HeldCondition is the condition the assigner raises when it holds a fingerprint of a module (ADR 0429 §6, ADR 0539 §19), or
// clears (raised false) when the hold is released. It names the module, the node whose health check failed and the check's
// words, under the assigner's own name (ADR 0437 clause 9), keyed by the held fingerprint. needsYou is true for a module holding
// data, whose restore ask waits for the operator.
func HeldCondition(module, fp, node, words string, raised, needsYou bool) oc.Condition {
key := assignment.Module() + ".held." + module + "." + strings.TrimPrefix(fp, meshcatalogclient.SourcePrefix2)
subject := oc.ConditionSubject{Scope: "module", ID: module, Machine: node}
if !raised {
return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared}
}
c := oc.Condition{Key: key, Severity: occ.SeverityWarning, Subject: subject, State: occ.StateRaised,
Summary: fmt.Sprintf("%s is held at %s", module, short(fp)),
Headline: fmt.Sprintf("%s is held at %s", module, short(fp))}
what := fmt.Sprintf("The health check of %s failed on %s at %s: %s. The assigner holds that version and wants no node to move to it.", module, node, short(fp), words)
if needsYou {
c.Needs = "answer whether to restore " + module + " on " + node
c.Explanation = occ.NeedsYou + c.Needs + ". " + what
} else {
c.Explanation = occ.NothingToDo + " " + what + " Nodes that wanted it are set back to the version that ran healthy."
}
return c
}
// NodeFailedCondition is the condition the assigner raises for a module whose newest fingerprint fp failed its health check on
// node while other nodes run it healthy (ADR 0557 §1, issue 1496): the failure is the node's, so nothing is held or set back and
// the node keeps its want. It names the node, the module, fp and the have's words, keyed by node and module, and is cleared
// (raised false) once the node no longer fails alone.
func NodeFailedCondition(node, module, fp string, f NodeFailure, raised bool) oc.Condition {
key := assignment.Module() + ".failed-alone." + node + "." + module
subject := oc.ConditionSubject{Scope: "module", ID: module, Machine: node}
if !raised {
return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared,
Resolved: fmt.Sprintf("%s on %s no longer fails its health check alone.", module, node)}
}
headline := fmt.Sprintf("%s fails on %s alone", module, node)
if n := len([]rune(headline)); n > occ.HeadlineLength {
headline = string([]rune(headline)[:occ.HeadlineLength])
}
return oc.Condition{Key: key, Severity: occ.SeverityWarning, Subject: subject, State: occ.StateRaised,
Summary: headline, Headline: headline,
Explanation: fmt.Sprintf("%s The health check of %s at %s failed on %s (%s), while it runs healthy on %s. The failure is %s's, "+
"so the assigner holds nothing and sets no node back, and %s keeps its want. This clears once %s runs it healthy or "+
"another version is the newest.", occ.NothingToDo, module, short(fp), node, f.Words, strings.Join(f.Others, ", "), node,
node, node)}
}
// NeitherCondition is the condition of a module assigned to node that holds no want for it (issue 1945): the node's own composition
// gives up every module with a fingerprint of the new form, so nothing applies it there and its files are gone. It is raised (urgent)
// naming the module, the node and the rule holding its first want back with the rule's words, under the assigner's own name keyed by
// node and module, and cleared (raised false) when the node has a want for it again.
func NeitherCondition(node, module string, h Holding, raised bool) oc.Condition {
key := assignment.Module() + ".neither." + node + "." + module
subject := oc.ConditionSubject{Scope: "module", ID: module, Machine: node}
if !raised {
return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared,
Resolved: fmt.Sprintf("%s on %s has a want again, so its own composition is applied there.", module, node)}
}
headline := fmt.Sprintf("%s on %s is on no list", module, node)
if n := len([]rune(headline)); n > occ.HeadlineLength {
headline = string([]rune(headline)[:occ.HeadlineLength])
}
return oc.Condition{Key: key, Severity: occ.SeverityUrgent, Subject: subject, State: occ.StateRaised,
Summary: headline, Headline: headline,
Explanation: fmt.Sprintf("%s %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
"hold it either, so nothing applies it there. The assigner's rule %d holds its first want back: %s. The assigner writes the want once the "+
"rule holds, and clears this.", occ.NothingToDo, module, node, h.Rule, h.Words)}
}
// NeitherLeftCondition clears the neither condition of module on node when it is no longer on neither list: why is "" when the
// node has a live want for it again, else why it has none (it is held, its move waits for an ask, the node no longer runs it), so
// the clear never says a want the node does not hold (issue 1945: "nats on novox has a want again" while it had none).
func NeitherLeftCondition(node, module, why string) oc.Condition {
c := NeitherCondition(node, module, Holding{}, false)
if why != "" {
c.Resolved = fmt.Sprintf("%s on %s is no longer on neither list and has no want: %s.", module, node, why)
}
return c
}
// LiftedCondition clears the condition of a held fingerprint the assigner lifted by itself, saying why (issue 1906).
func LiftedCondition(module, fp string) oc.Condition {
c := HeldCondition(module, fp, "", "", false, false)
c.Resolved = LiftedWords(module, fp) + "."
return c
}
// The options of the record ask (ADR 0539 §14) and of the backup ask (§16).
const (
OptionYes = "yes"
@@ -288,27 +157,6 @@ func NextBackupAsking(after int) int {
return max(after+1, 2)
}
var backupDigest = regexp.MustCompile(`\(composition (sha256:[0-9a-f]{12})\)`)
// BackupAskDigest is the digest of the composition a backup ask was made about, from its explanation, cut to its first 12 digits as
// the ask says it (ShortDigest); empty for an ask that names
// none. An answer is taken for the composition of the ask it answers, and a new revision at the same fingerprint is asked again
// under a new number (ADR 0539 §16).
func BackupAskDigest(a asks.Ask) string {
ask, err := AskOf(a)
if err != nil {
return ""
}
if m := backupDigest.FindStringSubmatch(ask.Explanation); m != nil {
return m[1]
}
return ""
}
// ShortDigest is a composition's digest as a backup ask names it, its first 12 digits (said): the ask stays bound to its
// composition, and a new composition is asked again.
func ShortDigest(digest string) string { return said(digest) }
// BackupAnswerWords are the words of a want carrying the answer to the asking-th ask: each asking's answer is another want
// value, so a backup that fails again after "back up again" and is answered "again" again changes the key and the node retries.
func BackupAnswerWords(option string, asking int) string {
@@ -316,14 +164,14 @@ func BackupAnswerWords(option string, asking int) string {
}
// BackupAsk is the ask about a backup of a module on node that failed before an apply (ADR 0539 §16): back up again, or go on
// without a backup. digest is the composition about to be applied and asking the number of the asking, 0 the first; words are the
// without a backup, at the build fp the want names. asking is the number of the asking, 0 the first; words are the
// backup's own words, from the node's have.
func BackupAsk(key, node, module, fp, digest, words string, asking int, now time.Time) asks.Ask {
func BackupAsk(key, node, module, fp, words string, asking int, now time.Time) asks.Ask {
return assignerAsk(asking, oc.Ask{
ID: meshdeliveryclient.AskingID(key, asking),
Headline: fmt.Sprintf("The backup of %s failed on %s", module, node),
Explanation: fmt.Sprintf("The backup before applying %s at %s on %s failed: %s. %s keeps running what it has until you answer. (composition %s)",
module, short(fp), node, words, module, digest),
Explanation: fmt.Sprintf("The backup before applying %s at %s on %s failed: %s. %s keeps running what it has until you answer.",
module, short(fp), node, words, module),
Options: []oc.Option{
{ID: OptionBackupAgain, Label: "Back up again", Does: "the node tries the backup again before it applies", Level: occ.Acknowledge},
{ID: OptionBackupSkip, Label: "Go on without a backup", Does: "the node applies the new version without a backup", Level: occ.Acknowledge},
@@ -23,11 +23,10 @@ import (
"git.novox.be/novox/mesh-sdk/go/views"
)
// 1297/B9 (the assigner's half): the ask about a held fingerprint is a valid ask of the router and of mesh-sdk's layout at
// its key, of a kind the view may answer.
func TestTheHoldAskIsValid(t *testing.T) {
t.Run("hold", theHoldAskIsValid)
t.Run("restore", theRestoreAskIsValid)
// 1297/B9 (the assigner's half): the backup ask is a valid ask of the router and of mesh-sdk's layout at its key, of a kind the
// view may answer; the hold and restore asks went with the hold of a fingerprint (design 47 H §750, H-A-55).
func TestTheAssignersAsks(t *testing.T) {
t.Run("backup", theBackupAskIsValid)
t.Run("ended at the channel", anAskEndedAtTheChannelIsWrittenAgain)
t.Run("ended at the channel, read at bind", anAskEndedAtTheChannelIsReadAtBind)
}
@@ -40,8 +39,6 @@ func anAskEndedAtTheChannelIsWrittenAgain(t *testing.T) {
asked := time.Date(2026, 11, 9, 22, 0, 0, 0, time.UTC)
now := asked.Add(8 * 24 * time.Hour)
digest := "sha256:" + strings.Repeat("e", 64)
holdKey, _ := asks.HoldAskKey("nats", fNew)
restoreKey, _ := asks.RestoreAskKey("novox", "nats", fNew)
backupKey, _ := asks.BackupAskKey("novox", "nats", fNew)
revertKey, _ := asks.RevertAskKey("novox", "nats", fNew)
recordKey, _ := asks.RecordAskKey("novox", "nats", digest)
@@ -63,9 +60,7 @@ func anAskEndedAtTheChannelIsWrittenAgain(t *testing.T) {
put *puts.Put
why string // "" when it is written again
}{
{name: "hold", key: holdKey, a: ptr(HoldAsk(holdKey, "nats", fNew, "novox", "w", AskingSeconds(asked), asked)), outcome: occ.OutcomeExpired},
{name: "restore", key: restoreKey, a: ptr(RestoreAsk(restoreKey, "novox", "nats", fNew, "w", AskingSeconds(asked), asked)), outcome: occ.OutcomeCancelled},
{name: "backup", key: backupKey, a: ptr(BackupAsk(backupKey, "novox", "nats", fNew, digest, "w", 0, asked)), outcome: occ.OutcomeCancelled},
{name: "backup", key: backupKey, a: ptr(BackupAsk(backupKey, "novox", "nats", fNew, "w", 0, asked)), outcome: occ.OutcomeCancelled},
{name: "revert", key: revertKey, a: ptr(assignerAsk(AskingSeconds(asked), oc.Ask{ID: meshdeliveryclient.AskingID(revertKey, AskingSeconds(asked)),
Headline: "h", Explanation: "e", Who: occ.Operator, Expires: occ.FormatTime(asked.Add(AskLasts)), OnExpiry: "o",
Options: []oc.Option{{ID: "hold", Label: "Hold", Does: "d", Level: occ.Acknowledge}}})), outcome: occ.OutcomeExpired},
@@ -73,11 +68,11 @@ func anAskEndedAtTheChannelIsWrittenAgain(t *testing.T) {
{name: "record of another put", key: recordKey, a: &record, outcome: occ.OutcomeExpired, put: &other},
{name: "record a put asked", key: recordKey, a: &byPut, outcome: occ.OutcomeExpired, put: &put, why: "put"},
{name: "assignments", key: assignmentsKey, a: &assignments, outcome: occ.OutcomeExpired, why: "ADR 0567"},
{name: "another asking stands", key: holdKey, a: ptr(HoldAsk(holdKey, "nats", fNew, "novox", "w", AskingSeconds(now), now)),
ended: meshdeliveryclient.AskingID(holdKey, AskingSeconds(asked)), outcome: occ.OutcomeExpired, why: "holds the asking"},
{name: "taken away", key: holdKey, ended: "hold-x", outcome: occ.OutcomeExpired, why: "taken away"},
{name: "refused", key: holdKey, a: ptr(HoldAsk(holdKey, "nats", fNew, "novox", "w", 0, asked)), outcome: occ.OutcomeRefused, why: "refused"},
{name: "replaced", key: holdKey, a: ptr(HoldAsk(holdKey, "nats", fNew, "novox", "w", 0, asked)), outcome: occ.OutcomeReplaced, why: "replaced"},
{name: "another asking stands", key: backupKey, a: ptr(BackupAsk(backupKey, "novox", "nats", fNew, "w", AskingSeconds(now), now)),
ended: meshdeliveryclient.AskingID(backupKey, AskingSeconds(asked)), outcome: occ.OutcomeExpired, why: "holds the asking"},
{name: "taken away", key: backupKey, ended: "backup-x", outcome: occ.OutcomeExpired, why: "taken away"},
{name: "refused", key: backupKey, a: ptr(BackupAsk(backupKey, "novox", "nats", fNew, "w", 0, asked)), outcome: occ.OutcomeRefused, why: "refused"},
{name: "replaced", key: backupKey, a: ptr(BackupAsk(backupKey, "novox", "nats", fNew, "w", 0, asked)), outcome: occ.OutcomeReplaced, why: "replaced"},
} {
t.Run(c.name, func(t *testing.T) {
var current []byte
@@ -134,12 +129,12 @@ func anAskEndedAtTheChannelIsWrittenAgain(t *testing.T) {
// answered or another asker's ask, and one the key no longer holds, are not.
func anAskEndedAtTheChannelIsReadAtBind(t *testing.T) {
asked := time.Date(2026, 11, 9, 22, 0, 0, 0, time.UTC)
holdKey, _ := asks.HoldAskKey("nats", fNew)
restoreKey, _ := asks.RestoreAskKey("novox", "nats", fNew)
backupKey, _ := asks.BackupAskKey("novox", "nats", fNew)
hold := HoldAsk(holdKey, "nats", fNew, "novox", "w", AskingSeconds(asked), asked)
restore := RestoreAsk(restoreKey, "novox", "nats", fNew, "w", AskingSeconds(asked.Add(time.Hour)), asked.Add(time.Hour))
backup := BackupAsk(backupKey, "novox", "nats", fNew, "sha256:"+strings.Repeat("e", 64), "w", 0, asked.Add(-time.Hour)) // no asking
holdKey, _ := asks.BackupAskKey("g14", "nats", fNew)
restoreKey, _ := asks.BackupAskKey("ace", "nats", fNew)
hold := BackupAsk(holdKey, "g14", "nats", fNew, "w", AskingSeconds(asked), asked)
restore := BackupAsk(restoreKey, "ace", "nats", fNew, "w", AskingSeconds(asked.Add(time.Hour)), asked.Add(time.Hour))
backup := BackupAsk(backupKey, "novox", "nats", fNew, "w", 0, asked.Add(-time.Hour)) // no asking
held := map[string][]byte{}
for key, a := range map[string]asks.Ask{holdKey: hold, restoreKey: restore, backupKey: backup} {
v, err := EncodeAsk(key, a)
@@ -174,27 +169,27 @@ func anAskEndedAtTheChannelIsReadAtBind(t *testing.T) {
func ptr(a asks.Ask) *asks.Ask { return &a }
func theHoldAskIsValid(t *testing.T) {
key, err := asks.HoldAskKey("mail", fNew)
func theBackupAskIsValid(t *testing.T) {
key, err := asks.BackupAskKey("ace", "mail", fNew)
if err != nil {
t.Fatal(err)
}
now := time.Now()
a := HoldAsk(key, "mail", fNew, "ace", "health failed, phase running: connection refused", 0, now)
a := BackupAsk(key, "ace", "mail", fNew, "health failed, phase running: connection refused", 0, now)
if err := occ.Check(mustAsk(t, a), now); err != nil {
t.Errorf("the router refuses the ask: %v", err)
}
if _, err := EncodeAsk(key, a); err != nil {
t.Errorf("the ask cannot be written: %v", err)
}
// 1906/H3: the ask names the machine whose check failed and the hold's words.
// The ask names the machine whose backup failed and the backup's words.
if !strings.Contains(mustAsk(t, a).Explanation, "on ace") || !strings.Contains(mustAsk(t, a).Explanation, "connection refused") {
t.Errorf("the ask does not name the node and the hold's words: %q", mustAsk(t, a).Explanation)
}
// 1906/H2: an ask written again after a lift is another asking, under an id of its own.
again := HoldAsk(key, "mail", fNew, "ace", "x", 7, now)
// An ask written again is another asking, under an id of its own.
again := BackupAsk(key, "ace", "mail", fNew, "x", 7, now)
if again.Asking != 7 || mustAsk(t, again).ID != meshdeliveryclient.AskingID(key, 7) || mustAsk(t, again).ID == mustAsk(t, a).ID {
t.Errorf("the asking after a lift = %d %s", again.Asking, mustAsk(t, again).ID)
t.Errorf("the asking again = %d %s", again.Asking, mustAsk(t, again).ID)
}
if _, err := EncodeAsk(key, again); err != nil {
t.Errorf("the ask asked again cannot be written: %v", err)
@@ -204,7 +199,7 @@ func theHoldAskIsValid(t *testing.T) {
}
// The join (A1d ruling 1): mesh-sdk no longer refuses another id, no option or no ask in an assigner's ask; EncodeAsk does.
t.Run("EncodeAsk refuses what mesh-sdk no longer does", func(t *testing.T) {
good := HoldAsk(key, "mail", fNew, "ace", "x", 0, time.Unix(100, 0))
good := BackupAsk(key, "ace", "mail", fNew, "x", 0, time.Unix(100, 0))
other := good
other.Asking = 3 // its id is the asking 0's
empty := good
@@ -219,68 +214,11 @@ func theHoldAskIsValid(t *testing.T) {
})
}
// 1906/D3 (ADR 0539 §19): the ask about a data module held on a node is a valid ask of the router and of mesh-sdk's layout at
// its restore key, of a kind the view may answer; it offers restoring (losing the newer writes) and leaving it stopped.
func theRestoreAskIsValid(t *testing.T) {
key, err := asks.RestoreAskKey("ace", "mail", fNew)
if err != nil {
t.Fatal(err)
}
now := time.Now()
a := RestoreAsk(key, "ace", "mail", fNew, "connection refused", 2, now)
if err := occ.Check(mustAsk(t, a), now); err != nil {
t.Errorf("the router refuses the ask: %v", err)
}
if _, err := EncodeAsk(key, a); err != nil {
t.Errorf("the ask cannot be written: %v", err)
}
if _, err := viewstate.ViewAnswerKey(key); err != nil {
t.Errorf("the view may not answer it: %v", err)
}
var ids []string
for _, o := range mustAsk(t, a).Options {
ids = append(ids, o.ID)
}
if len(ids) != 2 || ids[0] != OptionRestore || ids[1] != OptionLeaveStopped {
t.Errorf("options = %v", ids)
}
if !strings.Contains(mustAsk(t, a).Explanation, "connection refused") || !strings.Contains(mustAsk(t, a).Explanation, "ace") {
t.Errorf("the ask does not name the node and the check's words: %q", mustAsk(t, a).Explanation)
}
}
// 1906/D3: the condition for a held fingerprint names the module, the node and the check's words, under the assigner's own
// name (ADR 0437 clause 9), and is cleared under the same key.
// 1952/WM6: the assigner's conditions are findings.
func TestTheAssignersConditions(t *testing.T) {
t.Run("the held condition names module, node and words", heldConditionNamesModuleNodeAndWords)
t.Run("a lifted hold clears its condition saying why", liftedHoldClearsItsConditionSayingWhy)
t.Run("1952/WM6: each said condition is a finding, its clear deletes it, said straight only with no findings bucket", saidConditionsAreFindings)
}
func heldConditionNamesModuleNodeAndWords(t *testing.T) {
c := HeldCondition("mail", fNew, "ace", "connection refused", true, true)
if _, err := occ.EncodeCondition(c); err != nil {
t.Fatalf("the condition is refused: %v", err)
}
if c.State != occ.StateRaised || c.Subject.ID != "mail" || c.Subject.Machine != "ace" || !strings.Contains(c.Explanation, "connection refused") || !strings.Contains(c.Key, "mail") {
t.Errorf("condition = %+v", c)
}
if got := HeldCondition("mail", fNew, "", "", false, false); got.State != occ.StateCleared || got.Key != c.Key {
t.Errorf("cleared = %+v", got)
}
}
// 1906/H2: a hold lifted by itself clears the held condition under its key, saying why.
func liftedHoldClearsItsConditionSayingWhy(t *testing.T) {
c := LiftedCondition("mail", fNew)
if _, err := occ.EncodeCondition(c); err != nil {
t.Fatalf("the condition is refused: %v", err)
}
if c.State != occ.StateCleared || c.Key != HeldCondition("mail", fNew, "ace", "x", true, false).Key || !strings.Contains(c.Resolved, "lifted its hold by itself") {
t.Errorf("condition = %+v", c)
}
}
// 1952/WM6 (ADR 0564 §2, issue 1952 note 16): every condition the assigner says is a finding in its own bucket, under
// `<scope>.<id>.<kind>`, written at the raise and deleted at the clear, and only as a finding: the keeper is the one
// source since the controller's end (novox/hq ADR 0590 §3, issue 1126 B8). Said straight to the router only where the
@@ -291,9 +229,6 @@ func saidConditionsAreFindings(t *testing.T) {
raised, cleared oc.Condition
key string
}{
{HeldCondition("mail", fNew, "ace", "refused", true, true), LiftedCondition("mail", fNew), "module.mail." + strings.TrimPrefix(fNew, meshcatalogclient.SourcePrefix2) + ".held"},
{NodeFailedCondition("ace", "mail", fNew, NodeFailure{Words: "x", Others: []string{"g14"}}, true), NodeFailedCondition("ace", "mail", fNew, NodeFailure{}, false), "module.ace.mail.failed-alone"},
{NeitherCondition("ace", "mail", Holding{Rule: 3, Words: "w"}, true), NeitherCondition("ace", "mail", Holding{}, false), "module.ace.mail.neither"},
{SeatWaitsCondition([]string{"assignments"}), SeatWaitsCondition(nil), "module.mesh-delivery-assigner.seat-states"},
{BackupCondition("ace", "mail", fNew, "disk full", true), BackupCondition("ace", "mail", fNew, "", false), "module.ace.mail." + strings.TrimPrefix(fNew, meshcatalogclient.SourcePrefix2) + ".backup"},
} {
@@ -336,8 +271,7 @@ func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
if err != nil {
t.Fatal(err)
}
dgst := "sha256:" + strings.Repeat("b", 64)
b := BackupAsk(bkey, "ace", "mail", fNew, dgst, "no space left", 0, now)
b := BackupAsk(bkey, "ace", "mail", fNew, "no space left", 0, now)
vkey, err := asks.RevertAskKey("ace", "mail", fNew)
if err != nil {
t.Fatal(err)
@@ -360,18 +294,15 @@ func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
if !strings.Contains(mustAsk(t, b).Explanation, "no space left") || mustAsk(t, b).Options[0].ID != mwc.BackupAgain || mustAsk(t, b).Options[1].ID != mwc.BackupSkip {
t.Errorf("the backup ask = %+v", mustAsk(t, b))
}
// the id binds the ask to the composition and to its number (review round 1, items 3 and 4)
if again := BackupAsk(bkey, "ace", "mail", fNew, dgst, "x", NextBackupAsking(0), now); mustAsk(t, again).ID == mustAsk(t, b).ID {
// the id binds the ask to its number (review round 1, item 4); its key binds it to the build (design 47 H §750: a want names no composition)
if again := BackupAsk(bkey, "ace", "mail", fNew, "x", NextBackupAsking(0), now); mustAsk(t, again).ID == mustAsk(t, b).ID {
t.Errorf("asked again under the same id")
}
if NextBackupAsking(-1) != 0 || NextBackupAsking(0) != 2 || NextBackupAsking(2) != 3 {
t.Errorf("the numbers of the askings: %d %d %d", NextBackupAsking(-1), NextBackupAsking(0), NextBackupAsking(2))
}
if BackupAskDigest(b) != ShortDigest(dgst) || ShortDigest(dgst) == dgst {
t.Errorf("the ask names composition %q, want %q", BackupAskDigest(b), ShortDigest(dgst))
}
// novox 2026-11-16: the router refuses a long hexadecimal string, so no word of an ask carries a digest or fingerprint whole.
held := BackupAsk(bkey, "novox", "mail", fNew, dgst, "the kept data between "+fNew+" and "+fNew+" is not known", 0, now)
held := BackupAsk(bkey, "novox", "mail", fNew, "the kept data between "+fNew+" and "+fNew+" is not known", 0, now)
if a := mustAsk(t, held); regexp.MustCompile(`[0-9a-f]{20,}`).MatchString(a.Headline + a.Explanation + a.Details) {
t.Errorf("the ask carries a long hexadecimal string: %q", a.Explanation)
}
@@ -401,6 +332,7 @@ func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
// ADR 0546 clauses 7 and 8: a put's ask is valid at the record ask's key; for the bus its yes approves and binds the act,
// so the view cannot answer it, and it lasts no longer than an approving ask may.
put := puts.Put{FP: fNew, Why: "the bus's user list", Hold: true}
dgst := "sha256:" + strings.Repeat("b", 64)
d := ComposedDigest{Digest: dgst, Judgement: JudgedCannot, Words: "the have names no composition"}
for _, bus := range []bool{false, true} {
a, err := PutAsk(rkey, "ace", "mail", put, d, bus, now)
@@ -445,22 +377,6 @@ func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
}
}
// 1945/AC21: the condition of a module on neither list is urgent, names the module, the node and the rule holding its want back,
// under the assigner's own name keyed by node and module, and is cleared under the same key.
func TestTheNeitherConditionNamesModuleNodeAndRule(t *testing.T) {
c := NeitherCondition("g14", "i3status-rust", Holding{Rule: mwc.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}, true)
if _, err := occ.EncodeCondition(c); err != nil {
t.Fatalf("the condition is refused: %v", err)
}
if c.State != occ.StateRaised || c.Severity != occ.SeverityUrgent || c.Subject.ID != "i3status-rust" || c.Subject.Machine != "g14" ||
!strings.Contains(c.Explanation, "x11-display") || !strings.Contains(c.Explanation, "rule 5") || c.Key != "mesh-delivery-assigner.neither.g14.i3status-rust" {
t.Errorf("condition = %+v", c)
}
if got := NeitherCondition("g14", "i3status-rust", Holding{}, false); got.State != occ.StateCleared || got.Key != c.Key {
t.Errorf("cleared = %+v", got)
}
}
// mustAsk is the router's ask an asks-state value carries.
func mustAsk(t *testing.T, a asks.Ask) oc.Ask {
t.Helper()
@@ -6,13 +6,11 @@ import (
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
"sort"
"strings"
"time"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
"git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
@@ -48,66 +46,15 @@ func PolicyOf(m catalogue.Manifest, chosen map[string]catalogue.PersonPolicy) (p
return catalogue.Upgrade(m, c)
}
func contains(list []string, s string) bool {
for _, v := range list {
if v == s {
return true
}
}
return false
}
// KindProvisionFailed is the assigner's finding of a provision its module's want waits for that failed (ADR 0570 §4).
const KindProvisionFailed = "provision-failed"
// ProvisionFailedKey is a failed provision's finding's key, `provider.<node>.<consumer>.<provision>[.<local>].provision-failed`.
func ProvisionFailedKey(key string) string {
return mcc.Key(mcc.ScopeProvider, key, KindProvisionFailed)
}
// ProvisionFailedFinding is the finding of a failed provision, naming its consumer, its provider and why.
func ProvisionFailedFinding(key string, p meshprovisionerclient.Provision, at time.Time) (mcseat.Finding, error) {
summary := fmt.Sprintf("%s for %s on %s failed at %s on %s: %s, so %s is not moved there until it is provided", p.Provision,
p.Consumer.Module, p.Consumer.Node, p.Provider.Module, p.Provider.Node, p.Why, p.Consumer.Module)
return encode(Observed{Scope: mcc.ScopeProvider, ID: key, Kind: KindProvisionFailed, Machine: p.Consumer.Node, Also: also(p.Provider.Node, p.Consumer.Node),
Summary: summary, Said: p.Why, Headline: clipHeadline(p.Consumer.Module + " waits for a provision that failed"),
Explanation: occ.NothingToDo + " " + summary + ". It clears once the provider provides it.",
Source: "assigner." + KindProvisionFailed}, at)
}
// encode is the observation as the assigner's finding at at.
func encode(o Observed, at time.Time) (mcseat.Finding, error) { return o.Finding(at), nil }
func also(provider, consumer string) []string {
if provider == "" || provider == consumer {
return nil
}
return []string{provider}
}
// KindProvisionMissing is the assigner's finding of a required provision that has had no key for longer than its bound.
const KindProvisionMissing = "provision-missing"
// ProvisionMissingKey is its key, `provider.<node>.<module>.<provision>.provision-missing`.
func ProvisionMissingKey(key string) string {
return mcc.Key(mcc.ScopeProvider, key, KindProvisionMissing)
}
// ProvisionMissingFinding is the finding of a required provision with no key past bound (key `<node>.<module>.<provision>`).
func ProvisionMissingFinding(key string, bound time.Duration, at time.Time) (mcseat.Finding, error) {
parts := strings.SplitN(key, ".", 3)
if len(parts) != 3 {
return mcseat.Finding{}, fmt.Errorf("%q is not <node>.<module>.<provision>", key)
}
node, module, provision := parts[0], parts[1], parts[2]
summary := fmt.Sprintf("%s on %s requires %s and mesh-provisioner has composed no key for it in %s, so %s is not moved there: "+
"no provider of it may be placed", module, node, provision, bound, module)
return encode(Observed{Scope: mcc.ScopeProvider, ID: key, Kind: KindProvisionMissing, Machine: node,
Summary: summary, Headline: clipHeadline(module + " on " + node + " lacks " + provision),
Explanation: occ.NothingToDo + " " + summary + ". It clears once the provision has its key.",
Source: "assigner." + KindProvisionMissing}, at)
}
// KindUnplaced is, from the lease write, a node that runs but no assignments names (ADR 0571 §6).
const KindUnplaced = "unplaced"
@@ -212,9 +212,9 @@ func needsPerson(w PutWorld, d ComposedDigest) bool {
return d.Judgement == JudgedRecreates || d.Judgement == JudgedMovesData || d.Judgement == JudgedCannot
}
// PutWant is the want a put writes once it is wanted: the fingerprint and composition, marked put, and held when it holds.
// PutWant is the want a put writes once it is wanted: the fingerprint, marked put, and held when it holds.
func PutWant(p puts.Put) mwc.Want {
return mwc.Want{Fp: p.FP, Composed: p.Composed, Put: true, Hold: p.Hold, Words: "put: " + p.Why}
return mwc.Want{Fp: p.FP, Put: true, Hold: p.Hold, Words: "put: " + p.Why}
}
// PutOwns says what a put does to the node's want: owns says the rules pass over it, want the want it writes (nil: what
@@ -1,38 +0,0 @@
package assign
import (
"slices"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
)
// verbWaiting says why the module's want on node waits for a verb it invokes (novox/hq ADR 0614 §2): the first seat verb or
// module tool of Invokes (the module graph's verb edges, those it leaves out on a cycle not among them, §3) that no
// running healthy build serves. Nil Served (nothing read of what runs) waits for none, and a verb a running build serves
// already adds no wait. It holds under the provisions' rule (RuleProvisions): no reader knows a rule number of its own
// yet, so the words name the edge (graph.VerbWaitWords, which the delivery view reads).
func (m Module) verbWaiting(node string) (string, bool) {
if m.Served == nil {
return "", false
}
for _, v := range slices.Sorted(slices.Values(m.Invokes)) {
if !m.Served(v) {
return graph.VerbWaitWords(v, m.Name, node), true
}
}
return "", false
}
// busStateWaiting says why the module's want on node waits for bus state (ADR 0614 §5): its manifest asks bus state (a
// bus-state entry, as the module graph reads it) and no build whose manifest offers bus-state runs healthy on a node
// present. The provision bus-state is answered with no credential of a consumer's own, so ADR 0603 §5 counts the
// provider's own have and no provisioner key is waited for, handed or not. Nil Offering (nothing read of what runs) waits
// for none, and neither does a module that offers bus-state itself. It holds under the provisions' rule, its words naming bus state (graph.BusStateWaitWords).
func (m Module) busStateWaiting(node string) (string, bool) {
if len(m.Manifest.BusState) == 0 || m.Offering == nil || m.Offering(graph.BusState) ||
slices.Contains(m.Manifest.Offers(), graph.BusState) { // the bus module offering it waits on nobody for it
return "", false
}
return graph.BusStateWaitWords(m.Name, node), true
}
@@ -2,324 +2,75 @@ package assign
import (
"fmt"
"slices"
"strings"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey"
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
"maps"
"mesh-delivery-assigner/internal/puts"
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds/forms"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
"mesh-delivery-assigner/internal/puts"
)
// Module is what the six rules read of one module (ADR 0429 §6, ADR 0528 clause 3): its newest fingerprint F at the
// trunk (the indexer's modules), whether the builder's build of F is done, whether F is held, whether the controller
// issued its credential for F, its manifest at F, and the nodes it is assigned to.
// A want names the newest released version whose pipeline passed (design 47 H §750): for every node whose assignment places
// a module on a watched branch, the want names the version, the pipeline and the mover's rank in it, written tier by tier
// (§751). It is held for nothing else: the rules of ADR 0429 §6 (the build done, not held, the canary and step, every
// provider healthy, the grant, the composition and the reader capabilities, the provisions, the served verbs and the bus
// state), the network hold and A-428's exceptions end (H-A-55). A put owns its node's want as before (ADR 0546), and the
// operator's yes on a module of policy record is the node-engine's have and the asker's (the pass's yeses).
// Target is the build a node's want names: its fingerprint and released version, with the pipeline whose passed end released
// it and the mover's rank in that pipeline; a placement's build names no pipeline and no rank (design 47 H §750, H-A-37).
type Target struct {
Fp string
Version string
Pipeline string
Rank int64
}
// Module is what the want rule reads of one module: the nodes it is assigned to, the build each node's want names, the puts
// that own a node's want and the nodes that compose it themselves.
type Module struct {
Name string
FP string
Built bool
// BuildFailed and BuildError say a build that ended failed, with the error as the builder's `build.ended` carries it whole
// (design 47 G §575, A-162): the words of a hold under rule 1.
BuildFailed bool
BuildError string
Held bool
Granted bool
// GrantedAt says whether the controller issued the module's credential for another fingerprint than FP: the first want of a node
// that runs an older one names it (issue 1945). Nil: none.
GrantedAt func(fp string) bool
// GrantedOn says whether the controller's grant of FP names the module's accounts on a node, when the grant names any:
// a grant written for the module's other nodes says nothing of a node whose account is not issued and sealed (issue
// 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus). Nil: the grant alone.
GrantedOn func(node string) bool
Manifest catalogue.Manifest
// Provisions is the seat mesh-provisioner's provisions this module consumes on a node, by key (ADR 0570 §4): the want
// waits until each is provided. Nil: the seat's state is not read (not delivered yet), and nothing waits on it.
Provisions func(node string) map[string]meshprovisionerclient.Provision
// Answering is the manifests of the modules placed somewhere that provide a provision: what catalogue.Awaited
// judges a want's provisions by (issue 774). Nil: none known, and no key is waited for that is not there.
Answering func(provision string) []catalogue.Manifest
// Offered says whether any module offers a provision by name or by its own name (catalogue.Manifest.Offers) among every
// manifest the trunk holds and the modules the mesh provides itself, placed or not. Nil: not known, and no requirement
// is waited on for want of an offer.
Offered func(provision string) bool
// Invokes are the seat verbs and module tools the module graph says the module invokes and keeps as edges to the
// modules serving them, those it leaves out on a cycle not among them (ADR 0614 §2, §3): the want waits until each is
// served (Served). Nil: no graph node is read, and no want waits on a verb.
Invokes []string
// Served says whether a build serving a seat verb or module tool runs healthy, counted from its holder's have (ADR 0614
// §2, as ADR 0603 §5 counts a provider's). Nil: nothing is read of what runs, and no want waits on a verb.
Served func(invoke string) bool
// Offering says whether a build whose manifest offers a provision runs healthy on a node present, counted from the
// provider's own have (ADR 0603 §5): what a want asking bus state waits for (ADR 0614 §5). Nil: nothing is read of what
// runs, and no want waits on bus state.
Offering func(provision string) bool
Name string
Manifest catalogue.Manifest
// Chosen is a person's choice of the module's upgrade policy (the assigner's `policies`, ADR 0565 §7); nil: none.
Chosen *catalogue.PersonPolicy
Nodes []string
// Composed answers the composition of the module for a node at a fingerprint: the digest the node's own have names (a
// placed node composes its own, ADR 0571 §6), and whether it names one. Nil: none exists (ADR 0539 §1, §6).
Composed func(node, fp string) (ComposedDigest, bool)
// Target is the build node's want names: the newest version of the module released on the branch the node's placement
// names whose pipeline passed and whose tier is written (§750, §751), or the placement's build (H-A-5); false: none yet,
// and the want written stays.
Target func(node string) (Target, bool)
// Allowed says whether the operator answered yes to the record ask about the move of this module on node to the composition
// with this digest (ADR 0539 §14). Nil: none did.
Allowed func(node, digest string) bool
// Released is a hold on FP the operator released (ADR 0539 §19): FP is not held again by a health check that failed.
Released bool
// Restored says whether the operator answered "restore" to the restore ask of this data module on node: it is then set back
// as any module is, losing the writes made since (ADR 0539 §19). Nil: none did.
Restored func(node string) bool
// Kept says the operator answered "keep" to the hold ask of FP: a hold kept so is not lifted by itself.
Kept bool
// Now is when this pass reads the haves: the held record names it as when the health check was seen to fail. Zero: not said.
Now time.Time
// Puts are the puts of the module, by node (ADR 0546): while one owns a node's want (PutOwns), the rules pass over it.
// Puts are the puts of the module, by node (ADR 0546): while one owns a node's want (PutOwns), the rule passes over it.
Puts map[string]puts.Put
// Needs are the reader capabilities the manifest at FP needs (the indexer's modules value, catalogue.ManifestNeeds, ADR
// 0614 §4): a node's want moves only while that node's engine states each of them. Nil: none.
Needs []string
// FormNeeds are the reference forms the module's build at FP is written in (`form:<name>`, the builder's build key's
// needs, ADR 0618 §1): a node's want is held only while that node's engine reads none of them (ADR 0624 §2). Nil: the
// build names none, and holds nothing.
FormNeeds []string
// Self says whether node composes the module itself: the catalogue's assignments names the node (ADR 0547 §2). It is the one
// source of a want's Self: every want the rules write or keep is derived from it (selfDerived), so a want written Self while
// the node was placed does not outlive the assignments (issue 1126). Nil: no node does.
// Self says whether node composes the module itself: the catalogue's assignments names the node (ADR 0547 §2). Nil: no
// node does.
Self func(node string) bool
// KeptBuilds answers what is known of the build at a fingerprint, its released version among it (design 47 G §379, A-96). Nil:
// none is read.
KeptBuilds func(fp string) (KeptBuild, bool)
// MachineHeld says whether the node's own network condition is open, which holds the machine as a whole: no want moves to it
// until that clears (design 47 G §706 *M's own network*). Nil: none is held.
MachineHeld func(node string) bool
}
// KeptBuild is what the assigner knows of a build: the digest of each of its parts by `<module>:<artifact>` and the build node that
// keeps it, as the builder's `build.ended` says them, by which its released version is found; for a trunk build the released version
// the catalog's `version-released` says (design 47 G §379, A-96). A want names the version alone (design 47 H §750).
type KeptBuild struct {
Parts map[string]string
KeptOn string
Version string
}
// withKept is w naming the released version of the build at its fingerprint; a want names a version, never a part's digest or a
// keeping node (design 47 H §750, §764: A-436's digests end, G §379's go), so a build with no version known is named by its
// fingerprint alone.
func (n Nodes) withKept(m Module, node string, w mwc.Want) mwc.Want {
if m.KeptBuilds == nil {
return w
}
if k, ok := m.KeptBuilds(w.Fp); ok && k.Version != "" {
w.Version = k.Version
}
return w
}
// NameKept is w naming the build at its fingerprint as the want of a module does (withKept), for a want held as it stands.
func (n Nodes) NameKept(m Module, node string, w mwc.Want) mwc.Want { return n.withKept(m, node, w) }
func (m Module) selfOn(node string) bool { return m.Self != nil && m.Self(node) }
func (m Module) composition(node, fp string) (ComposedDigest, bool) {
if m.Composed == nil {
return ComposedDigest{}, false
}
return m.Composed(node, fp)
}
// Node is what the rules read of the nodes: whether each is online (the watchman's presence) and each module's have.
// Nodes is what the pass reads of the nodes: whether each is online (the watchman's presence), each module's have and the
// node-engine seat's last `running`.
type Nodes struct {
Online map[string]bool
Haves map[string]map[string]nodeengine.HaveByNodeModuleValue
// Running is, by node and module, the last `running` the node-engine seat said (design 47 G §309): a provider runs healthy
// while the build it runs does, as this says it, whatever a newer want of it waits for (rule 5, main file *A provider
// before its consumers*). Nil: the seat's events are not heard, and the have says it.
// Running is, by node and module, the last `running` the node-engine seat said (design 47 G §309). Nil: the seat's events
// are not heard.
Running map[string]map[string]nodeengine.Running
// Providers is, by provision, the modules that provide it, with the nodes each is assigned to.
Providers map[string]map[string][]string
// Reads is, by node, the reader capabilities its node-engine states in its node record (nodeengine.NodeRecord.ManifestReads,
// ADR 0614 §4). A node with none, or none read, is read as stating the baseline (catalogue.StatedManifestReads).
// Reads is, by node, the reader capabilities its node-engine states (nodeengine.NodeRecord.ManifestReads, ADR 0614 §4).
Reads map[string][]string
}
// healthy says whether a have runs fp healthy.
func healthy(h nodeengine.HaveByNodeModuleValue, fp string) bool {
return h.Fp == fp && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK
}
// Failed says whether a have's health check failed once the node is done moving.
func Failed(h nodeengine.HaveByNodeModuleValue) bool { return failed(h) }
// Canary is the module's canary: the first of its nodes, by name, that is online (ADR 0435 §3; one ring, ordered by
// name, ADR 0528 clause 3). Empty when none is online.
func Canary(nodes []string, online map[string]bool) string {
sorted := slices.Sorted(slices.Values(nodes))
for _, n := range sorted {
if online[n] {
return n
}
}
return ""
}
// Holds is whether every rule holds for moving node's want for m to m.FP, and if not, the first rule that does not,
// with why (ADR 0429 §6). Rule 4, at most `step` others ahead, always holds: the mesh keeps no step, and step 0 is no
// limit (ADR 0528 clause 3). Rule 5, every provider healthy, holds no want (issue 1960): a module whose provider is down
// is wanted all the same, and said as not reached (Unreached), since a site runs while the proxy in front of it is
// down and one module never blocks another.
func Holds(m Module, node string, n Nodes) (bool, int, string) {
return holds(m, node, n, true)
}
// holds is Holds; with ordering false it leaves out the canary (3), which orders a move among nodes, and keeps those
// without which there is nothing to want: the build done, not held, the grant and the composition.
func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
switch {
case !m.Built:
if m.BuildFailed {
return false, mwc.RuleBuildDone, fmt.Sprintf("the build of %s at %s failed: %s", m.Name, short(m.FP), m.BuildError)
}
return false, mwc.RuleBuildDone, fmt.Sprintf("the build of %s at %s is being built", m.Name, short(m.FP))
case m.Held:
return false, mwc.RuleNotHeld, fmt.Sprintf("%s is held at %s", m.Name, short(m.FP))
case m.MachineHeld != nil && m.MachineHeld(node):
return false, mwc.RuleNotHeld, fmt.Sprintf("the condition machine.%s.network is open: no want moves to %s until its own network is healthy", node, node)
}
if ordering {
canary := Canary(m.Nodes, n.Online)
switch {
case canary == "":
return false, mwc.RuleCanary, fmt.Sprintf("none of the nodes of %s is online, so it has no canary", m.Name)
case node != canary && !n.RunsHealthy(canary, m.Name, m.FP):
return false, mwc.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP))
}
}
if !m.Granted {
return false, mwc.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s for %s", m.Name, short(m.FP))
}
if m.GrantedOn != nil && !m.GrantedOn(node) {
return false, mwc.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s on %s for %s", m.Name, node,
short(m.FP))
}
if words, lacking := m.readerLacking(node, n); lacking {
return false, mwc.RuleComposed, words
}
if _, ok := m.composition(node, m.FP); !ok {
return false, mwc.RuleComposed, fmt.Sprintf("%s has named no composition of its own for %s at %s yet", node, m.Name, short(m.FP))
}
if words, waiting := m.provisionWaiting(node); waiting {
return false, mwc.RuleProvisions, words
}
if words, waiting := m.verbWaiting(node); waiting {
return false, mwc.RuleProvisions, words
}
if words, waiting := m.busStateWaiting(node); waiting {
return false, mwc.RuleProvisions, words
}
return true, 0, ""
}
// readerLacking says why the module's want on node is held for the node's engine (ADR 0614 §4): a reader capability its
// manifest needs that the engine does not state, read as the baseline when it states none, a bootstrapped seat counting as
// read by every engine (graph.MissingReads, ADR 0636 §2). It is held under rule 7 (the
// composition), since no rule of its own is numbered in this release, with words naming the capability; it moves once the
// engine states it. The node-engine's own new build is held only by a word of its own manifest no engine states.
//
// A build's reference forms hold the want only when the engine reads none of them (ADR 0624 §2): every engine reads a
// first form, so a build written in one is never held by its forms; the words name the forms it reads none of.
func (m Module) readerLacking(node string, n Nodes) (string, bool) {
missing := graph.MissingReads(m.Needs, n.Reads[node])
if len(missing) == 0 && forms.Current.Held(m.FormNeeds, n.Reads[node]) {
missing = slices.Clone(m.FormNeeds)
}
if len(missing) == 0 {
return "", false
}
return graph.ReaderWaitWords(node, missing, m.Name, short(m.FP)), true // the words the delivery view names the edge by
}
// provisionWaiting says why the module's want on node waits for a provision (ADR 0570 §4): a provision it requires that has
// no key yet, or the first of its keys that is not provided. Nil Provisions (the seat's state not on the bus) waits for none.
func (m Module) provisionWaiting(node string) (string, bool) {
if m.Provisions == nil {
return "", false
}
if missing := m.MissingProvisions(node); len(missing) > 0 {
return fmt.Sprintf("provision %s has no key yet for %s on %s", missing[0], m.Name, node), true
}
keyed := m.Provisions(node)
// Only a provision the newest manifest still requires is waited on (ADR 0603 §9): a key of an older revision's edge,
// asked or retiring for the operator's answer (ADR 0570 §5), holds nothing. A requirement catalogue.Awaited leaves out is
// not waited on when something offers it (a module's own name, a module the mesh provides itself, a provision the
// composition does not key: optional, a toolchain on the module's own machine, a login the controller makes). One that
// nothing offers is waited on (ADR 0603 §5: an unknown provider has provided nothing), so a consumer never moves ahead of
// a provider whose module the indexer has not yet taken in; it holds loudly, since the watchman raises the hold past its
// bound (ADR 0603 §8).
needed := m.Manifest.Requires
if m.Answering != nil {
needed = catalogue.Awaited(m.Manifest, m.Answering)
have := map[string]bool{}
for _, p := range keyed {
have[p.Provision] = true
}
for _, r := range slices.Sorted(slices.Values(m.Manifest.Requires)) {
if have[r] && !m.Manifest.OptionalRequires[r] && !slices.Contains(needed, r) {
needed = append(needed, r) // the provisioner keys it from the manifests, whoever is placed yet
continue
}
if m.Offered == nil || m.Manifest.OptionalRequires[r] || len(m.Answering(r)) > 0 || m.Offered(r) || slices.Contains(needed, r) {
continue
}
needed = append(needed, r)
if !have[r] {
return fmt.Sprintf("no module offers %s yet for %s on %s: an unknown provider has provided nothing, so the want waits for one, and the watchman raises it past its bound", r, m.Name, node), true
}
}
}
for _, key := range slices.Sorted(maps.Keys(keyed)) {
if p := keyed[key]; p.State != meshprovisionerclient.Provided && slices.Contains(needed, p.Provision) {
words := fmt.Sprintf("the provision %s of %s on %s is %s, not provided", key, m.Name, node, p.State)
if p.Why != "" {
words += ": " + p.Why
}
return words, true
}
}
return "", false
}
// MissingProvisions are the provisions a want of the module waits for (catalogue.Awaited, the one rule the plan
// composition and the seat mesh-provisioner key by, issue 774) that have no key on node yet, sorted; none while the
// seat's state is not read. A requirement the composition keys nothing for (a toolchain on the module's own machine, a
// login the controller still makes for a module not handed) is not waited for: on 2026-10-30 every new build of
// grafana, dnsmasq, mesh-knowledge, mesh-mail and the builders waited here for a key nobody would make.
func (m Module) MissingProvisions(node string) []string {
if m.Provisions == nil || m.Answering == nil {
return nil
}
have := map[string]bool{}
for _, p := range m.Provisions(node) {
have[p.Provision] = true
}
var out []string
for _, r := range catalogue.Awaited(m.Manifest, m.Answering) {
if !have[r] {
out = append(out, r)
}
}
return out
}
// runsHealthy says whether module runs healthy on node: the node-engine seat's last `running` of it names a build, does not say it
// stopped and does not say `healthy` false (design 47 G §520); nothing heard of it counts as not healthy.
func (n Nodes) runsHealthy(node, module string) bool {
@@ -328,317 +79,49 @@ func (n Nodes) runsHealthy(node, module string) bool {
}
// RunsHealthy says whether the build of module at fp runs healthy on node, from the node-engine seat's last `running` of it alone
// (design 47 G §520): a `running` with no `healthy`, said by a node-engine of before G, reads as healthy while it stands, and a
// module of which nothing was heard counts as not healthy. The have is never read for it.
// (design 47 G §520). The have is never read for it.
func (n Nodes) RunsHealthy(node, module, fp string) bool {
r, ok := n.Running[node][module]
return ok && n.runsHealthy(node, module) && r.Fingerprint == fp
}
// providedHealthy is rule 5 for one provision: some module providing it runs healthy on the node, or, assigned
// elsewhere only, at its canary. A provision no module provides is not a module's to judge (the mesh or the node
// offers it) and holds.
func providedHealthy(provision, node string, n Nodes) (bool, string) {
providers := n.Providers[provision]
if len(providers) == 0 {
return true, ""
}
names := make([]string, 0, len(providers))
for p := range providers {
names = append(names, p)
}
sort.Strings(names)
for _, p := range names {
nodes := providers[p]
at := node
if !slices.Contains(nodes, node) {
at = Canary(nodes, n.Online)
}
if at != "" && n.runsHealthy(at, p) {
return true, ""
}
}
return false, fmt.Sprintf("no provider of %s runs healthy on %s or at its canary", provision, node)
}
// unreached is why m on node may be wanted and not reached: the first provision it requires that no provider runs
// healthy on node or at its canary (rule 5, said and never holding a want; issue 1960); empty when all do.
func unreached(m Module, node string, n Nodes) string {
for _, provision := range m.Manifest.Requires {
if ok, why := providedHealthy(provision, node, n); !ok {
return why
}
}
return ""
}
// keepsData says whether a module holds data a set-back would lose: an own item of a class that keeps (irreplaceable,
// valuable or rebuildable; cache and none do not).
func keepsData(m catalogue.Manifest) bool {
for _, it := range m.DataItems() {
if it.Class == catalogue.ClassIrreplaceable || it.Class == catalogue.ClassValuable || it.Class == catalogue.ClassRebuildable {
return true
}
}
return false
}
// Outcome is what one module's pass decides: each node's want to write (only those that change), and a hold to write.
// Outcome is what one module's pass decides: each node's want to write (only those that change).
type Outcome struct {
Wants map[string]mwc.Want
// Held is the hold of m.FP to write, when a node's health check failed on it and it is not held yet.
Held *mwc.Held
// Lifted is the hold of m.FP lifted by itself: every node that runs the module runs m.FP healthy (issue 1906). The held
// record and its ask are taken away, and the wants move by the rules again in this pass.
Lifted bool
// NodeFailed is, by node, a failed health check of m.FP on a node while another node runs m.FP healthy: the failure is
// that node's, so m.FP is not held and nothing is set back (ADR 0557 §1, issue 1496). The pass raises its condition once.
NodeFailed map[string]NodeFailure
// Record is the moves a person must allow before the want names them: each is asked once (ADR 0539 §14).
Record []RecordNeed
// Restore is the nodes of a module holding data whose held fingerprint failed its health check there and that are not set
// back: the operator is asked whether to restore each (ADR 0539 §19).
Restore []string
// Rules is, by node, the rule holding the node back from m.FP and why.
Rules map[string]Holding
// Unreached is, by node, why the module may be wanted there and not reached: no provider of a provision it requires runs
// healthy (rule 5, issue 1960). It holds no want back; the pass says it.
Unreached map[string]string
// Neither is, by node, the nodes the module is assigned to that run it (or whose have names it), hold no want for it after this
// pass and are not held (a held module is on the held list), with the rule holding the first want back: the node's own composition does not hold a module with a fingerprint of the new form, so such a module is on
// neither list and nothing applies it on that node (issue 1945). A module with no fingerprint of the new form goes with the node's
// own composition and is never here.
Neither map[string]Holding
}
// RecordNeed is a move of a module of policy record on a node that waits for the operator's yes: the composition's digest, the
// controller's judgement of the move and its words.
type RecordNeed struct {
Node string
Digest string
Judgement Judgement
Words string
}
// Holding is a rule holding a node back, and why. The bus module is held back by the rules alone, as every module (ADR 0601 §1,
// §2): no bar of its own.
type Holding struct {
Rule int
Words string
}
// Reason is the holding in words: the rule and its words.
func (h Holding) Reason() string {
return fmt.Sprintf("rule %d: %s", h.Rule, h.Words)
}
// Wants is the pass of one module m: for each of its nodes, the want by the six rules, starting from the want written
// before (wants, by node); lastHealthy is, by node, the newest fingerprint other than m.FP the node ran healthy. A failed
// health check of m.FP on a node holds it: every node wanting m.FP is set back to its last healthy fingerprint, but a
// module holding data is not set back but asked about (Restore). A
// node that wants m but is no longer among its nodes gets its want written removed, never deleted (ADR 0408 §4).
func Wants(m Module, n Nodes, wants map[string]mwc.Want, lastHealthy map[string]string) Outcome {
out := Outcome{Wants: map[string]mwc.Want{}, Rules: map[string]Holding{}, Neither: map[string]Holding{}}
held := m.Held
if held && !m.Kept && m.FP != "" && allRunHealthy(m, n) {
out.Lifted, held = true, false
}
if !held && !out.Lifted && !m.Released && m.FP != "" {
for _, node := range slices.Sorted(slices.Values(m.Nodes)) {
h, ok := n.Haves[node][m.Name]
if !ok || h.Fp != m.FP || !failed(h) {
continue
}
if others := healthyElsewhere(m, n, node); len(others) > 0 && !failedElsewhere(m, n, node) {
// F runs healthy on another node, so its failure here is this node's, not F's: F is not held, no node is set back,
// and the failing node keeps its want (ADR 0557 §1; issue 1496: on ace a newly assigned mesh-build-node failed its
// first check and the hold moved shanks, g14 and novox, healthy on F for an hour, back). The pass raises the node's
// condition. With no online healthy witness, or with F failed on a second node, the hold and the set-back stand (ADR 0557
// §2, ADR 0429 §6).
if out.NodeFailed == nil {
out.NodeFailed = map[string]NodeFailure{}
}
out.NodeFailed[node] = NodeFailure{Others: others, Words: HeldWords(h, time.Time{})} // said once: no clock in it
continue
}
out.Held = &mwc.Held{Node: node, Words: HeldWords(h, m.Now)}
held = true
break
}
}
mm := m
mm.Held = held
// Wants is the pass of one module m: for each of its nodes, the want naming its Target, starting from the want written before
// (wants, by node). A put that owns a node's want writes the put's; a node with no target keeps the want written. A node that
// wants m but is no longer among its nodes gets its want written removed, never deleted (ADR 0408 §4).
func Wants(m Module, wants map[string]mwc.Want) Outcome {
out := Outcome{Wants: map[string]mwc.Want{}}
for _, node := range m.Nodes {
if m.FP == "" {
break
}
before, had := wants[node]
var next mwc.Want
p, put := m.Puts[node]
if !put && had && before.Put && before.Hold {
// The want is the truth (ADR 0546 clause 5): a want marked put and held stands while its put is not heard (the
// puts not open, or not yet handed after a start); only a put heard released gives it back to the rules. A put
// without hold is not kept this way: its put takes it further once heard.
p, put = puts.Put{FP: before.Fp, Composed: before.Composed, Hold: true, Why: strings.TrimPrefix(before.Words, "put: "),
Step: puts.Holding}, true
// The want is the truth (ADR 0546 clause 5): a want marked put and held stands while its put is not heard; only a put
// heard released gives it back to the rule.
p, put = puts.Put{FP: before.Fp, Hold: true, Why: strings.TrimPrefix(before.Words, "put: "), Step: puts.Holding}, true
}
if put {
if want, owns := PutOwns(p); owns {
if want != nil {
w := *want
// A held bus still takes a revision of its composition that changes only its user list (ADR 0539 §7).
if ul, ok := m.userListRevision(node, before, had, n); ok && ul.Fp == w.Fp {
w.Composed = ul.Composed
}
if !had || !mwc.SameWant(normal(before), normal(w)) {
out.Wants[node] = w
}
if want != nil && (!had || !mwc.SameWant(normal(before), normal(*want))) {
out.Wants[node] = *want
}
continue
}
}
// A revision of the bus module's composition that differs only in its user list is written at once, at the fingerprint the
// node runs, outside the rules, the record hold and the bar on a new fingerprint: a new module's credential must reach the
// bus before the module can connect (ADR 0539 §7). The controller marks it; whatever it judges of the file's mount stands.
if w, ok := m.userListRevision(node, before, had, n); ok {
if !had || !mwc.SameWant(normal(before), normal(w)) {
out.Wants[node] = w
}
t, ok := Target{}, false
if m.Target != nil {
t, ok = m.Target(node)
}
if !ok || t.Fp == "" {
continue
}
ok, rule, words := Holds(mm, node, n)
target := m.FP
if have, ran := n.Haves[node][m.Name]; !ok && (!had || before.Removed) && rule == mwc.RuleCanary {
// The canary orders a move, never the first want of a node that does not run the module (issue 1960: de-spiegel,
// link2pay and photos on novox were left with no want). A node that runs an older fingerprint of the new form,
// composed and granted, gets its first want at what it runs, and rule 3 then orders its move to F as from any want.
// A node that runs an old-form build, which no want can name, stays under the canary (issue 1945, a design limit):
// its first want names F once the canary runs F healthy. Any other node that runs it (F itself, or an older
// fingerprint with no composition or grant) gets F by the rules without which there is nothing to want.
older := ran && runs(have) && have.Fp != m.FP && strings.HasPrefix(have.Fp, meshcatalogclient.SourcePrefix2)
if older {
_, composed := m.composition(node, have.Fp)
older = composed && m.GrantedAt != nil && m.GrantedAt(have.Fp)
}
oldForm := ran && runs(have) && have.Fp != m.FP && !strings.HasPrefix(have.Fp, meshcatalogclient.SourcePrefix2)
switch {
case older:
ok, target = true, have.Fp
case oldForm:
// held back by rule 3 as written: on neither list until the canary runs F healthy
default:
ok, rule, words = holds(mm, node, n, false)
}
}
if why := unreached(m, node, n); why != "" {
if out.Unreached == nil {
out.Unreached = map[string]string{}
}
out.Unreached[node] = why
}
if ok && target == m.FP && len(out.NodeFailed) > 0 && !(had && !before.Removed && before.Fp == m.FP) &&
n.Haves[node][m.Name].Fp != m.FP {
// While a node fails F alone, F reaches no further node: a node that neither wants nor runs F gets no new want
// naming it, and wants already written stay (ADR 0557 §2). A node that runs F with no live want gets its want at F, or
// it would be on neither list and its node-engine would give the module up (issue 1945).
ok, rule, words = false, mwc.RuleNotHeld, FailedAloneWords(m.Name, m.FP, slices.Sorted(maps.Keys(out.NodeFailed)))
}
switch {
case ok:
d, _ := m.composition(node, target)
next = n.withKept(m, node, mwc.Want{Fp: target, Composed: d.Digest})
at := m
if target != m.FP {
// the first want at the older fingerprint the node runs: held back from F by the rule, and judged as a want at it
next.Newest, next.Rule, next.Words = m.FP, int64(rule), words
at.FP = target
}
// A placed node's want names the build alone: it is the same target while the build is.
if had && !before.Removed && !before.Put && before.Fp == target && (before.Composed == d.Digest || (before.Self && m.selfOn(node))) {
next.Backup, next.Words = before.Backup, before.Words // the answer to a failed backup stays while its target stays (ADR 0539 §16)
}
running, ran := n.Haves[node][m.Name]
// A removed want is no want: a first want after it goes through the record ask as a first want does.
asked, hadLive := before, had
if before.Removed {
asked, hadLive = mwc.Want{}, false
}
if need, wait := at.mustAsk(node, asked, hadLive, ran, running.Fp, running.Composed == "", d); wait {
if need != nil {
out.Record = append(out.Record, *need)
}
continue
}
case held && had && before.Fp == m.FP:
out.Rules[node] = Holding{Rule: rule, Words: words}
last := lastHealthy[node]
switch {
case keepsData(m.Manifest) && (m.Restored == nil || !m.Restored(node)):
next = before
next.Words = fmt.Sprintf("%s holds data, so it is not set back from %s: the operator is asked whether to restore it", m.Name, short(m.FP))
out.Restore = append(out.Restore, node)
case last == "":
next = before
next.Words = fmt.Sprintf("%s is held at %s and %s ran no other fingerprint healthy to set it back to", m.Name, short(m.FP), node)
default:
d, there := m.composition(node, last)
if !there {
next = before
next.Words = fmt.Sprintf("%s is held at %s, and the composition of %s at %s is gone, so %s is not set back", m.Name, short(m.FP), m.Name, short(last), node)
break
}
next = mwc.Want{Fp: last, Composed: d.Digest, Newest: m.FP, Rule: int64(rule), Words: words}
}
default:
out.Rules[node] = Holding{Rule: rule, Words: words}
if !had || before.Removed {
// The node runs the module, or its have names it, and gets no want for it: the module is on neither list (issue 1945), and
// the pass says so. A new assignment, with no have, waits for the rules and is on no list yet by design. A held module (rule 2,
// held before or by this pass) is on the held list instead, whose hold is said and answered on its own: it is never on neither
// list (issue 1945: systemd-resolved, held on shanks by the operator, was said as urgent every hour).
h, there := n.Haves[node][m.Name]
switch {
case there && h.Phase != nodeengineclient.PhaseRemoved && !held:
out.Neither[node] = Holding{Rule: rule, Words: words}
case had && (!there || h.Phase == nodeengineclient.PhaseRemoved):
// A removed want of a node that does not run the module says why no want follows it (issue 1945), so the view's
// assigner_want answers it; it stays removed. A node with no key gets none: a removed want written where none was
// would have its node agent remove the module and stop holding its composition's resources back.
next = mwc.Want{Removed: true, Words: NoWantWords(rule, words)}
if !mwc.SameWant(normal(before), normal(next)) {
out.Wants[node] = next
}
}
continue
}
next = mwc.Want{Fp: before.Fp, Composed: before.Composed, Newest: m.FP, Rule: int64(rule), Words: words}
if before.Fp == m.FP {
// Held at F by the rules: a new revision of F's composition waits for them, as a new fingerprint does (ADR 0539 §5).
next = mwc.Want{Fp: m.FP, Composed: before.Composed}
} else if d, there := m.composition(node, before.Fp); there && d.Digest != "" && d.Digest != before.Composed {
// Held back at an earlier fingerprint whose kept composition the controller composed again with today's config
// (novox/hq ADR 0545): the want follows its new revision, so the node does not run that build with the config of
// the moment it was made (on g14 a capacity of 0 after a rollback). Asked as any move of a record module is.
at := m
at.FP = before.Fp
running, ran := n.Haves[node][m.Name]
need, wait := at.mustAsk(node, before, true, ran, running.Fp, running.Composed == "", d)
if need != nil {
out.Record = append(out.Record, *need)
}
if !wait {
next.Composed = d.Digest
}
}
}
next, write := m.selfDerived(node, before, had, next, n, &out)
if !write {
continue
}
// Every want that names a build names its version (A-430a), whichever rule made or kept it: a want held at its fingerprint
// by the rules, a node's own (self) want too.
if !next.Removed && next.Fp != "" && next.Version == "" {
next = n.withKept(m, node, next)
next := mwc.Want{Fp: t.Fp, Version: t.Version, Pipeline: t.Pipeline, Rank: t.Rank, Self: m.selfOn(node)}
if had && !before.Removed && !before.Put && before.Fp == t.Fp {
// The answers the want carries stay while its build does: a failed backup's (ADR 0539 §16), a held revert's, the yes.
next.Backup, next.Words, next.Revert, next.Yes, next.Release = before.Backup, before.Words, before.Revert, before.Yes, before.Release
}
if !had || !mwc.SameWant(normal(before), normal(next)) {
out.Wants[node] = next
@@ -652,143 +135,17 @@ func Wants(m Module, n Nodes, wants map[string]mwc.Want, lastHealthy map[string]
return out
}
// selfDerived is next with its Self derived from the assignments now (Module.Self), whichever rule made it, and whether it is
// written. A placed node's want names the build alone (ADR 0547 §2). An unplaced node's want that a rule kept from the want
// written while the node was placed (held at F, held back by rule 3 or 5, a data module asked about) names the controller's
// composition of the same fingerprint, asked as any move of a record module is (ADR 0539 §14): on 2026-10-27, after the
// assignments seed's revert, g14's systemd-resolved (held) and forticlient (rule 5), and shanks's forticlient (rule 3), kept
// their Self wants, which their node agents, no longer placed, could not compose (issue 1126). With no such composition, or
// while its ask waits, the want written stays and the node's holding says why.
func (m Module) selfDerived(node string, before mwc.Want, had bool, next mwc.Want, n Nodes, out *Outcome) (mwc.Want, bool) {
switch {
case next.Removed:
return next, true
case m.selfOn(node):
next.Self, next.Composed = true, ""
return next, true
case !next.Self && next.Composed != "":
return next, true
}
next.Self, next.Yes = false, ""
d, there := m.composition(node, next.Fp)
if !there || d.Digest == "" {
out.Rules[node] = Holding{Rule: mwc.RuleComposed, Words: fmt.Sprintf("%s on %s was composed by the node while it was placed, "+
"and no composition of its own is named at %s now, so its want stays", m.Name, node, short(next.Fp))}
return next, false
}
at := m
at.FP = next.Fp
running, ran := n.Haves[node][m.Name]
if need, wait := at.mustAsk(node, before, had, ran, running.Fp, running.Composed == "", d); wait {
if need != nil && !slices.Contains(out.Record, *need) { // the branch held back at an older fingerprint may have asked it
out.Record = append(out.Record, *need)
}
return next, false
}
next.Composed = d.Digest
return next, true
}
// NoWantWords say why a node gets no want for a module: the rule that holds its first want back and why (issue 1945).
func NoWantWords(rule int, words string) string {
return fmt.Sprintf("no want: rule %d: %s", rule, words)
}
// failed says whether a have's health check failed once the node is done moving: phase running or failed (never removed, held, or
// moving). A failed health while it
// backs up, applies or checks is a step of the move, not its outcome (on novox at 15:27:36 on 2026-10-25, "health failed, phase
// applying" held the controller's canary two seconds before the new build was healthy). Every healthy read (healthy, providedHealthy,
// the world's lastHealthy) already asks for phase running.
// moving).
func failed(h nodeengine.HaveByNodeModuleValue) bool {
return h.Health == nodeengineclient.HealthFailed && (h.Phase == nodeengineclient.PhaseRunning || h.Phase == nodeengineclient.PhaseFailed)
}
// healthyElsewhere is every online node of m but node that runs m.FP healthy, by name.
func healthyElsewhere(m Module, n Nodes, node string) []string {
var out []string
for _, other := range slices.Sorted(slices.Values(m.Nodes)) {
// A witness is online: a node that sleeps keeps the have it last wrote, and a stale have is no evidence that F runs
// healthy now.
if other != node && n.Online[other] && n.RunsHealthy(other, m.Name, m.FP) {
out = append(out, other)
}
}
return out
}
// failedElsewhere says whether another node of m has a have naming m.FP failed: then the failure is not one node's alone, and
// the hold and the set-back stand at the second failure (ADR 0557 §2).
func failedElsewhere(m Module, n Nodes, node string) bool {
for _, other := range m.Nodes {
if h, ok := n.Haves[other][m.Name]; ok && other != node && h.Fp == m.FP && failed(h) {
return true
}
}
return false
}
// FailedAloneWords say why a node gets no new want naming fp: fp fails its health check on the nodes named, alone.
func FailedAloneWords(module, fp string, failing []string) string {
return fmt.Sprintf("%s at %s fails its health check on %s, so no further node moves to it until that is over", module, short(fp),
strings.Join(failing, ", "))
}
// NodeFailure is a failed health check of a module's newest fingerprint on one node while it runs healthy on Others: the have's
// health, phase and words (HeldWords without a time), and the healthy witnesses by name.
type NodeFailure struct {
Others []string
Words string
}
// runs says whether a have is of a module the node runs: running, whatever its health.
func runs(h nodeengine.HaveByNodeModuleValue) bool {
return h.Phase == nodeengineclient.PhaseRunning
}
// allRunHealthy says whether every node of m that runs the module (its have's phase is running, or failed) runs m.FP healthy, and
// at least one does: a hold whose fingerprint runs healthy wherever the module runs holds nothing back any more (issue 1906: seven
// modules stayed held on every machine after one transient failed check). A node set back to another fingerprint does not run
// m.FP, and a failed one does not run it healthy, so the hold stays; a have that is removed, held for its backup or still moving
// (backing up, applying, checking) is not counted, or it would keep the hold up for ever.
func allRunHealthy(m Module, n Nodes) bool {
ran := 0
for _, node := range m.Nodes {
h, ok := n.Haves[node][m.Name]
if !ok || (h.Phase != nodeengineclient.PhaseRunning && h.Phase != nodeengineclient.PhaseFailed) {
continue
}
if !n.RunsHealthy(node, m.Name, m.FP) {
return false
}
ran++
}
return ran > 0
}
// HeldWords are the words a hold keeps of its cause (issue 1906): the health and phase the node reported, when the assigner saw
// it, and the node's own words. The node is the held record's own field.
func HeldWords(h nodeengine.HaveByNodeModuleValue, at time.Time) string {
words := fmt.Sprintf("health %s, phase %s", h.Health, h.Phase)
if !at.IsZero() {
words += ", seen at " + at.UTC().Format(time.RFC3339)
}
if d := strings.TrimSpace(h.Detail); d != "" {
return words + ": " + d
}
return words + ": the node gave no words"
}
// LiftedWords say why a hold was lifted by itself.
func LiftedWords(module, fp string) string {
return fmt.Sprintf("every node that runs %s runs %s healthy, so the assigner lifted its hold by itself", module, short(fp))
}
// normal is a want without its version, as a decoded and a composed one are compared.
func normal(w mwc.Want) mwc.Want {
w.V = 0
return w
}
func short(fp string) string {
part, err := catalogkey.KeyPart(fp)
if err != nil || len(part) < 12 {
@@ -797,69 +154,20 @@ func short(fp string) string {
return part[:12]
}
// mustAsk says whether the move to the composition d on node waits for the operator's yes (need names the ask, ADR 0539 §14). A
// module the node never ran (it has no want and no have), a move judged neither, a module that is not of policy record and a want
// that stays are written as any want. The bus module is no exception: it moves by the rules (ADR 0601 §1), its node backing up its
// streams before it applies (§2); the assigner asks no snapshot for it.
func (m Module) mustAsk(node string, before mwc.Want, had, ran bool, running string, haveNamesNoComposition bool, d ComposedDigest) (*RecordNeed, bool) {
if (!had && !ran) || before.Removed || (before.Fp == m.FP && before.Composed == d.Digest) {
return nil, false
}
// Right after the switch no node has a readable want, and a have from the old node-engine names a fingerprint and no composition
// (ADR 0539 §9, §11): a node running this fingerprint is given its want at it with no hold when the controller judges the composition
// unchanged, equal to the declaration the node last applied (ADR 0540 §9), since the node agent then applies nothing. A composition that
// differs from it (a changed setting or secret), or that the controller could not judge, waits for the ask. A new fingerprint still waits.
if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == JudgedUnchanged {
return nil, false
}
if !m.recorded(d) || d.Judgement == JudgedNeither || d.Judgement == JudgedUnchanged {
return nil, false
}
if m.Allowed != nil && m.Allowed(node, d.Digest) {
return nil, false
}
return &RecordNeed{Node: node, Digest: d.Digest, Judgement: d.Judgement, Words: d.Words}, true
// RecordNeed is a move of a module of policy record on a node that waits for the operator's yes: the digest the node-engine's have
// waits on, its judgement of the move and its words (ADR 0539 §14).
type RecordNeed struct {
Node string
Digest string
Judgement Judgement
Words string
}
// recorded says whether the module is of policy record: a person's choice at the assigner over the manifest's word (ADR
// 0565 §7, catalogue.Upgrade); for a module with none, as the controller holds it beside the composition's digest until the
// lease write (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it.
func (m Module) recorded(d ComposedDigest) bool {
if m.Chosen != nil {
policy, _, _, _ := catalogue.Upgrade(m.Manifest, m.Chosen)
return policy == catalogue.PolicyRecord
}
if d.Policy != "" {
return d.Policy == catalogue.PolicyRecord
}
policy, _, _ := catalogue.DerivedUpgrade(m.Manifest)
return policy == catalogue.PolicyRecord
}
// userListRevision is the want that names the bus module's composition at the fingerprint the node runs when that composition
// differs from the one the node runs only in the bus's user list (ComposedDigest.UserListOnly), and whether there is one. A node
// whose want names another fingerprint than the one it runs, or a module that does not provide the bus, has none.
func (m Module) userListRevision(node string, before mwc.Want, had bool, n Nodes) (mwc.Want, bool) {
if !catalogue.ProvidesBus(m.Manifest) {
return mwc.Want{}, false
}
running, ran := n.Haves[node][m.Name]
if !ran || running.Fp == "" || (had && (before.Removed || before.Fp != running.Fp)) {
return mwc.Want{}, false
}
d, ok := m.composition(node, running.Fp)
if !ok || !d.UserListOnly {
return mwc.Want{}, false
}
w := mwc.Want{Fp: running.Fp, Composed: d.Digest}
if had && before.Composed == d.Digest {
w.Backup, w.Words = before.Backup, before.Words
}
return w, true
}
// GrantNamesNode says whether a grant of module names its accounts on node (mesh-sdk's Grant.NamesNode, which a
// put reads by too).
func GrantNamesNode(g grantsstate.Grant, node, module string) bool {
return g.NamesNode(node, module)
// KeptBuild is what the assigner knows of a build: the digest of each of its parts by `<module>:<artifact>` and the build node that
// keeps it, as the builder's `build.ended` says them, and its released version as the catalog's `version-released` says it (design
// 47 G §379, A-96). A want names the version alone (design 47 H §750).
type KeptBuild struct {
Parts map[string]string
KeptOn string
Version string
}
File diff suppressed because it is too large Load Diff
@@ -1,19 +0,0 @@
package assigngraph
import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// Runner is the module that holds the seat mesh-pipeline-runner, read from the acting edges' seat
// (A-334); "" when none is read.
func Runner(entries []meshwants.AssignmentEntry) string {
for _, e := range entries {
for _, edge := range e.Edges {
if edge.Kind == EdgeActing && edge.Seat == meshpipelinerunner.Seat {
return edge.Module
}
}
}
return ""
}
@@ -63,10 +63,6 @@ const (
Graph = IndexerModule + "_graph"
ModulesRead = IndexerModule + "." + catalog.StateModules
ManifestsRead = IndexerModule + "." + catalog.StateManifests
// ProvisionerModule is the holder of the seat mesh-provisioner, named from this module's manifest (reads its provisions); Provisions
// is its state provisions.
ProvisionerModule = "mesh-provisioner"
Provisions = ProvisionerModule + "_provisions"
// TellerModule is the module whose hand the operator's answers pass: the one that holds each ask's answer, and the
// name the hand-act log records a warrant under. Held equal to the manifest's reads by its test.
TellerModule = "mesh-delivery-teller"
@@ -12,9 +12,9 @@ func TestBucketsAreTheEarlierNames(t *testing.T) {
for name, c := range map[string][2]string{
"Have": {Have, "node-engine_have"}, "ComposedRead": {ComposedRead, "mesh-host.composed"},
"Modules": {Modules, "mesh-delivery-indexer_modules"}, "Manifests": {Manifests, "mesh-delivery-indexer_manifests"},
"Answers": {Answers, "mesh-delivery-teller_answers"},
"Grants": {Grants, "mesh-grants_grants"},
"Provisions": {Provisions, "mesh-provisioner_provisions"}, "TellerModule": {TellerModule, "mesh-delivery-teller"},
"Answers": {Answers, "mesh-delivery-teller_answers"},
"Grants": {Grants, "mesh-grants_grants"},
"TellerModule": {TellerModule, "mesh-delivery-teller"},
} {
if c[0] != c[1] {
t.Errorf("%s = %q, want %q", name, c[0], c[1])
@@ -87,21 +87,6 @@ func Causes(conds []Cond, m string, moved func(owner string) bool) []Cause {
return out
}
// OwnNetworkKey is the key of a machine's own open network condition, which holds the machine as a whole (§706 *M's own
// network*).
func OwnNetworkKey(m string) string { return "machine." + m + ".network" }
// OwnNetwork are the machines whose own network condition is open among conds.
func OwnNetwork(conds []Cond) map[string]bool {
out := map[string]bool{}
for _, c := range conds {
if p := strings.Split(c.Key, "."); len(p) == 3 && p[0] == "machine" && p[2] == "network" && c.Key == OwnNetworkKey(p[1]) {
out[p[1]] = true
}
}
return out
}
// Wait is one outside wait as the assigner keeps it: the move of Module at FP on Node, or its settle (Settling). Decided is
// what ended it for the rest of that fingerprint on that node: `go-on` (counted healthy by the operator's word) or `failed`
// (the delivery's failure), so a check that goes on failing is not waited on again.
@@ -31,9 +31,6 @@ func TestCauses(t *testing.T) {
if !slices.Equal(got, want) {
t.Errorf("causes = %v, want %v (a rewritten naming a moved module, an unreachable not listing ace and ace's own network are none)", got, want)
}
if own := OwnNetwork(conds); !own["ace"] || !own["hub2"] || own["hub"] {
t.Errorf("own network = %v", own)
}
}
func facts(now time.Time, conds ...Cond) Facts {
+13 -16
View File
@@ -50,14 +50,14 @@ func TestManifest(t *testing.T) {
// §431, §507: the gate's source is read through its verb `source` and its event `source-moved`, and no `merges` is read.
// A-175, A-178: the teller's answers are followed through its first view and its views family, never its bucket.
// §389: the watchman's machine-away and machine-back are consumed, never its presence.
want := []string{"mesh-grants.grants", "mesh-host.composed",
"mesh-provisioner.provisions", "node-engine.have"}
// A5/AC1 (design 47 H §750): no want waits for a provision, so the provisioner's provisions are no read; the grants stay
// for a put's own judgement (ADR 0546 clause 4).
want := []string{"mesh-grants.grants", "mesh-host.composed", "node-engine.have"}
if !reflect.DeepEqual(reads, want) {
t.Errorf("reads = %v, want exactly %v", reads, want)
}
// the buckets the program names are the manifest's reads, `<module>.<state>` as `<module>_<state>`
named := []string{buckets.Grants,
buckets.ComposedRead, "mesh-provisioner_provisions", buckets.Have}
named := []string{buckets.Grants, buckets.ComposedRead, buckets.Have}
slices.Sort(named)
for i, r := range want {
if got := strings.Replace(named[i], "_", ".", 1); got != r {
@@ -70,7 +70,7 @@ func TestManifest(t *testing.T) {
}
if want := map[string]int{"asks": 5, "differences": 64, "puts": 20, "wants": 20,
"assignments": 64, "placement": 64, "copies": 1, "proposals": 5, "candidates": 5, "pulses": 0, "findings": 5, "policies": 20, "seat-holders": 5, views.MarksState: 1, views.StatusState: 1,
"engine-reads": 1, "written": 5, "waits": 5, "put-waits": 1, "inputs": 1, "composition-runs": 1, "pins": 5}; !reflect.DeepEqual(states, want) {
"engine-reads": 1, "written": 5, "waits": 5, "put-waits": 1, "inputs": 1, "composition-runs": 1}; !reflect.DeepEqual(states, want) {
t.Errorf("state = %v, want %v", states, want)
}
if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 {
@@ -105,21 +105,18 @@ func TestManifest(t *testing.T) {
// which lists and asks again what each node's engine reads.
if !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts", "assigner_waits", "assigner_inputs", "assigner_inputs_drop", "assigner_compositions", "assigner_compositions_drop", "assignments", "seat-holders", "wants", "asks",
"assigner_read_assignments", "assigner_read_seat_holders", "assigner_read_wants", "assigner_read_findings", "assigner_read_differences", "assigner_read_copies",
"assigner_first_views", "assigner_first_view_again", "assigner_engine_reads", "assigner_seat_list", "assigner_seat_list_again", "assigner_ask_again", "assigner_written", "assigner_why", "assigner_pins", "assigner_unpin"}) {
"assigner_first_views", "assigner_first_view_again", "assigner_engine_reads", "assigner_seat_list", "assigner_seat_list_again", "assigner_ask_again", "assigner_written", "assigner_why"}) {
t.Errorf("the assigner serves tools %v, want its three put tools, three first views, six read tools, assigner_engine_reads, the seat list's two (ADR 0636 §7) assigner_ask_again (ADR 0668 §5) and assigner_why (design 47 G A-439)", m.Tools)
}
if m.RunsAs != m.Module || m.SecretsOwner != m.RunsAs {
t.Errorf("runs-as = %q: the bundle runs on the assigner's own account (ADR 0621 §1)", m.RunsAs)
}
// ADR 0621 §2: each first-view tool answers exactly its readers, never the assigner.
// Design 47 G §717: the teller and the status view take their first views of the seat mesh-wants' asks and wants, and of
// the assignments, from the tools named for the states.
callers := map[string]string{"assignments": "mesh-grants,node-tools,mesh-delivery-indexer,mesh-delivery-builder,mesh-delivery-checker,mesh-delivery-view",
"seat-holders": "node-tools,mesh-delivery-builder,mesh-delivery-view,mesh-delivery-teller,mesh-delivery-checker,mesh-delivery-indexer", "wants": "mesh-grants,mesh-delivery-view", "asks": "mesh-delivery-teller,mesh-delivery-view"}
env := m.Build.Artifacts[0].Env
for tool, list := range callers {
if env[views.CallersEnv(tool)] != list || views.ParseCallers(env[views.CallersEnv(tool)]).Allows("n1."+m.Module) {
t.Errorf("%s callers = %q, want %q", tool, env[views.CallersEnv(tool)], list)
// design 47 H §752, H-A-66: every first-view tool answers every caller, so the manifest names no callers list for any.
for _, a := range m.Build.Artifacts {
for k := range a.Env {
if strings.HasPrefix(k, "MESH_VIEW_CALLERS_") {
t.Errorf("the artifact %s names the callers list %s: no seat or tool keeps one (design 47 H §752)", a.Name, k)
}
}
}
// ADR 0635 §5: its views family is its ordinary events, `views.<state>` among its emits, on no stream of its own.
@@ -138,7 +135,7 @@ func TestManifest(t *testing.T) {
// issue 1966 note 151: and the teller's ask-ended, its subject the one the assigner hears; §389: the watchman's machine-away
// and machine-back, read whole at its start. A1/AC1 (design 47 H §750, §764): no version-delivered, delivery-waits,
// delivered-again or assigner_delivered_again: a want waits on no dependency's delivery.
if !slices.Equal(m.Consumes, []string{"mesh-conditions.findings-asked", "mesh-conditions.condition-raised", "mesh-conditions.condition-changed", "mesh-conditions.condition-cleared", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-delivery.source-moved", "mesh-catalog.version-released", "mesh-watchdogs.machine-away", "mesh-watchdogs.machine-back", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", builds.BuilderModule + "." + builds.BuildsStreamEvents, buckets.TellerModule + "." + tellerevents.AskEnded, buckets.TellerModule + ".views." + buckets.AnswersState, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-build-go.step-ended", "node-build-go.ask-taken", "node-build-go.step-progress", "node-engine.manifest-reads-changed", "node-engine.running", "mesh-delivery.pinned", "mesh-delivery.freed", "mesh-delivery.heads-replaced", "mesh-delivery.pins-stated"}) {
if !slices.Equal(m.Consumes, []string{"mesh-conditions.findings-asked", "mesh-conditions.condition-raised", "mesh-conditions.condition-changed", "mesh-conditions.condition-cleared", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-delivery.source-moved", "mesh-catalog.version-released", "mesh-watchdogs.machine-away", "mesh-watchdogs.machine-back", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", builds.BuilderModule + "." + builds.BuildsStreamEvents, buckets.TellerModule + "." + tellerevents.AskEnded, buckets.TellerModule + ".views." + buckets.AnswersState, "mesh-pipeline-runner.pipeline-ended", "node-build-go.step-ended", "node-build-go.ask-taken", "node-build-go.step-progress", "node-engine.manifest-reads-changed", "node-engine.running"}) {
t.Errorf("consumes = %v, want the catalog half's revisions (design 47 G §195), the teller's ask-ended, the engines' reads changed and what they run", m.Consumes)
}
if want := []string{"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules",
+4 -19
View File
@@ -34,7 +34,6 @@
"reads": [
"mesh-grants.grants",
"mesh-host.composed",
"mesh-provisioner.provisions",
"node-engine.have"
],
"uses": [
@@ -57,16 +56,11 @@
"mesh-delivery-teller.ask-ended",
"mesh-delivery-teller.views.answers",
"mesh-pipeline-runner.pipeline-ended",
"mesh-pipeline-runner.group-ended",
"node-build-go.step-ended",
"node-build-go.ask-taken",
"node-build-go.step-progress",
"node-engine.manifest-reads-changed",
"node-engine.running",
"mesh-delivery.pinned",
"mesh-delivery.freed",
"mesh-delivery.heads-replaced",
"mesh-delivery.pins-stated"
"node-engine.running"
],
"invokes": [
"mesh-delivery-indexer.manifests",
@@ -135,9 +129,7 @@
"assigner_seat_list_again",
"assigner_ask_again",
"assigner_written",
"assigner_why",
"assigner_pins",
"assigner_unpin"
"assigner_why"
],
"state": [
{ "name": "assignments", "history": 64 },
@@ -160,8 +152,7 @@
{ "name": "waits", "history": 5 },
{ "name": "put-waits", "history": 1 },
{ "name": "inputs", "history": 1, "max-mib": 64 },
{ "name": "composition-runs", "history": 1, "key-ttl": true, "max-mib": 64 },
{ "name": "pins", "history": 5, "max-mib": 4 }
{ "name": "composition-runs", "history": 1, "key-ttl": true, "max-mib": 64 }
],
"config": {
"outside-wait-minutes": {
@@ -344,13 +335,7 @@
"binary": "mesh-delivery-assigner",
"loads": [
"mesh-delivery-assigner"
],
"env": {
"MESH_VIEW_CALLERS_ASKS": "mesh-delivery-teller,mesh-delivery-view",
"MESH_VIEW_CALLERS_ASSIGNMENTS": "mesh-grants,node-tools,mesh-delivery-indexer,mesh-delivery-builder,mesh-delivery-checker,mesh-delivery-view",
"MESH_VIEW_CALLERS_SEAT_HOLDERS": "node-tools,mesh-delivery-builder,mesh-delivery-view,mesh-delivery-teller,mesh-delivery-checker,mesh-delivery-indexer",
"MESH_VIEW_CALLERS_WANTS": "mesh-grants,mesh-delivery-view"
}
]
}
]
}