diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildended_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildended_test.go index 8d50c068..37311826 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildended_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildended_test.go @@ -102,10 +102,6 @@ func TestATrunkEntryTakesItsVersionFromTheReleaseOfItsFingerprintsSource(t *test } // The release names each artifact by its whole reference, as the catalog says it, and the parts are keyed so (A-96); the // assigner reads the entry's fp-src from its inputs (A-419). - // A-429: a released version's build counts as done for rule 1; a kept build (no version) and an unknown one do not. - if !releasedBuild(p.namedBuild("mail", "t", 7), "t") || releasedBuild(p.namedBuild("mail", "t", 7), "k") || releasedBuild(p.namedBuild("mail", "t", 8), "t") || releasedBuild(nil, "t") { - t.Error("rule 1's released build: the release of the entry's fingerprint counts, a kept build, another source or none does not") - } if v, _, _ := fromEntry(meshcatalog.CatalogEntry{Module: "mail", Fp: "t", Inputs: &meshcatalog.EntryInputs{Src: 9, FpSrc: 7}}); v.FpSrc != 7 || v.Src != 9 { t.Errorf("the entry's module value carries fp-src %d and src %d, want 7 and 9", v.FpSrc, v.Src) } @@ -130,25 +126,3 @@ func TestATrunkEntryTakesItsVersionFromTheReleaseOfItsFingerprintsSource(t *test t.Error("no revision kept names a release") } } - -// A-162: the assigner knows a failed build from its `build.ended`: the error whole; a done end or none heard is not failed. -func TestTheStandingOfABuildIsHeardFromItsEnd(t *testing.T) { - w := newWorld() - say := func(e builds.BuildEnded) { - raw, _ := json.Marshal(e) - if err := w.ends.put(raw); err != nil { - t.Fatal(err) - } - } - if failed, _ := w.buildStanding("web", "fp1"); failed { - t.Error("a build nothing was heard of failed") - } - say(builds.BuildEnded{Module: "web", Fingerprint: "fp1", Outcome: builds.BuildFailed, Error: "transport fault: connection"}) - if failed, why := w.buildStanding("web", "fp1"); !failed || why != "transport fault: connection" { - t.Errorf("failed end: failed %v why %q", failed, why) - } - say(builds.BuildEnded{Module: "web", Fingerprint: "fp1", Outcome: builds.BuildDone}) - if failed, _ := w.buildStanding("web", "fp1"); failed { - t.Error("a later done end leaves the build failed") - } -} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go index c2aad0c6..aa066399 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go @@ -504,7 +504,7 @@ func caseOf(half mcc.Half) (map[string]composer.Manifest, map[string]json.RawMes if e.Verdict == "" || e.Verdict == mcc.VerdictAccepted { continue } - v := meshcatalog.VerdictsByDigestValue{V: 1, Module: name, State: e.Verdict} + v := caseVerdict{V: 1, Module: name, State: e.Verdict} if e.Verdict != mcc.VerdictRefused { v.State = mcc.VerdictManifestWaits } @@ -520,6 +520,17 @@ func caseOf(half mcc.Half) (map[string]composer.Manifest, map[string]json.RawMes return manifests, verdicts, none, nil } +// caseVerdict is a manifest's verdict as the composer's case reads it, by an entry's own verdict: the form the seat mesh-catalog's +// state verdicts had, which left the seat with the take-in's validation (design 47 H §745, H-A-79). +type caseVerdict struct { + V int64 `json:"v,omitempty"` + Module string `json:"module"` + State string `json:"state"` + Problems []string `json:"problems,omitempty"` + Unknown []string `json:"unknown,omitempty"` + Warnings []string `json:"warnings,omitempty"` +} + // nodeEngineEntry is the entry of the module that claims the seat node-engine, or false. func nodeEngineEntry(entries map[string]meshcatalog.CatalogEntry) (meshcatalog.CatalogEntry, bool) { for _, name := range slices.Sorted(mapsKeys(entries)) { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_integration_test.go index 7d32c033..44f6b09d 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_integration_test.go @@ -346,7 +346,7 @@ func TestOneCompositionRunPerCandidateAndMachine(t *testing.T) { sup := asks(2) readers(runKeyOf("run-r3", "n1"), 2) readers(runKeyOf("run-r3", "n2"), 2) - pe, _ := json.Marshal(mpr.PipelineEnded{V: 1, Pipeline: "run-r3.a", Module: "a", Branch: trunkBranch, Commit: commit, Outcome: mpr.PipelineEndedOutcomeSuperseded, Run: "r3", Steps: []mpr.StepResult{}}) + pe, _ := json.Marshal(mpr.PipelineEnded{V: 1, Pipeline: "run-r3.a", Branch: trunkBranch, Outcome: mpr.PipelineEndedOutcomeSuperseded, Run: "r3", Movers: []mpr.MoverEnded{{Module: "a", Steps: []mpr.StepResult{}}}}) if _, err := b.js.Publish(b.ctx, "mesh.seat."+mpr.Seat+".event."+mpr.EventPipelineEnded+".n1", pe); err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go index 3f91b28d..ab2c8fd9 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go @@ -3,7 +3,6 @@ package main import ( "context" "encoding/json" - "strings" "sync" "time" @@ -11,7 +10,6 @@ import ( meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-sdk/go/process" - "git.novox.be/novox/mesh-sdk/go/seatdef" ) // The composition step (design 47 G §22, §354, §63): the assigner takes the seat mesh-wants' accepted kind `step`, the @@ -40,8 +38,6 @@ type compositionSteps struct { ended func(id string, run meshwants.StepRun, at time.Time) now func() time.Time logf func(string, ...any) - // asker is the name the step's progress names as its asker, read when it is said (stepAskerOf). - asker func() string // ticks is a ticker of every d, its channel and its stop (time.NewTicker); nil is the clock's. A test hands its own. ticks func(d time.Duration) (<-chan time.Time, func()) mu sync.Mutex @@ -49,32 +45,6 @@ type compositionSteps struct { stops map[string]context.CancelFunc } -// stepAskerOf is the asker a composition step's progress names: the one caller the seat mesh-wants' definition lists for its -// accepted kind step, read from the seat list when it is said, since that subject names no asker (design 47 G A-294, which -// corrects A-267). Empty while the seat list holds no definition naming exactly one. -func stepAskerOf(definition func() (seatdef.Definition, bool)) func() string { - return func() string { - d, ok := definition() - if !ok { - return "" - } - var one string - for _, m := range d.Accepts { - if m.Name != meshwants.AcceptStep || m.Callers == nil { - continue - } - for _, e := range m.Callers.List { - name, isSeat := strings.CutPrefix(e, seatdef.CallerSeatPrefix) - if !isSeat || (one != "" && one != name) { - return "" - } - one = name - } - } - return one - } -} - // progressEvery is how often a running composition step says its progress (design 47 G §496, A-262). const progressEvery = time.Minute @@ -154,7 +124,7 @@ func (c *compositionSteps) say1(ctx context.Context, id string, run meshwants.St stopWait() }() phase := &stepPhase{phase: meshwants.StepProgressEventPhaseFetch} - stop := c.sayProgress(id, phase) + stop := c.sayProgress(id, run.Asker, phase) o := how(context.WithValue(ctx, phaseKey{}, phase), id, run) stop() if ctx.Err() != nil { @@ -222,7 +192,7 @@ func (c *compositionSteps) leave(ids []string) { // sayProgress says the step's progress on the seat's step-progress. once a minute until the returned stop is called (design // 47 G §496, A-262): a ticker inside the running step, as the build seats' step runner says its own beat. -func (c *compositionSteps) sayProgress(id string, phase *stepPhase) (stop func()) { +func (c *compositionSteps) sayProgress(id, asker string, phase *stepPhase) (stop func()) { ticks := c.ticks if ticks == nil { ticks = func(d time.Duration) (<-chan time.Time, func()) { t := time.NewTicker(d); return t.C, t.Stop } @@ -239,7 +209,7 @@ func (c *compositionSteps) sayProgress(id string, phase *stepPhase) (stop func() return case <-tick: at := c.now().UTC() - p := meshwants.StepProgressEvent{Ask: id, Asker: c.asker(), Phase: phase.get(), At: at, Shared: phase.waitsOn()} + p := meshwants.StepProgressEvent{Ask: id, Asker: asker, Phase: phase.get(), At: at, Shared: phase.waitsOn()} if err := c.say(meshwants.EventStepProgress+"."+c.node, p, id+".progress."+at.Format(time.RFC3339Nano)); err != nil { c.logf("saying the progress of the composition step %s: %v", id, err) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_test.go index 07e25cb8..ee2541c1 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_test.go @@ -3,7 +3,6 @@ package main import ( "context" "encoding/json" - "git.novox.be/novox/mesh-sdk/go/seatdef" "slices" "strings" "sync" @@ -84,23 +83,7 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) { stopped := make(chan struct{}) at := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC) release := make(chan struct{}) - // A-294: the asker is the one caller mesh-wants' definition lists for step, read from the seat list; none while unread - // or while it lists more than one. - runner := seatdef.Definition{Seat: meshwants.Seat, Accepts: []seatdef.Message{{Name: meshwants.AcceptStep, - Callers: &seatdef.Callers{List: []string{"seat:mesh-pipeline-runner"}}}}} - two := seatdef.Definition{Seat: meshwants.Seat, Accepts: []seatdef.Message{{Name: meshwants.AcceptStep, - Callers: &seatdef.Callers{List: []string{"seat:mesh-pipeline-runner", "seat:mesh-catalog"}}}}} - for _, d := range []struct { - def seatdef.Definition - read bool - want string - }{{runner, true, "mesh-pipeline-runner"}, {runner, false, ""}, {two, true, ""}} { - if got := stepAskerOf(func() (seatdef.Definition, bool) { return d.def, d.read })(); got != d.want { - t.Errorf("the asker of %+v (read %v) = %q, want %q", d.def.Accepts, d.read, got, d.want) - } - } c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return at }, logf: t.Logf, - asker: stepAskerOf(func() (seatdef.Definition, bool) { return runner, true }), ticks: func(d time.Duration) (<-chan time.Time, func()) { if d != time.Minute { t.Errorf("the progress ticks every %s, want a minute", d) @@ -113,7 +96,8 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) { <-release return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed} }} - body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4.m", Step: meshwants.StepRunStepComposition, Module: "m", Branch: "main"}) + // A11/AC1 (design 47 H §752, H-A-88): with no callers in mesh-wants, the progress names the asker the step run carries. + body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4", Step: meshwants.StepRunStepComposition, Module: "m", Branch: "main", Asker: "mesh-pipeline-runner"}) if !c.take(context.Background(), "ask-1", body) { t.Fatal("the composition step is not taken") } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept.go index aaf3abba..7cfce906 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept.go @@ -6,7 +6,6 @@ import ( "encoding/hex" "encoding/json" "fmt" - "maps" "slices" "strings" "time" @@ -37,8 +36,8 @@ type graphKeeper struct { said string // full says the next revision did not fit the store, so no want is written (graphfill.go); started says the first pass judged the fill. full, started bool - // pins removes a revision that ceased to be current once nothing pins it (pins.go); nil keeps every revision. - pins *pinKeeper + // revisions removes a revision that ceased to be current (revisions.go, H-A-86); nil keeps every revision. + revisions *revisionRemover // computed is the revision computed by the last pass, with its entries: its edges. computed assigngraph.Computed } @@ -51,9 +50,6 @@ func (p *passer) keepGraph(ctx context.Context) { } c := assigngraph.Compute(p.graphInputs()) g.computed = c - if p.ends != nil { - p.ends.setRunner(assigngraph.Runner(slices.Collect(maps.Values(c.Entries)))) // the runner's member of a group goes first (A-334) - } for _, u := range c.Uncomputed { if g.said != u { g.said = u @@ -87,8 +83,8 @@ func (p *passer) keepGraph(ctx context.Context) { return } p.fillingAfter(ctx, used) - if g.pins != nil { - g.pins.judge() // the revision before ceased to be current + if g.revisions != nil { + g.revisions.judge() // the revision before ceased to be current } id := fmt.Sprintf("%s-%s-%d", meshwants.EventAssignmentRevision, rev.Kind, rev.Revision) if err := p.wantsSay(meshwants.EventAssignmentRevision, assigngraph.Event(rev), id); err != nil { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go index 7d003427..4e492075 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go @@ -637,7 +637,7 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) { if err != nil || want.Fp != fNew { t.Fatalf("the canary ace's want = %+v, %v", want, err) } - if w := b.wantNow(t, "shanks", "web", "shanks' first want at what it runs", func(w mwc.Want) bool { return w.Fp != "" }); w.Fp != fOld || w.Composed != "" || !w.Self || w.Newest != fNew { + if w := b.wantNow(t, "shanks", "web", "shanks' first want at what it runs", func(w mwc.Want) bool { return w.Fp != "" }); w.Fp != fOld || !w.Self { t.Errorf("shanks' want = %+v, want the fingerprint it runs, the node composing it, held back from F", w) } // DLV-O5 (ADR 0615 §5, ADR 0619 §2): each want is said on the assigner's views family, its value whole at its revision, @@ -665,7 +665,7 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) { // there by itself: the condition is cleared. healthyF, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) b.putJSON(t, buckets.Have, "ace.web", healthyF) - if w := b.wantNow(t, "g14", "web", "g14's first want at F once the canary runs it healthy", func(w mwc.Want) bool { return w.Fp != "" }); w.Fp != fNew || w.Composed != "" || !w.Self || w.Newest != "" { + if w := b.wantNow(t, "g14", "web", "g14's first want at F once the canary runs it healthy", func(w mwc.Want) bool { return w.Fp != "" }); w.Fp != fNew || !w.Self { t.Errorf("g14's want = %+v, want F, the node composing it", w) } cleared := false @@ -689,7 +689,7 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) { b.putJSON(t, plc.AssignmentsBucket(), k, values[k]) } } - if w := b.wantNow(t, "ace", "web", "ace's want once placed: the build alone, composed by the node", func(w mwc.Want) bool { return w.Self }); w.Composed != "" || w.Fp != fNew { + if w := b.wantNow(t, "ace", "web", "ace's want once placed: the build alone, composed by the node", func(w mwc.Want) bool { return w.Self }); w.Fp != fNew { t.Errorf("ace's want once placed = %+v", w) } nkey, _ := mwc.WantNodeKey("ace") @@ -720,7 +720,7 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) { if err != nil { t.Fatal(err) } - b.putJSON(t, buckets.Provisions, "ace.app.database", raw) + b.putJSON(t, "mesh-provisioner_provisions", "ace.app.database", raw) } // The provisioner keys it from the manifests, as soon as they are known and whether the provider is placed yet or not. asked(meshprovisionerclient.Asked, "the provider's build is not done") @@ -842,8 +842,8 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) { t.Errorf("g14, whose engine answers nothing and which has no record, reads %+v", s) } if w := b.wantNow(t, "ace", "forms", "the want held on ace, naming the form", func(w mwc.Want) bool { - return w.Newest == formsNew && strings.Contains(w.Words, "form:new") - }); w.Fp != formsOld || w.Rule != mwc.RuleComposed { + return strings.Contains(w.Words, "form:new") + }); w.Fp != formsOld { t.Errorf("forms' want = %+v, want it held at what ace runs under rule 7", w) } // Issue 1966 note 170, 1966/AC170-4: ace's engine says its statement, and while the assigner's ask of it is held, @@ -916,32 +916,6 @@ func (s bedState) Await(key string, after uint64, done func([]byte) bool, bound return nil, fmt.Errorf("no answer at %s within %s", key, bound) } -// 1297/B9 (the assigner's half, ADR 0528 clause 6): started whole, the assigner holds a fingerprint whose health check -// failed on a node and writes the ask about it in its own asks, which the view and the teller read; its asks bucket is -// the one its manifest declares (ownBuckets, held equal to the manifest by TestOpensWhatItsManifestNames). -func TestAHeldFingerprintIsAskedAbout(t *testing.T) { - b := startBed(t, "") - fNew := meshcatalogclient.SourcePrefix2 + strings.Repeat("c", 64) - digest := b.manifest(t, "web", `{"module":"web","version":"1"}`) - mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fNew, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) - b.putModule(t, "web", mod) - failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"}) - b.putJSON(t, buckets.Have, "ace.web", failed) - b.place(t, "ace", "web") - b.start(t) - hkey, _ := mwc.HeldKey("web", fNew) - // 1906/H3: the held record names the machine and keeps the health, when and the node's words. - if h, err := mwc.DecodeHeld(b.get(t, buckets.Wants(), hkey)); err != nil || h.Node != "ace" || - !strings.HasPrefix(h.Words, "health failed, phase running, seen at ") || !strings.HasSuffix(h.Words, ": connection refused") { - t.Fatalf("held = %+v, %v", h, err) - } - akey, _ := asks.HoldAskKey("web", fNew) - a, err := decodeAsk(akey, b.get(t, buckets.Asks(), akey)) - if err != nil || a.Asking <= 0 || a.Ask.ID != meshdeliveryclient.AskingID(akey, a.Asking) || len(a.Ask.Options) != 2 { - t.Fatalf("the ask = %+v, %v", a, err) - } -} - // conditionsSaid reads what the assigner said of its conditions, by finding key: each a finding in its own bucket, which // the condition keeper carries (novox/hq ADR 0590 §3, issue 1126), raised while it stands and cleared once deleted, each // with the words its last finding said and the subject the finding names (`[.]`). @@ -972,47 +946,6 @@ func (b *bed) conditionsSaid(t *testing.T) map[string]oc.Condition { return out } -// 1906/D3 (ADR 0539 §19), whole on the bus: a module holding data whose new fingerprint failed its health check on the -// canary is held, not set back, and asked about (restore it, or leave it stopped); the condition names the module, the node -// and the check's words. -func TestAFailedDataModuleIsAskedToRestore(t *testing.T) { - b := startBed(t, "") - fNew := meshcatalogclient.SourcePrefix2 + strings.Repeat("d", 64) - digest := b.manifest(t, "mail", `{"module":"mail","version":"1","resources":[{"id":"data","type":"directory","path":"/var/lib/mail"}],"data":{"own":[{"id":"mail","path":"${dir:data}","class":"irreplaceable","backup":"copy"}]}}`) - mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fNew, Manifest: digest, Repository: "novox/mail", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) - b.putModule(t, "mail", mod) - failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"}) - b.putJSON(t, buckets.Have, "ace.mail", failed) - wantKey, _ := mwc.WantKey("ace", "mail") - want, _ := mwc.EncodeWant(mwc.Want{Fp: fNew, Composed: "sha256:" + strings.Repeat("c", 64)}) - b.putJSON(t, buckets.Wants(), wantKey, want) - b.place(t, "ace", "mail") - b.start(t) - - akey, _ := asks.RestoreAskKey("ace", "mail", fNew) - a, err := decodeAsk(akey, b.get(t, buckets.Asks(), akey)) - if err != nil || len(a.Ask.Options) != 2 || a.Ask.Options[0].ID != "restore" || a.Ask.Options[1].ID != "leave-stopped" { - t.Fatalf("the restore ask = %+v, %v", a, err) - } - if w, err := mwc.DecodeWant(b.get(t, buckets.Wants(), wantKey)); err != nil || w.Fp != fNew { - t.Errorf("the data module was set back: want = %+v, %v", w, err) - } - key := "" - for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && key == ""; time.Sleep(100 * time.Millisecond) { - for k, c := range b.conditionsSaid(t) { - if c.State == occ.StateRaised && strings.HasPrefix(c.Subject.ID, "mail.") && strings.HasSuffix(k, ".held") { - if c.Subject.Machine != "ace" || !strings.Contains(c.Explanation, "connection refused") { - t.Fatalf("the condition = %+v, want it to name ace and the check's words", c) - } - key = k - } - } - } - if key == "" { - t.Fatal("no condition for the held fingerprint was said") - } -} - // 1906/D8 (scenario D of acceptance test 18, ADR 0539 §19), whole on the bus: the assigner is stopped while it writes the wants // of forty modules and another is started; every module's want is written, and each exactly once, by revision: nothing lost // and nothing written again. @@ -1136,43 +1069,6 @@ func (b *bed) hasKey(bucket, key string) bool { return err == nil } -// 1906/D4 (ADR 0539 §19), whole on the bus: the operator's release of a held fingerprint writes its want again, by the rules, -// and clears the condition. -func TestAReleaseAnswerWritesTheWantAgain(t *testing.T) { - b := startBed(t, "") - fNew, fOld := meshcatalogclient.SourcePrefix2+strings.Repeat("b", 64), meshcatalogclient.SourcePrefix2+strings.Repeat("a", 64) - b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`) - b.compose(t, "ace", 'b') - b.compose(t, "ace", 'a') - healthy, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) - b.putJSON(t, buckets.Have, "ace.web", healthy) // it ran the older fingerprint healthy before - failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"}) - b.putJSON(t, buckets.Have, "ace.web", failed) - b.putWant(t, "ace", "web", mwc.Want{Fp: fNew, Composed: "sha256:" + strings.Repeat("b", 64)}) - b.start(t) - w := b.wantNow(t, "ace", "web", "set back to the older fingerprint", func(w mwc.Want) bool { return w.Fp == fOld }) - if w.Newest != fNew { - t.Fatalf("the set-back want = %+v", w) - } - hold, _ := asks.HoldAskKey("web", fNew) - b.get(t, buckets.Asks(), hold) - b.answer(t, hold, "release") - w = b.wantNow(t, "ace", "web", "the held fingerprint's want written again", func(w mwc.Want) bool { return w.Fp == fNew }) - if w.Composed != "" || !w.Self || w.Rule != 0 { - t.Errorf("the want = %+v, want F by the rules, the node composing it", w) - } - for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) { - cleared := false - for k, c := range b.conditionsSaid(t) { - cleared = cleared || (strings.HasPrefix(k, "module.web.") && c.State == occ.StateCleared && strings.HasSuffix(k, ".held")) - } - if cleared { - return - } - } - t.Error("the condition of the held fingerprint was not cleared") -} - // askNow waits for the backup ask at key to be the one numbered asking and answers it. func (b *bed) askNow(t *testing.T, key string, asking int) decodedAsk { t.Helper() @@ -1213,7 +1109,7 @@ func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) { } b.answerTo(t, akey, mwc.BackupAgain, a.Ask.ID) w1 := b.wantNow(t, "ace", "web", "the answer in the want", func(w mwc.Want) bool { return w.Backup == mwc.BackupAgain }) - if w1.Fp != fNew || w1.Composed != "" || !w1.Self { + if w1.Fp != fNew || !w1.Self { t.Errorf("the want = %+v", w1) } // the node retries: its have is backing up, not held. The ask stands and its condition is not cleared (A-453a). @@ -1260,112 +1156,6 @@ func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) { const recorded = `{"module":"web","version":"1","upgrade":{"policy":"record","why":"a person takes each build"}}` -// 1906/H2, whole on the bus: given a fingerprint held after one failed check, when every node that runs the module then runs -// it healthy, the assigner lifts the hold by itself: the held record and its ask are taken away, the held condition is -// cleared saying why, and the want stays at the fingerprint. -func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) { - b := startBed(t, "") - fNew := meshcatalogclient.SourcePrefix2 + strings.Repeat("b", 64) - b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`) - b.compose(t, "ace", 'b') - failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"}) - b.putJSON(t, buckets.Have, "ace.web", failed) - b.putWant(t, "ace", "web", mwc.Want{Fp: fNew, Composed: "sha256:" + strings.Repeat("b", 64)}) - b.start(t) - hkey, _ := mwc.HeldKey("web", fNew) - akey, _ := asks.HoldAskKey("web", fNew) - b.get(t, buckets.Wants(), hkey) - b.get(t, buckets.Asks(), akey) - healthy, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) - b.putJSON(t, buckets.Have, "ace.web", healthy) - for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) { - if b.hasKey(buckets.Wants(), hkey) || b.hasKey(buckets.Asks(), akey) { - continue - } - for k, c := range b.conditionsSaid(t) { - if strings.HasPrefix(k, "module.web.") && c.State == occ.StateCleared && strings.HasSuffix(k, ".held") { - if w := b.wantNow(t, "ace", "web", "the want at the lifted fingerprint", func(w mwc.Want) bool { return w.Fp == fNew }); w.Rule != 0 { - t.Errorf("the want = %+v", w) - } - return - } - } - } - t.Fatalf("the hold was not lifted: held %v, ask %v", b.hasKey(buckets.Wants(), hkey), b.hasKey(buckets.Asks(), akey)) -} - -// 1906/H2, whole on the bus: a hold lifted by itself and written again after the assigner was started again is asked under a -// new id, so the teller raises it; the answer to the first hold does not carry over (1906/H4). -func TestAHoldWrittenAgainAfterALiftAndARestartIsAskedAgain(t *testing.T) { - b := startBed(t, "") - fNew := meshcatalogclient.SourcePrefix2 + strings.Repeat("b", 64) - b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`) - b.compose(t, "ace", 'b') - failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"}) - healthy, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) - b.putJSON(t, buckets.Have, "ace.web", failed) - b.putWant(t, "ace", "web", mwc.Want{Fp: fNew, Composed: "sha256:" + strings.Repeat("b", 64)}) - _, stop := b.startStoppable(t) - hkey, _ := mwc.HeldKey("web", fNew) - akey, _ := asks.HoldAskKey("web", fNew) - first, err := decodeAsk(akey, b.get(t, buckets.Asks(), akey)) - if err != nil { - t.Fatal(err) - } - b.putJSON(t, buckets.Have, "ace.web", healthy) - for deadline := time.Now().Add(15 * time.Second); b.hasKey(buckets.Wants(), hkey); time.Sleep(100 * time.Millisecond) { - if time.Now().After(deadline) { - t.Fatal("the hold was not lifted") - } - } - stop() - time.Sleep(1100 * time.Millisecond) // the next hold is written in another second - b.answerTo(t, akey, "keep", first.Ask.ID) // an answer to the first hold, which the later hold must not take - b.putJSON(t, buckets.Have, "ace.web", failed) - _, stop = b.startStoppable(t) - again, err := decodeAsk(akey, b.get(t, buckets.Asks(), akey)) - if err != nil { - t.Fatal(err) - } - if again.Ask.ID == first.Ask.ID { - t.Fatalf("the hold written again is asked under the first id %s", first.Ask.ID) - } - // Issue 1966 note 155, ADR 0668 §4: that asking lapses at the operator channel while no assigner runs, so the teller's - // ask-ended is said before the next one's consumer binds; at its bind the next reads the operator-channel seat's asks and - // writes the hold again under a new asking. - stop() - nc, err := nats.Connect(b.srv.ClientURL()) - if err != nil { - t.Fatal(err) - } - defer nc.Close() - for _, subject := range []string{"mesh.seat.operator-channel.tool.asks", "mesh.seat.operator-channel.call.asks.*"} { - if _, err := nc.Subscribe(subject, func(m *nats.Msg) { - answer, _ := json.Marshal(map[string]any{"asks": []map[string]string{{"key": "mesh-delivery-teller." + again.Ask.ID, "state": "expired"}}, "matched": 1}) - reply, _ := json.Marshal(map[string]any{"result": json.RawMessage(answer)}) - _ = m.Respond(reply) - }); err != nil { - t.Fatal(err) - } - } - subscribed(t, nc) - b.start(t) - for deadline := time.Now().Add(15 * time.Second); ; time.Sleep(50 * time.Millisecond) { - if third, err := decodeAsk(akey, b.get(t, buckets.Asks(), akey)); err == nil && third.Ask.ID != again.Ask.ID { - break - } - if time.Now().After(deadline) { - t.Fatalf("the hold whose asking %s lapsed before the assigner's bind was not asked again", again.Ask.ID) - } - } - b.putJSON(t, buckets.Have, "ace.web", healthy) - for deadline := time.Now().Add(15 * time.Second); b.hasKey(buckets.Wants(), hkey); time.Sleep(100 * time.Millisecond) { - if time.Now().After(deadline) { - t.Fatal("the stale \"keep\" of the first hold kept the second one held") - } - } -} - // bedStore is the runtime's states over the bed's buckets, as the seat's verbs read and write them: the assigner's own by // their local names, another module's `.`. type bedStore struct{ b *bed } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index a4bbd374..f8c0a2f6 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -77,7 +77,6 @@ func main() { var readBuckets = []string{ buckets.Grants, buckets.Have, - buckets.Provisions, } // readFamilies are the states kept one bucket per node the assigner reads, each named in its manifest's reads: every @@ -90,7 +89,7 @@ func ownBuckets() []string { return []string{buckets.Asks(), buckets.Differences(), buckets.Puts(), buckets.Wants(), plc.AssignmentsBucket(), plc.ProposalsBucket(), plc.CandidatesBucket(), plc.PulsesBucket(), assign.FindingsBucket(), assign.PoliciesBucket(), plc.SeatHoldersBucket(), - engineReadsBucket(), retiredBucket(), copiesBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.Inputs(), buckets.CompositionRuns(), pinsBucket()} + engineReadsBucket(), retiredBucket(), copiesBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.Inputs(), buckets.CompositionRuns()} } // engineReadsState is the assigner's state of what each node's engine states it reads (ADR 0618 §3), one key ``, as @@ -137,7 +136,6 @@ var keptStates = []announce.State{ {Name: buckets.PutWaitsState, Readable: true}, // the put waits said (putwaits.go, design 47 G A-211) {Name: buckets.InputsState, Readable: true}, // read by the tool process alone: the verb input and assigner_inputs (design 47 G §354) {Name: buckets.CompositionRunsState, Readable: true}, // the composition runs, read by the process and its tools (design 47 G §719, A-468c) - {Name: pinsState, Readable: true}, // the records pinning each assignment graph revision (design 47 G §579) } // kvSet is every bucket the assigner opened, by name. @@ -166,12 +164,6 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err) continue } - if name == buckets.Provisions && err != nil { - // The seat mesh-provisioner's state, with its own delivery (ADR 0570 §4): bound once the bus says it was made; - // until then no want waits for a provision, as before. - logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err) - continue - } if name == assign.PoliciesBucket() && err != nil { // The seat mesh-wants' state, new with it (ADR 0565 §7): bound once the bus says it was made; until then no // person's policy is read, so the controller's beside each composition holds, as before. @@ -195,11 +187,6 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any logf("the bucket %s cannot be opened (%v): it is read by the tool process alone", name, err) continue } - if name == pinsBucket() && err != nil { - // New with the pins (design 47 G §579): without it no pin is kept and no revision is removed. - logf("the bucket %s cannot be opened (%v): no pin is kept and no revision removed until it starts again", name, err) - continue - } if name == buckets.Waits() && err != nil { // New with the wait on an outside cause (design 47 G §706): without it no wait is made, and a failing check is judged // as before. @@ -234,8 +221,8 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any // states and those new with a delivery of their own, bound once made, and the two that choose the grants' source. func startsWithout(name string) bool { switch name { - case plc.SeatHoldersBucket(), buckets.Provisions, assign.PoliciesBucket(), assign.FindingsBucket(), - buckets.Puts(), engineReadsBucket(), retiredBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.Inputs(), pinsBucket(): + case plc.SeatHoldersBucket(), assign.PoliciesBucket(), assign.FindingsBucket(), + buckets.Puts(), engineReadsBucket(), retiredBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.Inputs(): return true } return slices.Contains(seatBuckets(), name) || name == buckets.Grants @@ -291,9 +278,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The inputs of the composition steps it asks a build seat to run: the seat mesh-wants' verb input is answered here in chunks // of the core connection, of any size (design 47 G §354, §712, A-237). inputs := inputsOver(kvs[buckets.Inputs()], js, account.Module, logger.Printf) - // The seat list (seatlist.go), followed in follow over the connection and the hub: the verb input's callers are checked - // against the seat's definition it holds (A-186). - gate := seatlist.NewPart("assigner", seatListAgainCallers) + // The seat list (seatlist.go), followed in follow over the connection and the hub. + gate := seatlist.NewPart("assigner") if inputs != nil { inputs.dropAtStart() // the objects of keys another part deleted (A-465) stopInput, err := said.AnswerSeat(meshwants.Seat, seatlist.Definition(gate.Reader, meshwants.Seat), map[string]process.SeatHandler{meshwants.VerbInput: inputs.verb(conn, buscalls.SeatAckWithin)}) @@ -376,7 +362,6 @@ func watched() []string { buckets.Differences(), buckets.Wants(), assign.PoliciesBucket(), - buckets.Provisions, plc.SeatHoldersBucket(), engineReadsBucket(), writtenBucket(), @@ -389,8 +374,6 @@ func watched() []string { // each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends. func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, kvs kvSet, say func(oc.Condition) error, findings assign.Findings, bus caller, iv *catalogViews, inputs *inputKeeper, gate *seatlist.Part, logger *log.Logger, getenv func(string) string) error { w := newWorld() - // The callers' seat entries are resolved from the assigner's own seat-holders as it last read them (A-298). - gate.Holders(seatlist.HoldersIn(func() (map[string][]byte, error) { return w.read(plc.SeatHoldersBucket()), nil })) dirty := make(chan struct{}, 1) wake := func() { select { @@ -572,7 +555,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, } } // And the two that choose where grants are read from, with the same delivery as the grants part that writes one. - for _, name := range []string{buckets.Grants, assign.PoliciesBucket(), buckets.Provisions, plc.SeatHoldersBucket()} { + for _, name := range []string{buckets.Grants, assign.PoliciesBucket(), plc.SeatHoldersBucket()} { if kvs[name] == nil && js != nil { name := name process.AwaitBucket(ctx, js, nc, name, "until then the bucket is not read", @@ -622,7 +605,6 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, logger.Printf("%s is not heard, so a wait on an outside cause is judged again only when its check changes: %v", subject, err) } } - p.outside.readStart() } if js != nil { p.publishWants = wantsPublisher(ctx, js, thisNode) @@ -636,20 +618,10 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, // A held revert's running entry and its want's, asked of the seat mesh-catalog's verb entry (design 47 G §710, A-217). p.entries = meshcatalogclient.NewEntries(iv.h.call) p.entries.Changed = func(meshcatalogclient.EntryRow) { wake() } - // The pins of the revisions (pins.go, design 47 G §579): the gate's pinned, freed, heads-replaced and pins-stated - // on the hub, registered before it starts. - if kv := kvs[pinsBucket()]; kv != nil { - k := &pinKeeper{ctx: ctx, st: kvEnds{ctx, kv}, objects: objects, inUse: inUseOf(w), findings: findings, now: time.Now, logf: logger.Printf} - if err := k.made(); err != nil { - logger.Printf("the state %s is not marked made, so no revision is removed until it is: %v", pinsState, err) - } - if err := hearPins(hub, k); err != nil { - logger.Printf("a pin is not heard, so no revision is removed until the assigner starts again: %v", err) - } else { - p.graph.pins = k - k.watchUnpins(kv) - } - } + // The revisions that are not current are removed when one ceases to be current and once now (revisions.go, + // design 47 H §764, H-A-86). + p.graph.revisions = &revisionRemover{ctx: ctx, objects: objects, inUse: inUseOf(w), logf: logger.Printf} + p.graph.revisions.judge() } } // The node-engine seat's manifest-reads-changed, for one node (ADR 0618 §4): that node is asked again, with no timer. @@ -688,8 +660,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, if bus != nil && js != nil && inputs != nil && p.publishWants != nil && runs != nil { run := &compositionRun{runs: runs, inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept, trunk: trunkOf(iv, w.trunkName), running: w.runningOf, away: func(machine string) bool { return w.awayNow()[machine] }, put: runAskPublisher(js), ask: bus.CallFile, now: time.Now, logf: logger.Printf} - steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, resume: run.resume, ended: runs.stepEnded, now: time.Now, logf: logger.Printf, - asker: stepAskerOf(seatlist.Definition(gate.Reader, meshwants.Seat))} + steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, resume: run.resume, ended: runs.stepEnded, now: time.Now, logf: logger.Printf} move := steps.move(ctx) // the readers a rerun carried (§727, A-470d) runs.moved.Store(&move) // The state's watch wakes the steps waiting on a run and hears a bound's expiry; the runs standing at the start are taken up diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go index 5cfe19f8..0338bcbf 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go @@ -125,7 +125,7 @@ func TestOpensWhatItsManifestNames(t *testing.T) { } kvs[buckets.Puts()].(*watchKV).closed = true kvs[buckets.Wants()].(*watchKV).wrote = wrote - go follow(ctx, nil, nil, kvs, func(oc.Condition) error { return nil }, assign.Findings{}, nil, nil, nil, seatlist.NewPart("assigner", seatListAgainCallers), log.New(io.Discard, "", 0), func(string) string { return "" }) + go follow(ctx, nil, nil, kvs, func(oc.Condition) error { return nil }, assign.Findings{}, nil, nil, nil, seatlist.NewPart("assigner"), log.New(io.Discard, "", 0), func(string) string { return "" }) select { case key := <-wrote: if key != "want.ace.node" { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside.go index ad477697..721f9df3 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside.go @@ -76,9 +76,6 @@ type outsideActs struct { cleared map[string]bool // sigs is each check's last signature, by node.module. sigs map[string]string - // own is the machines whose own network is open; ownRead says the read at the start was answered. - own map[string]bool - ownRead bool // causes is, by condition key, the nodes whose waits stand on it, as the last pass kept them. causes map[string][]string // started says the first pass judged every wait again. @@ -93,7 +90,7 @@ func newOutsideActs(call func(string, any) (json.RawMessage, error), wake func() settle = outside.DefaultSettle } return &outsideActs{call: call, wake: wake, logf: logf, bound: bound, settle: settle, reading: map[string]bool{}, answered: map[string][]outside.Cond{}, - dirty: map[string]bool{}, cleared: map[string]bool{}, sigs: map[string]string{}, own: map[string]bool{}, causes: map[string][]string{}} + dirty: map[string]bool{}, cleared: map[string]bool{}, sigs: map[string]string{}, causes: map[string][]string{}} } // outsideSettings are the bounds of the wait, the settings outside-wait-minutes and outside-settle-seconds (design 47 G §633, §706), @@ -135,31 +132,8 @@ func settingFile(path string) (int, error) { return n, nil } -// readStart reads the open conditions of every machine once at the start, for the machines whose own network is held. -func (o *outsideActs) readStart() { - go func() { - answer, err := o.call(conditionsVerb, map[string]any{"scope": "machine"}) - if err != nil { - o.logf("the open conditions are not read at the start (%v): no machine is held for its own network until its event is heard", err) - return - } - conds, err := outside.ParseConditions(answer) - if err != nil { - o.logf("%v", err) - return - } - o.mu.Lock() - for m := range outside.OwnNetwork(conds) { - o.own[m] = true - } - o.ownRead = true - o.mu.Unlock() - o.wake() - }() -} - -// Heard is one of the seat's events condition-raised, -changed or -cleared: it says which machines to judge again and keeps the -// machines whose own network is open; nothing is kept of the condition. +// Heard is one of the seat's events condition-raised, -changed or -cleared: it says which machines to judge again; nothing is kept +// of the condition (the network hold ends, design 47 H §750, H-A-55). func (o *outsideActs) Heard(subject string, body []byte) { var e mcseat.ConditionEvent if err := json.Unmarshal(body, &e); err != nil || e.Key == "" { @@ -168,9 +142,6 @@ func (o *outsideActs) Heard(subject string, body []byte) { } cleared := strings.HasSuffix(subject, "."+mcseat.EventConditionCleared) o.mu.Lock() - if p := strings.Split(e.Key, "."); len(p) == 3 && p[0] == "machine" && p[2] == "network" { - o.own[p[1]] = !cleared - } if cleared { for _, node := range o.causes[e.Key] { o.dirty[node] = true @@ -186,16 +157,6 @@ func (o *outsideActs) Heard(subject string, body []byte) { o.wake() } -// Held says whether machine's own network is open: no want moves to it. -func (o *outsideActs) Held(machine string) bool { - if o == nil { - return false - } - o.mu.Lock() - defer o.mu.Unlock() - return o.own[machine] -} - // take is the conditions read for machine, consumed (one answer is one judging), or false while the kept call is unanswered, // which it starts when none is in flight. func (o *outsideActs) take(machine string) ([]outside.Cond, bool) { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside_test.go index 4bedb69a..7c8c383e 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside_test.go @@ -237,23 +237,6 @@ func TestTheWaitOnAnOutsideCause(t *testing.T) { }) } -func TestAMachineWhoseOwnNetworkIsOpenGetsNoWant(t *testing.T) { - m := assign.Module{Name: "nats", FP: fpTest, Built: true, Nodes: []string{"ace"}, MachineHeld: func(node string) bool { return node == "ace" }} - ok, rule, why := assign.Holds(m, "ace", assign.Nodes{Online: map[string]bool{"ace": true}}) - if ok || rule == 0 || !strings.Contains(why, "machine.ace.network") { - t.Errorf("holds = %v %d %q", ok, rule, why) - } - o := newOutsideActs(nil, func() {}, func(string, ...any) {}, 0, 0) - o.Heard("mesh.seat.mesh-conditions.event.condition-raised", []byte(`{"key":"machine.ace.network","subject":{"id":"ace","scope":"machine","machine":"ace"}}`)) - if !o.Held("ace") || o.Held("g14") { - t.Error("the own network condition's event does not hold the machine") - } - o.Heard("mesh.seat.mesh-conditions.event.condition-cleared", []byte(`{"key":"machine.ace.network","subject":{"id":"ace","scope":"machine","machine":"ace"}}`)) - if o.Held("ace") { - t.Error("the machine is held after its condition cleared") - } -} - func TestTheToolsReadAndReleaseAnOutsideWait(t *testing.T) { r := newOutsideRig(t) r.cond("machine.hub.unreachable", "ace") diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index 04cb41d1..8a868938 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -5,7 +5,6 @@ import ( "context" "encoding/json" "errors" - "fmt" "maps" "mesh-delivery-assigner/internal/buckets" "slices" @@ -18,16 +17,10 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate" - "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay" - "git.novox.be/novox/mesh-sdk/go/motion" "git.novox.be/novox/mesh-sdk/go/seatlist" "sync" - "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" - "git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient" - mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph" "git.novox.be/novox/mesh-sdk/go/catalogue" @@ -64,19 +57,12 @@ type passer struct { findings assign.Findings // calls is the wait of the kept manifest-reads calls (enginereads.ReadsCalls, ADR 0668 §3), made at the first pass. calls *enginereads.ReadsCalls - // failedProvisions are the failed provisions whose finding this process wrote (ADR 0570 §4). - failedProvisions map[string]bool - // missing are the required provisions with no key this process pulsed, and missingSaid those whose finding it wrote. - missing, missingSaid map[string]bool // unplaced are the nodes whose unplaced finding this process wrote. unplaced map[string]bool // answeredAt is, by the id of a backup ask, how many values the have had when its answer was written into the want. answeredAt map[string]uint64 // now is the clock; nil is time.Now. now func() time.Time - // neither is, by node and module, each module on neither list whose condition this process raised (issue 1945). Nil until the - // first pass after a start, which clears what a previous process may have raised (startClears). - neither map[string]neitherOne // ends are the pipeline ends the wants follow (pipelines.go), once the first registrations are heard; until then, and nil, the trunk-only want. ends *pipelineEnds // acts are the puts' acts (ADR 0546); nil: the puts are not taken further (no bus to call, or no puts bucket). @@ -254,14 +240,13 @@ func (p *passer) nodeWants(ctx context.Context) { } } -// wants writes, for every module the trunk holds a fingerprint of, each node's want by the six rules (ADR 0429 §6), and -// the hold of a fingerprint whose health check failed. The assigner compares its wants with nothing the controller -// sends: the differences bucket is kept read-only with its history (ADR 0539 §8). +// wants writes, for every module assigned or wanted, each node's want naming the newest version released on the branch its +// placement names whose pipeline passed, written tier by tier (design 47 H §750, §751). The want is held for nothing else: the +// rules of ADR 0429 §6, the network hold and A-428's exceptions end (H-A-55); a put owns its node's want, and the operator's yes +// on a module of policy record is the node-engine's have and the asker's (yeses). func (p *passer) wants(ctx context.Context) { assigned := p.w.assigned() assignedBranches := p.w.placedBranches() - // Every module of every branch's catalog half, for the parts that ask which exist; a node's entries come from the half of the - // branch its placement names (design 47 G §14, §184). trunk := p.w.trunk() views := map[string]*branchView{} viewOf := func(branch string) *branchView { @@ -282,9 +267,6 @@ func (p *passer) wants(ctx context.Context) { } wanted := p.w.wantsOf() chosen, _ := assign.DecodePolicies(p.w.read(assign.PoliciesBucket())) - provisionsOf := p.provisionsOf() - missing := map[string]bool{} - defer func() { p.provisionsMissing(ctx, missing) }() defer p.unplacedFindings() defer func() { p.seatHolders(ctx, viewOf(p.w.trunkName()).manifests) }() p.clearStaleCalls() @@ -298,397 +280,32 @@ func (p *passer) wants(ctx context.Context) { for name := range wanted { names[name] = true } - neither := map[string]neitherOne{} - // The first pass after a start remembers nothing a previous process said: a neither or held condition it raised would stay - // raised for ever once the module has its want or is no longer held. That pass clears, once, the neither condition of every - // assigned module not on neither list and the held condition of every module not held; a clear of a condition that is not open is - // quiet at the router. - first := p.neither == nil - stale := map[string]neitherOne{} - left := map[string]string{} - defer func() { p.sayNeither(neither, stale, left) }() - // Where grants are read from, once for the pass: the grants part's (issue 774, ADR 0571 clause 9). - grants := p.w.grantsBucket() for name := range names { groups := p.groups(name, nodesOf[name], assignedBranches[name]) for gi, g := range groups { - func() { - bv := viewOf(g.branch) - trunk, manifests, nodes, moduleGraph, served, offering := bv.trunk, bv.manifests, bv.nodes, bv.graph, bv.served, bv.offering - fp := trunk[name].Fp2 - name := name - nodesOf := map[string][]string{name: g.nodes} - wantedOf := map[string]map[string]mwc.Want{name: wantedHere(wanted[name], g.nodes, groups, gi == 0)} - // A want follows a passed pipeline: a module whose fingerprint no pipeline passed at on the branch its placements - // chose, and whose wants do not name it already, gets none (design 47 G §29, §35). - var branches []string - if p.ends.following() { - branches = []string{g.branch} - } - if fp != "" && !p.ends.mayWant(name, fp, branches, wantedOf[name]) { - if !runnerFirstStart(name, manifests, p.w.runningNow()) && !manifests[name].ClaimsSeat(meshcatalog.Seat) { - none := assign.Outcome{Rules: noPipelineRules(g.nodes, name, fp, branches)} - p.sayNoWant(name, g.nodes, wantedOf[name], none) - p.sayWhy(ctx, name, g.nodes, wantWhy{Branch: g.branch, Fp: fp, Step: whyNoPipeline}, wantedOf[name], &none, nil) - p.nameHeldBuilds(ctx, name, fp, g.nodes, nodes, wantedOf[name], p.namedBuild(name, fp, trunk[name].FpSrc)) - return - } - // The first start of the pipeline runner (design 47 G A-428): no runner runs to pass its pipeline, so its want is - // written from the trunk's entry by the other rules, and said once. So is the holder of mesh-catalog's: every - // pipeline runs on a revision it took in, and a revision naming a newer one is taken in only by that one (§16, - // G-5's whole test of indexer and assigner: the new indexer is wanted and granted, starts, and takes it in). - if key := "no-pipeline-start " + name + "@" + fp; !p.skipped[key] { - if p.skipped == nil { - p.skipped = map[string]bool{} - } - p.skipped[key] = true - if manifests[name].ClaimsSeat(meshcatalog.Seat) { - p.logf("%s at %s is wanted with no passed pipeline: it claims the seat %s, every pipeline runs on a revision it took in, and a revision naming it is taken in only by it (design 47 G §16)", name, shortHex(fp), meshcatalog.Seat) - } else { - p.logf("%s at %s is wanted with no passed pipeline: it claims the seat %s and no module holding that seat runs on any node, so none can pass one (design 47 G A-428)", name, shortHex(fp), mpr.Seat) - } - } - } - m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Needs: trunk[name].Needs, Chosen: chosenOf(chosen, name), Provisions: provisionsOf(name), Answering: answering(manifests, nodes), Offered: offered(manifests), Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name], FormNeeds: p.w.buildNeeds(name, fp), - Self: func(node string) bool { return placed[node] }, - Composed: func(node, fp string) (assign.ComposedDigest, bool) { - return p.w.composed(node, name, fp) // a placed node's own: the digest its own have names (ADR 0571 §6) - }, - GrantedAt: func(fp string) bool { - key, err := grantsstate.PartGrantKey(name, fp) - return err == nil && p.w.has(grants, key) - }, - Restored: func(node string) bool { - key, err := asks.RestoreAskKey(node, name, fp) - return err == nil && p.answeredWith(key, assign.OptionRestore) - }, - Allowed: func(node, digest string) bool { - key, err := asks.RecordAskKey(node, name, digest) - return err == nil && p.answeredWith(key, assign.OptionYes) - }} - if fp != "" { - held, _ := mwc.HeldKey(name, fp) - grant, _ := grantsstate.PartGrantKey(name, fp) - hold, _ := asks.HoldAskKey(name, fp) - noPipeline := runnerFirstStart(name, manifests, p.w.runningNow()) || manifests[name].ClaimsSeat(meshcatalog.Seat) - m.KeptBuilds = p.namedBuild(name, m.FP, trunk[name].FpSrc, noPipeline) - m.Released = p.answeredWith(hold, "release") - m.Kept = p.answeredWith(hold, "keep") - m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(buckets.Wants(), held) && !m.Released, p.w.has(grants, grant) - m.BuildFailed, m.BuildError = p.w.buildStanding(name, fp) - // A build whose version is released counts as done for rule 1; its build.ended counts only for a build not - // released (design 47 G A-429), so a restart that lost the end loses nothing of a released build. - if releasedBuild(m.KeptBuilds, fp) { - m.Built, m.BuildFailed, m.BuildError = true, false, "" - } - if raw := p.w.value(grants, grant); raw != nil { - if g, err := grantsstate.DecodeGrant(raw); err == nil { - m.GrantedOn = func(node string) bool { return assign.GrantNamesNode(g, node, name) } - } - } - if m.Released { - p.clearHeld(name, fp) - } - } - if served != nil { - m.Invokes, m.Served = keptInvokes(moduleGraph, name), served - } - m.Offering = offering - last := map[string]string{} - for _, node := range m.Nodes { - last[node] = p.w.lastHealthy(node, name, fp) - } - if p.outside != nil { - m.MachineHeld = p.outside.Held - nodes = p.outsideReview(ctx, name, fp, m.Nodes, nodes, manifests[name].ClaimsSeat(mcseat.Seat), p.movedTo(wanted, nodes)) - } - out := assign.Wants(m, nodes, wantedOf[name], last) - p.sayWhy(ctx, name, m.Nodes, wantWhy{Branch: g.branch, Fp: fp, Step: whyRules}, wantedOf[name], &out, &m) - for _, node := range m.Nodes { - for _, prov := range m.MissingProvisions(node) { - missing[node+"."+name+"."+prov] = true - } - } - p.yeses(ctx, name, m, wantedOf[name], &out) - gatherNeither(neither, name, out) - leftNeither(left, m, wantedOf[name], out) - p.sayNoWant(name, m.Nodes, wantedOf[name], out) - p.sayUnreached(name, m.Nodes, out) - p.sayNodeFailed(name, fp, m.Nodes, nodes, out, first) - if first { - startClears(stale, m, out) - if fp != "" && !m.Held && out.Held == nil && !out.Lifted { - p.clearHeld(name, fp) - } - } - if out.Held != nil { - key, err := mwc.HeldKey(name, fp) - if err == nil { - held := *out.Held - p.put(ctx, buckets.Wants(), key, func() ([]byte, error) { return mwc.EncodeHeld(held) }) - } - } - if out.Lifted { - p.lift(ctx, name, fp) - } - if out.Held != nil || (m.Held && !out.Lifted) { - p.holdAsk(ctx, name, fp, out.Held) - p.heldCondition(ctx, name, fp, out.Held, len(out.Restore) > 0) - } - for _, node := range out.Restore { - p.restoreAsk(ctx, name, node, fp) - } - for _, need := range out.Record { - p.recordAsk(ctx, name, fp, need) - } - for node, want := range out.Wants { - p.putWant(ctx, node, name, want) - } - if len(out.Wants) > 0 { - p.ends.wrote(name, fp, branches) - } - }() - } - } -} - -// movedTo says whether the delivery moved owner to node: owner's want there names a fingerprint its have there does not yet run -// healthy (design 47 G §706, *What counts as an outside cause*: a rewritten file of a module the delivery moved is that module's). -func (p *passer) movedTo(wanted map[string]map[string]mwc.Want, n assign.Nodes) func(owner, node string) bool { - return func(owner, node string) bool { - w, ok := wanted[owner][node] - return ok && !w.Removed && !n.RunsHealthy(node, owner, w.Fp) - } -} - -// sayNoWant says in the journal why a node of module gets no want after this pass, from the rule the outcome names for it -// (issue 1945: a new assignment whose trunk build was not done was held back with nothing said), once per reason, and says -// when the node has a want again. A node with a live want, written before and kept or written in this pass, has one. -func (p *passer) sayNoWant(module string, nodes []string, before map[string]mwc.Want, out assign.Outcome) { - if p.noWant == nil { - p.noWant = map[string]string{} - } - assigned := map[string]bool{} - for _, node := range nodes { - assigned[node] = true - key := node + "." + module - w, written := out.Wants[node] - if !written { - w, written = before[node] - } - reason := "" - if h, held := out.Rules[node]; held && (!written || w.Removed) { - reason = h.Reason() - } - last, said := p.noWant[key] - switch { - case reason != "" && reason != last: - p.logf("%s on %s gets no want: %s", module, node, reason) - p.noWant[key] = reason - case reason == "" && said && written && !w.Removed: - p.logf("%s on %s has a want", module, node) - delete(p.noWant, key) - } - } - for key := range p.noWant { - if node, m, _ := strings.Cut(key, "."); m == module && !assigned[node] { - delete(p.noWant, key) // no longer assigned there: its removed want says so - } - } -} - -// sayUnreached says in the journal, once per words, that a module is wanted on a node and may not be reached there, since -// no provider of what it requires runs healthy (rule 5, which holds no want: issue 1960), and once when it is reached again. -func (p *passer) sayUnreached(module string, nodes []string, out assign.Outcome) { - if p.unreached == nil { - p.unreached = map[string]string{} - } - for _, node := range nodes { - key := node + "." + module - why, last := out.Unreached[node], p.unreached[key] - switch { - case why != "" && why != last: - p.logf("%s on %s is wanted and may not be reached: %s", module, node, why) - p.unreached[key] = why - case why == "" && last != "": - p.logf("%s on %s is reached: its providers run healthy", module, node) - delete(p.unreached, key) - } - } -} - -// sayNodeFailed raises, once per node and words, the condition of a failed health check of a module's newest fingerprint fp -// that is the node's own, since fp runs healthy elsewhere and is not held (ADR 0557 §1, issue 1496), and says it in the journal; -// it clears the condition once when the node no longer fails alone, or is no longer one of the module's nodes. The first pass -// after a start remembers nothing a previous process raised: it clears, for every module, the condition of every node the -// presence bucket knows that does not fail alone, since a node that recovered while no process watched leaves no failed have to -// find it by (a clear of a condition not open is quiet at the router). A condition the router did not take is said again at the next pass. -func (p *passer) sayNodeFailed(module, fp string, nodes []string, known assign.Nodes, out assign.Outcome, first bool) { - if p.nodeFailed == nil { - p.nodeFailed = map[string]string{} - } - look := map[string]bool{} - for _, node := range nodes { - look[node] = true - } - for key := range p.nodeFailed { - if node, m, _ := strings.Cut(key, "."); m == module { - look[node] = true - } - } - if first { - for node := range known.Online { - look[node] = true - } - } - for _, node := range slices.Sorted(maps.Keys(look)) { - key := node + "." + module - f, failing := out.NodeFailed[node] - last, said := p.nodeFailed[key] - switch { - case failing && f.Words+"|"+strings.Join(f.Others, ",") != last: - p.logf("%s on %s: the health check at %s failed here (%s), while it runs healthy on %s: the failure is %s's, so nothing "+ - "is held or set back", module, node, fp, f.Words, strings.Join(f.Others, ", "), node) - if p.say != nil { - if err := p.say(assign.NodeFailedCondition(node, module, fp, f, true)); err != nil { - p.logf("the condition of %s failing on %s alone could not be said, and is said at the next pass: %v", module, node, err) - continue - } + bv := viewOf(g.branch) + wantedOf := wantedHere(wanted[name], g.nodes, groups, gi == 0) + target, ok := p.ends.target(name, g.branch) + m := assign.Module{Name: name, Manifest: bv.manifests[name], Chosen: chosenOf(chosen, name), Nodes: g.nodes, Puts: puts[name], + Self: func(node string) bool { return placed[node] }, + Target: func(string) (assign.Target, bool) { return target, ok }, + Allowed: func(node, digest string) bool { + key, err := asks.RecordAskKey(node, name, digest) + return err == nil && p.answeredWith(key, assign.OptionYes) + }} + if p.outside != nil && ok { + p.outsideReview(ctx, name, target.Fp, m.Nodes, bv.nodes, bv.manifests[name].ClaimsSeat(mcseat.Seat), p.movedTo(wanted, bv.nodes)) } - p.nodeFailed[key] = f.Words + "|" + strings.Join(f.Others, ",") - case !failing && (said || first): - if said { - p.logf("%s on %s no longer fails its health check alone", module, node) + out := assign.Wants(m, wantedOf) + p.sayWhy(ctx, name, m.Nodes, wantWhy{Branch: g.branch, Fp: target.Fp, Version: target.Version, Pipeline: target.Pipeline, Step: whyStep(ok)}, wantedOf, &out) + if ok { + p.yeses(ctx, name, m, wantedOf, &out) } - if p.say != nil { - if err := p.say(assign.NodeFailedCondition(node, module, fp, assign.NodeFailure{}, false)); err != nil && said { - p.logf("the condition of %s failing on %s alone could not be cleared, and is cleared at the next pass: %v", module, node, err) - continue - } - } - delete(p.nodeFailed, key) - } - } -} - -// neitherOne is a module assigned to a node that holds no want for it after a pass, and the rule holding its first want back. -type neitherOne struct { - node, module string - holding assign.Holding -} - -// gatherNeither adds the nodes on which module is on neither list in its outcome, keyed by node and module. -func gatherNeither(into map[string]neitherOne, module string, out assign.Outcome) { - for node, h := range out.Neither { - into[node+"."+module] = neitherOne{node: node, module: module, holding: h} - } -} - -// leftNeither says, by node and module, for every node of m not on neither list after the pass, why: "" when the node has a live -// want for it (written before and kept, or written in this pass), else why it has none. sayNeither clears a neither condition in -// these words, so a clear never says a want the node does not hold (issue 1945: "nats on novox has a want again" while the bus bar -// left it with none). -func leftNeither(into map[string]string, m assign.Module, before map[string]mwc.Want, out assign.Outcome) { - for _, node := range m.Nodes { - if _, on := out.Neither[node]; on { - continue - } - w, written := out.Wants[node] - if !written { - w, written = before[node] - } - why := "" - h, ruled := out.Rules[node] - switch { - case written && !w.Removed: - case (m.Held && !out.Lifted) || out.Held != nil: - why = "it is held, and its held condition says so" - case slices.ContainsFunc(out.Record, func(r assign.RecordNeed) bool { return r.Node == node }): - why = "its move waits for the operator's yes on its record ask" - case putOn(m, node): - why = "its put waits on its own steps and the operator's answer" - case m.FP == "": - why = "the trunk has no build of it" - case ruled: - why = h.Reason() - default: - why = "the node does not run it" - } - into[node+"."+m.Name] = why - } -} - -// putOn says whether an operator's put on node owns m's want there: one still on its steps (ADR 0546), so the rules write none -// meanwhile. An ended put (done, refused, released) stays in the bucket and owns nothing. -func putOn(m assign.Module, node string) bool { - p, put := m.Puts[node] - _, owns := assign.PutOwns(p) - return put && owns -} - -// startClears adds, for the first pass after a start, every node of m that is not on neither list after the pass: a neither -// condition a previous process raised for it is cleared. That covers a node with a want and one with none that is not on neither -// list, such as a held module (issue 1945: the hold is said on its own, so a neither raised for it before the restart would stay up -// for ever) or a move waiting for its record ask. Clearing a condition that is not open is quiet at the router. -func startClears(into map[string]neitherOne, m assign.Module, out assign.Outcome) { - for _, node := range m.Nodes { - if _, on := out.Neither[node]; !on { - into[node+"."+m.Name] = neitherOne{node: node, module: m.Name} - } - } -} - -// sayNeither raises, once, the condition of every module on neither list in this pass and writes it to the journal, and clears, -// once, the condition of each one said before that has a want again (issue 1945). A say that fails is made again at the next pass. -// stale is, on the first pass after a start, the modules whose condition a previous process may have raised and that have a want: -// each is cleared once. -// left is, by node and module, why each assigned node not on neither list after this pass has no want ("" when it has one); a key -// it does not hold is no longer assigned. -func (p *passer) sayNeither(now, stale map[string]neitherOne, left map[string]string) { - if p.neither == nil { - p.neither = map[string]neitherOne{} - for key, one := range stale { - if _, on := now[key]; !on { - p.neither[key] = one + for node, want := range out.Wants { + p.putWant(ctx, node, name, want) } } } - for key, one := range now { - if _, said := p.neither[key]; said { - continue - } - c := assign.NeitherCondition(one.node, one.module, one.holding, true) - if p.say != nil { - if err := p.say(c); err != nil { - p.logf("the condition of %s on neither list on %s could not be said, and is said at the next pass: %v", one.module, one.node, err) - continue - } - } - p.logf("%s is assigned to %s and on neither list: no want, and not in the node's own composition; what holds its first want back: %s", - one.module, one.node, one.holding.Reason()) - p.neither[key] = one - } - for key, one := range p.neither { - if _, still := now[key]; still { - continue - } - why, assigned := left[key] - if !assigned { - why = "it is no longer assigned there" - } - if p.say != nil { - if err := p.say(assign.NeitherLeftCondition(one.node, one.module, why)); err != nil { - p.logf("the condition of %s on neither list on %s could not be cleared, and is cleared at the next pass: %v", one.module, one.node, err) - continue - } - } - if why == "" { - p.logf("%s on %s has a want again", one.module, one.node) - } else { - p.logf("%s on %s is no longer on neither list and has no want: %s", one.module, one.node, why) - } - delete(p.neither, key) - } } // put writes one key of the assigner's own bucket; a write that fails is said and made again at the next pass. @@ -736,59 +353,6 @@ func (p *passer) putWant(ctx context.Context, node, module string, want mwc.Want } } -// holdAsk writes, once, the ask about a held fingerprint (ADR 0528 clause 6, ADR 0437 clause 10): the first ask that needs -// no proof of who answered, which the view may answer and the teller fills. In release 5 an answer changes nothing on a -// node (wants are only compared, ADR 0429 §7), and the options say so. -func (p *passer) holdAsk(ctx context.Context, module, fp string, fresh *mwc.Held) { - key, err := asks.HoldAskKey(module, fp) - if err != nil || p.w.has(buckets.Asks(), key) { - return - } - // The asking is the hold's time in seconds: a hold of the same fingerprint written again after a lift is another asking, under - // an id of its own, also after a restart, since the router drops a second ask under an id it keeps (issue 1906). - now := p.clock() - asking := int(now.Unix()) - h := p.heldHold(module, fp, fresh) - a := assign.HoldAsk(key, module, fp, h.Node, h.Words, asking, now) - p.put(ctx, buckets.Asks(), key, func() ([]byte, error) { return assign.EncodeAsk(key, a) }) -} - -// lift takes away a hold the assigner lifted by itself, since every node that runs the module runs the held fingerprint healthy -// (issue 1906): the held record and its ask (the teller then takes the ask back from the operator), and the held condition is -// cleared saying why, once. A hold of the same fingerprint written later is said and asked again. -func (p *passer) lift(ctx context.Context, module, fp string) { - held, err := mwc.HeldKey(module, fp) - if err != nil { - return - } - ask, err := asks.HoldAskKey(module, fp) - if err != nil { - return - } - c := assign.LiftedCondition(module, fp) - if !p.said[c.Key+".lifted"] { - if p.say != nil { - if err := p.say(c); err != nil { - // The hold stays written, so the next pass lifts it again and says so then. - p.logf("the condition of the lifted hold of %s could not be cleared, and is cleared at the next pass: %v", module, err) - return - } - } - p.logf("%s", assign.LiftedWords(module, fp)) - p.said[c.Key+".lifted"] = true - delete(p.said, c.Key+".raised") // a hold written again is said again - } - p.remove(ctx, buckets.Wants(), held) - p.remove(ctx, buckets.Asks(), ask) - // The restore asks of that fingerprint, on every node, go with the hold: every node runs it healthy, so no node waits to be - // set back from it (issue 1906, note 28). - for key := range p.w.read(buckets.Asks()) { - if k, err := asks.ParseAnyAskKey(key); err == nil && k.Kind == asks.AskKindRestore && k.Module == module && k.FP == fp { - p.remove(ctx, buckets.Asks(), key) - } - } -} - // staleAsk says why the hold, restore or record ask at key (read as the SDK reads it) is about something the trunk has left // behind, or "" when it is not: a module no longer on the trunk, a fingerprint that is no longer the module's trunk fingerprint, // or, for a record ask, a composition the module's node no longer waits on. A backup ask, and a revert ask (design 47 G §710), is stale when the module's want on its @@ -894,29 +458,6 @@ func (p *passer) yesWritten(k asks.AskKey) bool { return err == nil && w.Yes == k.Composed } -// fingerprintsOf are the fingerprints a node may still want of module: the trunk's, the one its want names and the one its have -// names, each once. -func (p *passer) fingerprintsOf(node, module, trunkFP string) []string { - var out []string - add := func(fp string) { - if fp != "" && !slices.Contains(out, fp) { - out = append(out, fp) - } - } - add(trunkFP) - if key, err := mwc.WantKey(node, module); err == nil { - if w, err := mwc.DecodeWant(p.w.value(buckets.Wants(), key)); err == nil { - add(w.Fp) - } - } - if key, err := nodeengineclient.HaveKey(node, module); err == nil { - if h, err := nodeengineclient.DecodeHave(p.w.value(buckets.Have, key)); err == nil { - add(h.Fp) - } - } - return out -} - // staleAsks takes away, at every pass, each hold, restore, record and backup ask that staleAsk names (issue 1906, note 28): an ask // outlived its fingerprint, and the teller takes its router ask back from the operator when the key goes. With the trunk not // read at all (no module), nothing is taken away. @@ -929,10 +470,6 @@ func (p *passer) staleAsks(ctx context.Context, trunk map[string]meshcatalog.Mod // The condition is cleared first and the ask goes only once that was said: the ask is what brings this key back at the // next pass, so a clear that failed once is made again, with no state of its own (issue 1906, note 29). k, kerr := asks.ParseAnyAskKey(key) - held := kerr == nil && (k.Kind == asks.AskKindHold || k.Kind == asks.AskKindRestore) - if held && !p.clearHeld(k.Module, k.FP) { - continue - } // A backup ask's condition is cleared with it, raised again if a move to that fingerprint is backed up again. if kerr == nil && k.Kind == asks.AskKindBackup && p.say != nil { if err := p.say(assign.BackupCondition(k.Node, k.Module, k.FP, "", false)); err != nil { @@ -947,12 +484,6 @@ func (p *passer) staleAsks(ctx context.Context, trunk map[string]meshcatalog.Mod } p.logf("the ask %s is about a fingerprint or composition the trunk has left behind and is taken away: %s", key, why) p.remove(ctx, buckets.Asks(), key) - // The held record stays: it is the memory that this fingerprint failed (ADR 0408 §4), and a revert returns the trunk to - // exactly it. Only the condition, which would else ask for an answer that can no longer be given, is cleared, and it is - // raised again if the trunk returns to the fingerprint. - if held { - delete(p.said, assign.HeldCondition(k.Module, k.FP, "", "", false, false).Key+".raised") - } } } } @@ -970,48 +501,6 @@ func (p *passer) remove(ctx context.Context, bucket, key string) { p.w.put(bucket, key, nil) } -// heldHold is the hold of fp as written: the node whose health check failed and its words. -func (p *passer) heldHold(module, fp string, fresh *mwc.Held) mwc.Held { - if fresh != nil { - return *fresh - } - key, err := mwc.HeldKey(module, fp) - if err != nil { - return mwc.Held{} - } - h, _ := mwc.DecodeHeld(p.w.read(buckets.Wants())[key]) - return h -} - -// heldCondition raises, once, the condition for a fingerprint the assigner holds (ADR 0429 §6, ADR 0539 §19): the module, the -// node whose health check failed and the check's words. A module holding data needs the operator's answer to its restore ask. -func (p *passer) heldCondition(ctx context.Context, module, fp string, fresh *mwc.Held, data bool) { - h := p.heldHold(module, fp, fresh) - c := assign.HeldCondition(module, fp, h.Node, h.Words, true, data) - if p.say == nil || p.said[c.Key+".raised"] { - return - } - if err := p.say(c); err != nil { - p.logf("the condition of the held %s could not be said, and is said at the next pass: %v", module, err) - return - } - p.said[c.Key+".raised"] = true - delete(p.said, c.Key+".lifted") // a hold raised again and lifted again is said lifted again - delete(p.said, c.Key+".cleared") // and released again is cleared again -} - -// restoreAsk writes, once, the ask about a module holding data whose held fingerprint failed its health check on node. -func (p *passer) restoreAsk(ctx context.Context, module, node, fp string) { - key, err := asks.RestoreAskKey(node, module, fp) - if err != nil || p.w.has(buckets.Asks(), key) { - return - } - words := p.heldHold(module, fp, nil).Words - now := p.clock() - a := assign.RestoreAsk(key, node, module, fp, words, assign.AskingSeconds(now), now) - p.put(ctx, buckets.Asks(), key, func() ([]byte, error) { return assign.EncodeAsk(key, a) }) -} - // answeredWith says whether the teller wrote an answer to the ask at key as it stands now and the option chosen is option. The // answer is the operator's, given in the view or on the phone, and stays under its key (ADR 0539 §14, §16, §19); it counts only // for the asking it answered (its warrant names that asking's id), so an answer to an ask taken away, or to an earlier hold of @@ -1029,21 +518,6 @@ func (p *passer) answeredWith(key, option string) bool { return id == ask.ID } -// clearHeld clears, once, the condition of a hold the operator released. -// It says whether nothing is left to say: the clear was said now or before, or there is no router to say it to. -func (p *passer) clearHeld(module, fp string) bool { - c := assign.HeldCondition(module, fp, "", "", false, false) - if p.say == nil || p.said[c.Key+".cleared"] { - return true - } - if err := p.say(c); err != nil { - p.logf("the condition of the released %s could not be cleared, and is cleared at the next pass: %v", module, err) - return false - } - p.said[c.Key+".cleared"] = true - return true -} - // recordAsk writes, once, the ask about a move of a module of policy record that waits for the operator's yes. func (p *passer) recordAsk(ctx context.Context, module, fp string, need assign.RecordNeed) { key, err := asks.RecordAskKey(need.Node, module, need.Digest) @@ -1091,7 +565,7 @@ func (p *passer) backups(ctx context.Context, reverting map[string]bool) { continue } current, currentAsk, there := assign.DecodeAsk(akey, p.w.read(buckets.Asks())[akey]) - if there != nil || assign.BackupAskDigest(current) != assign.ShortDigest(want.Composed) { + if there != nil { after := -1 if there == nil { after = current.Asking @@ -1126,7 +600,7 @@ func (p *passer) backups(ctx context.Context, reverting map[string]bool) { // backupAsk writes the backup ask at its asking and raises the condition of the failed backup, once. func (p *passer) backupAsk(ctx context.Context, key, node, module string, want mwc.Want, words string, asking int) { - a := assign.BackupAsk(key, node, module, want.Fp, want.Composed, words, asking, time.Now()) + a := assign.BackupAsk(key, node, module, want.Fp, words, asking, time.Now()) p.put(ctx, buckets.Asks(), key, func() ([]byte, error) { return assign.EncodeAsk(key, a) }) c := assign.BackupCondition(node, module, want.Fp, words, true) if p.say == nil || p.said[c.Key] { @@ -1176,13 +650,11 @@ func (p *passer) yeses(ctx context.Context, module string, m assign.Module, befo p.recordAsk(ctx, module, w.Fp, assign.RecordNeed{Node: node, Digest: h.Waits, Judgement: h.Judgement, Words: h.Detail}) } } - // A want carried over from before the node composed its own may still name the controller's composition; a placed - // node's want names the build alone (ADR 0547 §2). if yes == "" { release = "" } - if yes != w.Yes || release != w.Release || w.Composed != "" || !w.Self { - w.Yes, w.Release, w.Composed, w.Self = yes, release, "", true + if yes != w.Yes || release != w.Release || !w.Self { + w.Yes, w.Release, w.Self = yes, release, true out.Wants[node] = w } } @@ -1196,139 +668,6 @@ func chosenOf(chosen map[string]catalogue.PersonPolicy, module string) *catalogu return nil } -// provisionsOf answers, per module, the provisions it consumes on a node, by key (ADR 0570 §4), from the seat -// mesh-provisioner's state; nil while the state is not read (the seat not delivered yet), so no want waits on it. A failed -// provision is also said as the assigner's finding `provider..provision-failed` (a waiting one is the watchman's -// needs-operator), deleted once it is no longer failed. -func (p *passer) provisionsOf() func(module string) func(node string) map[string]meshprovisionerclient.Provision { - if p.kvs[buckets.Provisions] == nil { - return func(string) func(string) map[string]meshprovisionerclient.Provision { return nil } - } - by := map[string]map[string]map[string]meshprovisionerclient.Provision{} // module, node, key - failed := map[string]meshprovisionerclient.Provision{} - for key, raw := range p.w.read(buckets.Provisions) { - pr, err := meshprovisionerclient.Decode(raw) - if err != nil { - continue - } - node, module := pr.Consumer.Node, pr.Consumer.Module - if by[module] == nil { - by[module] = map[string]map[string]meshprovisionerclient.Provision{} - } - if by[module][node] == nil { - by[module][node] = map[string]meshprovisionerclient.Provision{} - } - by[module][node][key] = pr - if pr.State == meshprovisionerclient.Failed { - failed[key] = pr - } - } - p.provisionFindings(failed) - return func(module string) func(string) map[string]meshprovisionerclient.Provision { - return func(node string) map[string]meshprovisionerclient.Provision { return by[module][node] } - } -} - -// provisionFindings writes the finding of each failed provision and deletes the ones no longer failed. -func (p *passer) provisionFindings(failed map[string]meshprovisionerclient.Provision) { - if p.findings.Put == nil { - return - } - now := time.Now - if p.now != nil { - now = p.now - } - for key, pr := range failed { - if p.failedProvisions[key] { - continue - } - f, err := assign.ProvisionFailedFinding(key, pr, now()) - if err == nil { - err = p.findings.Put(f) - } - if err != nil { - p.logf("the failed provision %s is not said: %v", key, err) - continue - } - if p.failedProvisions == nil { - p.failedProvisions = map[string]bool{} - } - p.failedProvisions[key] = true - } - for key := range p.failedProvisions { - if _, still := failed[key]; still { - continue - } - if err := p.findings.Delete(assign.ProvisionFailedKey(key)); err != nil { - p.logf("the failed provision %s is not cleared: %v", key, err) - continue - } - delete(p.failedProvisions, key) - } -} - -// MissingBound is how long a required provision may have no key before it is said: a module's delivery bound, the default -// when its manifest names none (ADR 0564 clause 13 (d)). -var MissingBound = motion.DefaultDeliveryBound - -// missingPulse is the pulse key of a provision with no key, `provision-missing...`. -func missingPulse(key string) string { return "provision-missing." + key } - -// provisionsMissing is the loud end of a required provision with no key (the lead, 2026-10-29, ADR 0570 §4): each one is -// pulsed with the bound when it is first seen; once its pulse expired and it still has no key, the finding -// `provider....provision-missing` is written; one keyed again ends both. The pulse's expiry, heard on -// the watch, is the pass that writes it: no timer. -func (p *passer) provisionsMissing(ctx context.Context, missing map[string]bool) { - pulses := p.kvs[plc.PulsesBucket()] - if p.missing == nil { - p.missing = map[string]bool{} - } - for key := range missing { - switch { - case !p.missing[key]: - p.missing[key] = true - if pulses != nil { - p.pulse(ctx, missingPulse(key), MissingBound) - } - case pulses == nil || !p.pulsing(missingPulse(key)): - if p.missingSaid[key] || p.findings.Put == nil { - continue - } - f, err := assign.ProvisionMissingFinding(key, MissingBound, p.clock()) - if err == nil { - err = p.findings.Put(f) - } - if err != nil { - p.logf("the missing provision %s is not said: %v", key, err) - continue - } - if p.missingSaid == nil { - p.missingSaid = map[string]bool{} - } - p.missingSaid[key] = true - } - } - for key := range p.missing { - if missing[key] { - continue - } - delete(p.missing, key) - if pulses != nil && p.pulsing(missingPulse(key)) { - if err := pulses.Delete(ctx, missingPulse(key)); err == nil { - p.w.put(plc.PulsesBucket(), missingPulse(key), nil) - } - } - if p.missingSaid[key] && p.findings.Delete != nil { - if err := p.findings.Delete(assign.ProvisionMissingKey(key)); err != nil { - p.logf("the missing provision %s is not cleared: %v", key, err) - p.missing[key] = true - continue - } - delete(p.missingSaid, key) - } - } -} - // unplacedFindings writes, from the lease write, the finding `machine..unplaced` for each node that runs but no // assignments names, which gets no want (ADR 0571 §6), and deletes it once the node is placed. func (p *passer) unplacedFindings() { @@ -1423,40 +762,6 @@ func (p *passer) seatHolders(ctx context.Context, manifests map[string]catalogue } } -// offered says whether some manifest of the trunk, or a module the mesh provides itself, offers a provision under its name -// (a module offers its own name too, catalogue.Manifest.Offers): what a requirement nobody offers is told from. -func offered(manifests map[string]catalogue.Manifest) func(string) bool { - names := map[string]bool{} - add := func(m catalogue.Manifest) { - for _, o := range m.Offers() { - names[o] = true - } - } - for _, m := range manifests { - add(m) - } - for _, module := range overlay.ProvidedModules() { - if m, _, err := overlay.ProvidedManifest(module); err == nil { - add(m) - } - } - return func(provision string) bool { return names[provision] } -} - -// answering is, by provision, the manifests of the modules placed on some node that provide it: what a want's -// provisions are judged by (catalogue.Awaited, issue 774). A provider in the catalogue placed nowhere answers nobody. -func answering(manifests map[string]catalogue.Manifest, n assign.Nodes) func(string) []catalogue.Manifest { - return func(provision string) []catalogue.Manifest { - var out []catalogue.Manifest - for _, name := range slices.Sorted(maps.Keys(n.Providers[provision])) { - if m, ok := manifests[name]; ok && len(n.Providers[provision][name]) > 0 { - out = append(out, m) - } - } - return out - } -} - // branchView is what one pass reads of the catalog half of one watched branch (design 47 G §184, §195): its modules, their // manifests, the nodes the rules read through them, the module graph and the two readings of what runs. type branchView struct { @@ -1536,56 +841,11 @@ func wantedHere(wanted map[string]mwc.Want, own []string, groups []group, first return out } -// noPipelineRules is why each node of module gets no want while no pipeline passed at fp on its branches (design 47 G §29): said as -// any rule's reason (sayNoWant), once. -func noPipelineRules(nodes []string, module, fp string, branches []string) map[string]assign.Holding { - out := map[string]assign.Holding{} - for _, node := range nodes { - out[node] = assign.Holding{Rule: mwc.RuleNotHeld, Words: fmt.Sprintf("no pipeline of %s passed at %s on %s, and a want follows a passed pipeline (design 47 G §29)", - module, shortHex(fp), strings.Join(branches, ", "))} - } - return out -} - -// runnerFirstStart says whether module claims the seat mesh-pipeline-runner while no module claiming it runs on any node (design 47 -// G A-428): read from the node-engine seat's running statements the assigner hears, each module's claims from its manifest. With -// no running heard yet it says false: the gate stands until the assigner knows. -func runnerFirstStart(module string, manifests map[string]catalogue.Manifest, running map[string]map[string]nodeengine.Running) bool { - if !manifests[module].ClaimsSeat(mpr.Seat) || running == nil { - return false - } - for _, mods := range running { - for m, r := range mods { - if !r.Stopped && r.Fingerprint != "" && manifests[m].ClaimsSeat(mpr.Seat) { - return false - } - } - } - return true -} - -// releasedBuild says whether the build at fp is a released version's, as a want of it names it (namedBuild): such a build counts as -// done for rule 1 whatever the builder's build.ended said or the assigner kept of it (design 47 G A-429). -func releasedBuild(named func(fp string) (assign.KeptBuild, bool), fp string) bool { - if named == nil { - return false - } - k, ok := named(fp) - return ok && k.Version != "" -} - -// nameHeldBuilds rewrites, for a module the trunk's newer fingerprint of which passed no pipeline, each want held at an older build -// that names no version: with the release whose parts are exactly what that build made, everything else of the want as it stands (novox 2026-11-16: mesh-hand-acts held at 4eb3cf48, released 0.1.13, after the -// mesh-sdk merge moved its fingerprint). No want is written for the unpassed fingerprint, nor for a build with no release. -func (p *passer) nameHeldBuilds(ctx context.Context, name, trunkFP string, group []string, n assign.Nodes, held map[string]mwc.Want, named func(fp string) (assign.KeptBuild, bool)) { - for _, node := range group { - w, ok := held[node] - if !ok || w.Removed || w.Fp == "" || w.Fp == trunkFP || w.Version != "" { - continue - } - if k, found := named(w.Fp); !found || k.Version == "" { - continue - } - p.putWant(ctx, node, name, n.NameKept(assign.Module{KeptBuilds: named}, node, w)) +// movedTo says whether the delivery moved owner to node: owner's want there names a fingerprint its have there does not yet run +// healthy (design 47 G §706, *What counts as an outside cause*: a rewritten file of a module the delivery moved is that module's). +func (p *passer) movedTo(wanted map[string]map[string]mwc.Want, n assign.Nodes) func(owner, node string) bool { + return func(owner, node string) bool { + w, ok := wanted[owner][node] + return ok && !w.Removed && !n.RunsHealthy(node, owner, w.Fp) } } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index 195df7f5..5a710b79 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -6,7 +6,7 @@ import ( "encoding/json" "errors" "fmt" - stdio "git.novox.be/novox/mesh-sdk/go" + mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" "maps" "mesh-delivery-assigner/internal/buckets" "mesh-delivery-assigner/internal/puts" @@ -24,7 +24,6 @@ import ( mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" mwseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents" "git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate" "github.com/nats-io/nats.go/jetstream" @@ -39,7 +38,6 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph" "mesh-delivery-assigner/internal/assign" plc "mesh-delivery-assigner/internal/assignment" @@ -106,166 +104,13 @@ func composes(t *testing.T, w *world, node, digest string) { var fpTest = meshcatalogclient.SourcePrefix2 + strings.Repeat("b", 64) -// 1906/H2: given a hold lifted by itself, when the assigner is started again and the same fingerprint is held again, then the -// hold ask is written under another asking, with an id of its own, so the teller raises it (the router drops a second ask under -// an id it keeps). -func TestAHoldAskedAgainAfterALiftAndARestartHasANewID(t *testing.T) { - asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}} - kvs := kvSet{buckets.Asks(): asksKV, buckets.Wants(): wantsKV} - key, _ := asks.HoldAskKey("web", fpTest) - held := &mwc.Held{Node: "ace", Words: "health failed"} - first := newPasser(kvs, time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC)) - first.holdAsk(context.Background(), "web", fpTest, held) - a1, err := decodeAsk(key, asksKV.values[key]) - if err != nil { - t.Fatal(err) - } - first.lift(context.Background(), "web", fpTest) - if _, there := asksKV.values[key]; there { - t.Fatal("the lift left the hold ask") - } - again := newPasser(kvs, time.Date(2026, 10, 25, 2, 0, 0, 0, time.UTC)) // started again: it remembers nothing - again.holdAsk(context.Background(), "web", fpTest, held) - a2, err := decodeAsk(key, asksKV.values[key]) - if err != nil { - t.Fatal(err) - } - if a2.Ask.ID == a1.Ask.ID { - t.Errorf("the ask written again after a lift and a restart has the id %s of the first", a1.Ask.ID) - } - // Issue 1966, ADR 0668 §5: assigner_ask_again writes the hold again under a new asking and records the hand act; the - // asking the operator answered is refused, naming the answer; a proposal's authorising ask is refused by its rule. - answersKV := &memKV{values: map[string][]byte{}} - st := memStore{asksState: asksKV, answersState: answersKV} - var recorded []string - record := func(key string, body any) (json.RawMessage, error) { - raw, _ := json.Marshal(body) - recorded = append(recorded, key+" "+string(raw)) - return json.RawMessage(`{}`), nil - } - now := time.Date(2026, 10, 25, 3, 0, 0, 0, time.UTC) - out, err := askAgainByHand(st, record, key, "the channel lost it", "lead@shanks", now) - a3, derr := decodeAsk(key, asksKV.values[key]) - if err != nil || derr != nil || a3.Ask.ID == a2.Ask.ID || len(recorded) != 1 || !strings.Contains(recorded[0], askAgainTool+" "+key) { - t.Fatalf("ask again: %v, %v; asked %s (was %s), recorded %v", out, err, a3.Ask.ID, a2.Ask.ID, recorded) - } - answersKV.values[key], _ = mdc.EncodeAnswer(mdc.Answer{Option: "keep", Warrant: a3.Ask.ID + "@2026-10-25T03:01:00Z", Words: "the operator"}) - if _, err := askAgainByHand(st, record, key, "again", "lead@shanks", now); err == nil || !strings.Contains(err.Error(), "keep") { - t.Errorf("an answered asking is asked again: %v", err) - } - // Issue 1966 note 165 (AC165-2, AC165-3): the teller's ask-ended for a backup ask whose want names another fingerprint writes - // nothing and the journal says it is stale; the same ask with its want at the same fingerprint in another composition, or no - // want yet, is written again. - backKey, _ := asks.BackupAskKey("ace", "web", fpTest) - fpOther := meshcatalogclient.SourcePrefix2 + strings.Repeat("c", 64) - for _, c := range []struct { - name string - want *mwc.Want - have *nodeengine.HaveByNodeModuleValue // A-453: the module's have on the node - stale bool - }{ - {"its want names another fingerprint", &mwc.Want{Fp: fpOther, Self: true}, nil, true}, - {"its want is removed", &mwc.Want{Removed: true}, nil, true}, - {"its want names the fingerprint in another composition", &mwc.Want{Fp: fpTest, Composed: "sha256:" + strings.Repeat("e", 64)}, nil, false}, - {"no want yet", nil, nil, false}, - // A-453: only a held have with no wait is a failed backup; a hold that carries its wait, or a have no longer held, takes the ask away. - {"its have is held with no wait", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits}, false}, - {"its have is held with a wait", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits, - Wait: &nodeengine.Wait{Step: nodeengine.WaitStepCompose, Reason: "entry-missing", Since: now}}, true}, - {"its have runs", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}, true}, - {"its have backs up again (A-453a)", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseBackingUp, Health: nodeengineclient.HealthWaits}, false}, - {"its have is held, naming the build it keeps running", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpOther, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits}, false}, - {"its have applies at another fingerprint", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpOther, Phase: nodeengineclient.PhaseApplying, Health: nodeengineclient.HealthWaits}, true}, - } { - t.Run("ended backup ask: "+c.name, func(t *testing.T) { - bkv := &memKV{values: map[string][]byte{}} - pb := newPasser(kvSet{buckets.Asks(): bkv, buckets.Wants(): &memKV{values: map[string][]byte{}}, buckets.Have: &memKV{values: map[string][]byte{}}}, now) - ba := assign.BackupAsk(backKey, "ace", "web", fpTest, "sha256:"+strings.Repeat("d", 64), "w", 2, now) - bv, err := assign.EncodeAsk(backKey, ba) - if err != nil { - t.Fatal(err) - } - bkv.values[backKey] = bv - if c.want != nil { - wkey, _ := mwc.WantKey("ace", "web") - raw, _ := mwc.EncodeWant(*c.want) - pb.w.put(buckets.Wants(), wkey, raw) - } - if c.have != nil { - hkey, _ := nodeengineclient.HaveKey("ace", "web") - raw, err := nodeengineclient.EncodeHave(*c.have) - if err != nil { - t.Fatal(err) - } - pb.w.put(buckets.Have, hkey, raw) - } - _, bask, _ := assign.DecodeAsk(backKey, bv) - body, err := tellerevents.EncodeEnded(tellerevents.Ended{Key: backKey, Ask: bask.ID, Outcome: occ.OutcomeExpired}) - if err != nil { - t.Fatal(err) - } - var said []string - stale := func(key string) string { - return pb.staleAsk(key, map[string]meshcatalog.ModulesByModuleValue{"web": {Fp2: fpOther}}) - } - if err := askAgain(context.Background(), bkv, nil, stale, body, now.Add(time.Hour), func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }); err != nil { - t.Fatal(err) - } - written := !bytes.Equal(bkv.values[backKey], bv) - if written == c.stale || (c.stale && !strings.Contains(strings.Join(said, "\n"), "stale")) { - t.Errorf("written again = %v (stale %v), journal %q", written, c.stale, said) - } - }) - } - for _, c := range []struct{ key, by, why string }{{key, stdio.Unproven, "a caller"}, {"hold.none", "lead@shanks", "no ask"}} { - if _, err := askAgainByHand(st, record, c.key, "w", c.by, now); err == nil { - t.Errorf("%s: asked again", c.why) - } - } - // ADR 0668 §3 (review round 1): the bind's read of the operator channel's asks keeps its wait through transport faults; - // past five minutes with the channel's machine heard it is the finding call-unanswered, cleared once answered; with - // the machine away none is written. A refusal is an answer and ends the wait. - for _, heard := range []bool{false, true} { - clock, findings := now, map[string][]byte{} - cw := callWait{heard: func() bool { return heard }, now: func() time.Time { return clock }, - findings: assign.Findings{Put: putInto(findings), - Delete: func(k string) error { delete(findings, k); return nil }}} - tries, during := 0, 0 - answer, err := cw.keep(context.Background(), asksCallID, channelAsksVerb, func() (json.RawMessage, error) { - tries++ - clock = clock.Add(2 * time.Minute) - if tries == 5 { - during = len(findings) - return json.RawMessage(`{"asks":[]}`), nil - } - return nil, errors.New("nats: no responders available for request") - }, t.Logf) - if want := map[bool]int{false: 0, true: 1}[heard]; err != nil || string(answer) != `{"asks":[]}` || tries != 5 || during != want || len(findings) != 0 { - t.Errorf("heard %v: %s %v after %d tries; findings during %d, after %d", heard, answer, err, tries, during, len(findings)) - } - } - if _, err := (callWait{}).keep(context.Background(), asksCallID, channelAsksVerb, func() (json.RawMessage, error) { - return nil, &busretry.Answered{Words: "refused"} - }, t.Logf); err == nil { - t.Error("a refusal was asked again") - } -} - -// 1906/H4: an answer counts only for the asking it answered: an answer to an earlier hold of the same fingerprint, or to an ask -// taken away, is not taken for the hold that stands now. +// 1906/H4: an answer counts only for the asking it answered: an answer to an earlier asking of the same ask, or to an ask +// taken away, is not taken for the asking that stands now. func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) { - // ADR 0603 §5: a provision is offered by a manifest's provides, by a module's own name, and by a module the mesh provides - // itself (mesh-addressing, mesh-wireguard's); nothing else. - off := offered(map[string]catalogue.Manifest{"node-tools": {Module: "node-tools"}, "db": {Module: "db", Provides: []catalogue.Offer{{Name: "database"}}}}) - for provision, want := range map[string]bool{"node-tools": true, "database": true, "mesh-addressing": true, "queue": false} { - if off(provision) != want { - t.Errorf("offered(%s) = %v, want %v", provision, !want, want) - } - } - key, _ := asks.HoldAskKey("web", fpTest) + key, _ := asks.BackupAskKey("ace", "web", fpTest) p := newPasser(kvSet{}, time.Now()) put := func(asking int) string { - a := assign.HoldAsk(key, "web", fpTest, "ace", "health failed", asking, time.Now()) + a := assign.BackupAsk(key, "ace", "web", fpTest, "disk full", asking, time.Now()) v, err := assign.EncodeAsk(key, a) if err != nil { t.Fatal(err) @@ -274,20 +119,20 @@ func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) { return mustAskID(a) } first := put(100) - v, _ := mdc.EncodeAnswer(mdc.Answer{Option: "release", Warrant: first + "@2026-10-25T01:00:00Z", Words: "the operator"}) + v, _ := mdc.EncodeAnswer(mdc.Answer{Option: assign.OptionBackupSkip, Warrant: first + "@2026-10-25T01:00:00Z", Words: "the operator"}) p.w.put(buckets.Answers, key, v) - if !p.answeredWith(key, "release") { + if !p.answeredWith(key, assign.OptionBackupSkip) { t.Fatal("the answer to the ask that stands is not taken") } - if p.answeredWith(key, "keep") { + if p.answeredWith(key, assign.OptionBackupAgain) { t.Error("an answer was taken for an option not chosen") } - put(200) // the same fingerprint held again, after a lift - if p.answeredWith(key, "release") { - t.Error("the answer to an earlier hold carried over to a later one") + put(200) // the same backup asked again + if p.answeredWith(key, assign.OptionBackupSkip) { + t.Error("the answer to an earlier asking carried over to a later one") } p.w.put(buckets.Asks(), key, nil) // the ask taken away - if p.answeredWith(key, "release") { + if p.answeredWith(key, assign.OptionBackupSkip) { t.Error("an answer to an ask taken away was taken") } t.Run("a put of the bus acts on the operator's warrant alone", putOfTheBus) @@ -678,7 +523,7 @@ func putOfTheBus(t *testing.T) { <-woke got = step() w, err := mwc.DecodeWant(wantsKV.values["want.novox.nats"]) - if got.Step != puts.Wanted || err != nil || w.Fp != fp || w.Composed != digest || !w.Put || !w.Hold { + if got.Step != puts.Wanted || err != nil || w.Fp != fp || !w.Put || !w.Hold { t.Fatalf("snapshotted, the put stands at %s (%s); the want %+v, %v", got.Step, got.Words, w, err) } if call := <-bus.calls; !strings.HasPrefix(call, "seat:mesh-hand-acts.warranted ") || !strings.Contains(call, a.Ask.ID) { // ADR 0565 §11: into the seat @@ -755,266 +600,6 @@ func putOfTheBus(t *testing.T) { } } -// 1945/AC21: given a module on neither list on a node, when passes follow, then the assigner raises its condition once, says it -// again after a say that failed, and clears it once when the module has a want again; a module on neither list is also written to -// the journal, so it is loud with no operator channel. -func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) { - // A pair only one source names is a condition of the same kind (issue 1126): folded here to hold the count. - aAssignmentsAloneNamesWhereModulesRun(t) - // The verb edges a want waits on come from the graph and what runs (ADR 0614 §2, §3): folded here to hold the count. - aWantWaitsOnTheVerbEdgesTheGraphKeeps(t) - t.Run("gathers", testThePassGathersTheModulesOnNeitherList) - p := newPasser(kvSet{}, time.Date(2026, 10, 25, 15, 0, 0, 0, time.UTC)) - var said []oc.Condition - failing := true - p.say = func(c oc.Condition) error { - if failing { - failing = false - return errors.New("the channel is away") - } - said = append(said, c) - return nil - } - var logged []string - p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) } - one := map[string]neitherOne{"g14.screen-lock": {node: "g14", module: "screen-lock", - holding: assign.Holding{Rule: mwc.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}}} - p.sayNeither(one, nil, nil) // the say fails - p.sayNeither(one, nil, nil) - p.sayNeither(one, nil, nil) - if len(said) != 1 || said[0].State != occ.StateRaised || said[0].Subject.Machine != "g14" || said[0].Subject.ID != "screen-lock" { - t.Fatalf("said = %+v, want one raised condition of screen-lock on g14", said) - } - if !strings.Contains(strings.Join(logged, "\n"), "screen-lock") { - t.Errorf("the journal does not name the module on neither list: %q", logged) - } - wants := map[string]string{"g14.screen-lock": ""} - p.sayNeither(map[string]neitherOne{}, nil, wants) - p.sayNeither(map[string]neitherOne{}, nil, wants) - if len(said) != 2 || said[1].State != occ.StateCleared || said[1].Key != said[0].Key { - t.Errorf("said = %+v, want the condition cleared once under its key", said) - } - if last := logged[len(logged)-1]; last != "screen-lock on g14 has a want again" { - t.Errorf("the clear said %q", last) - } - // 1945: a module that leaves neither list with no want is cleared in words saying so, never "has a want again". The bus is - // on neither list by a rule as any module (ADR 0601 §1), nothing for the operator to do. - bus := map[string]neitherOne{"novox.nats": {node: "novox", module: "nats", holding: assign.Holding{Rule: mwc.RuleGrant, Words: "no credential"}}} - p.sayNeither(bus, nil, nil) - if c := said[len(said)-1]; c.State != occ.StateRaised || !strings.HasPrefix(c.Explanation, occ.NothingToDo) || !strings.Contains(c.Explanation, "rule 6 holds its first want back: no credential") { - t.Errorf("the bus's neither condition = %+v", c) - } - p.sayNeither(map[string]neitherOne{}, nil, map[string]string{"novox.nats": "it is held, and its held condition says so"}) - if c := said[len(said)-1]; c.State != occ.StateCleared || strings.Contains(c.Resolved, "has a want") { - t.Errorf("a module that left neither list with no want was cleared as %+v", c) - } - if last := logged[len(logged)-1]; last != "nats on novox is no longer on neither list and has no want: it is held, and its held condition says so" { - t.Errorf("the clear without a want said %q", last) - } -} - -// 1945/AC21: the pass gathers the modules on neither list of every module's outcome, keyed by node and module. -func testThePassGathersTheModulesOnNeitherList(t *testing.T) { - got := map[string]neitherOne{} - gatherNeither(got, "screen-lock", assign.Outcome{Neither: map[string]assign.Holding{"g14": {Rule: mwc.RuleGrant, Words: "w"}}}) - gatherNeither(got, "rofi", assign.Outcome{}) - if len(got) != 1 || got["g14.screen-lock"].node != "g14" || got["g14.screen-lock"].module != "screen-lock" || got["g14.screen-lock"].holding.Rule != mwc.RuleGrant { - t.Errorf("gathered = %+v", got) - } - // 1945 (the bus bar): a node off neither list is cleared in words that say why it has no want; an operator's put running - // through its steps owns the want, so it is named, never "the node does not run it". - for _, c := range []struct { - name string - puts map[string]puts.Put - before map[string]mwc.Want - why string - }{ - {"a live want", nil, map[string]mwc.Want{"novox": {Fp: fpTest}}, ""}, - {"a put on its steps", map[string]puts.Put{"novox": {FP: fpTest, Step: puts.Person}}, nil, "its put waits on its own steps and the operator's answer"}, - {"nothing", nil, nil, "the node does not run it"}, - {"an ended put", map[string]puts.Put{"novox": {FP: fpTest, Step: puts.Refused}}, nil, "the node does not run it"}, - } { - left := map[string]string{} - leftNeither(left, assign.Module{Name: "nats", FP: fpTest, Nodes: []string{"novox"}, Puts: c.puts}, c.before, assign.Outcome{}) - if why, there := left["novox.nats"]; !there || why != c.why { - t.Errorf("%s: left %q (%v), want %q", c.name, why, there, c.why) - } - } - // 1945 (a first want held back silently): the journal says why a node gets no want when the reason appears or changes, and - // when it has a want again: one line per change, not per pass. - p := newPasser(kvSet{}, time.Date(2026, 10, 25, 21, 30, 0, 0, time.UTC)) - var logged []string - p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) } - notDone := assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: mwc.RuleBuildDone, Words: "the build of rofi at 4e7b118e is being built"}}} - wanted := assign.Outcome{Wants: map[string]mwc.Want{"shanks": {Fp: fpTest, Composed: "sha256:" + strings.Repeat("c", 64)}}} - for i, c := range []struct { - before map[string]mwc.Want - out assign.Outcome - line string // the line said in this pass, "" for none - }{ - {nil, notDone, "rofi on shanks gets no want: rule 1: the build of rofi at 4e7b118e is being built"}, - {nil, notDone, ""}, - {map[string]mwc.Want{"shanks": {Removed: true}}, notDone, ""}, - {nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: mwc.RuleGrant, Words: "no credential"}}}, "rofi on shanks gets no want: rule 6: no credential"}, - {nil, wanted, "rofi on shanks has a want"}, - {map[string]mwc.Want{"shanks": wanted.Wants["shanks"]}, assign.Outcome{}, ""}, - } { - logged = nil - p.sayNoWant("rofi", []string{"shanks"}, c.before, c.out) - if got := strings.Join(logged, "\n"); got != c.line { - t.Errorf("pass %d said %q, want %q", i, got, c.line) - } - } - // 1496, ADR 0557 §1: a failure that is the node's own (its fingerprint healthy elsewhere, not held) raises the node's condition - // once, naming the node, the module, F and the have's words, and clears it once when it ends. - var conds []oc.Condition - p.say = func(c oc.Condition) error { conds = append(conds, c); return nil } - failing := map[string]assign.NodeFailure{"shanks": {Others: []string{"ace"}, Words: "health failed, phase running: w"}} - for i, c := range []struct { - failed map[string]assign.NodeFailure - state string // the condition said in this pass, "" for none - }{ - {failing, occ.StateRaised}, - {failing, ""}, - {nil, occ.StateCleared}, - {nil, ""}, - } { - conds = nil - p.sayNodeFailed("rofi", fpTest, []string{"shanks"}, assign.Nodes{}, assign.Outcome{NodeFailed: c.failed}, false) - switch { - case c.state == "" && len(conds) != 0, c.state != "" && (len(conds) != 1 || conds[0].State != c.state): - t.Errorf("node failed, pass %d said %+v, want %q", i, conds, c.state) - case c.state == occ.StateRaised && (conds[0].Key != "mesh-delivery-assigner.failed-alone.shanks.rofi" || - !strings.Contains(conds[0].Explanation, "failed on shanks (health failed, phase running: w), while it runs healthy on ace")): - t.Errorf("node failed, raised %+v", conds[0]) - } - } - // D2: a node that leaves the module's nodes while it failed alone is cleared once; the first pass after a start clears every - // node the presence bucket knows, for every module, failed have or not (a node that recovered unwatched leaves none). - conds = nil - p.sayNodeFailed("rofi", fpTest, []string{"shanks"}, assign.Nodes{}, assign.Outcome{NodeFailed: failing}, false) - p.sayNodeFailed("rofi", fpTest, nil, assign.Nodes{}, assign.Outcome{}, false) - p.sayNodeFailed("rofi", fpTest, nil, assign.Nodes{}, assign.Outcome{}, false) - if len(conds) != 2 || conds[1].State != occ.StateCleared || conds[1].Key != "mesh-delivery-assigner.failed-alone.shanks.rofi" { - t.Errorf("a node leaving the module said %+v, want one raise and one clear", conds) - } - known := assign.Nodes{Online: map[string]bool{"ace": true, "g14": false}, - Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"g14": {"rofi": {Fp: fpTest, Health: nodeengineclient.HealthFailed}}}} - for _, c := range []struct { - name string - known assign.Nodes - keys []string - }{ - {"a failed have", known, []string{"mesh-delivery-assigner.failed-alone.ace.rofi", "mesh-delivery-assigner.failed-alone.g14.rofi"}}, - {"no failed have", assign.Nodes{Online: known.Online}, []string{"mesh-delivery-assigner.failed-alone.ace.rofi", "mesh-delivery-assigner.failed-alone.g14.rofi"}}, - } { - conds = nil - fresh := newPasser(kvSet{}, time.Date(2026, 10, 26, 22, 0, 0, 0, time.UTC)) - fresh.logf = func(string, ...any) {} - fresh.say = func(c oc.Condition) error { conds = append(conds, c); return nil } - fresh.sayNodeFailed("rofi", fpTest, nil, c.known, assign.Outcome{}, true) - var keys []string - for _, cond := range conds { - if cond.State == occ.StateCleared { - keys = append(keys, cond.Key) - } - } - if !slices.Equal(keys, c.keys) || len(conds) != len(c.keys) { - t.Errorf("first pass, %s: said %+v, want clears of %q", c.name, conds, c.keys) - } - } -} - -// A neither or held condition a previous process raised is cleared by the first pass after a start: a fresh pass remembers -// nothing it said, so it clears, once, the neither condition of every assigned module not on neither list and the held condition of -// every module that is not held. Clearing a condition that is not open is quiet at the router. -func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) { - t.Run("a held module", testAHeldModuleRaisesNoNeitherAndClearsAStandingOne) - wantsKV := &memKV{values: map[string][]byte{}} - p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, time.Now()) - var said []oc.Condition - p.say = func(c oc.Condition) error { said = append(said, c); return nil } - mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) - putModuleRaw(p.w, trunkBranch, "web", mod) - placeOn(t, p.w, "g14", "web") - on := true - p.w.heardPresence("g14", !on, time.Now()) - have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) - p.w.put(buckets.Have, "g14.web", have) - key, _ := mwc.WantKey("g14", "web") - want, _ := mwc.EncodeWant(mwc.Want{Fp: fpTest, Composed: "sha256:" + strings.Repeat("c", 64)}) - p.w.put(buckets.Wants(), key, want) - p.wants(context.Background()) - cleared := map[string]bool{} - for _, c := range said { - if c.State != occ.StateCleared { - t.Errorf("the first pass raised %+v", c) - } - cleared[c.Key] = true - } - neither := assign.NeitherCondition("g14", "web", assign.Holding{}, false).Key - held := assign.HeldCondition("web", fpTest, "", "", false, false).Key - if !cleared[neither] || !cleared[held] { - t.Errorf("cleared = %v, want %s and %s", cleared, neither, held) - } - n := len(said) - p.wants(context.Background()) - if len(said) != n { - t.Errorf("a second pass said again: %+v", said[n:]) - } -} - -// 1945/AC24: given a module held at F under rule 2 on a node that runs it with no want, when the assigner passes, then it raises no -// neither condition for it, a neither condition this process raised for it before is cleared, and so is one a previous process -// raised (the first pass after a start), each once. -func testAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) { - setup := func() (*passer, *[]oc.Condition) { - p := newPasser(kvSet{buckets.Wants(): &memKV{values: map[string][]byte{}}, buckets.Asks(): &memKV{values: map[string][]byte{}}}, time.Now()) - said := &[]oc.Condition{} - p.say = func(c oc.Condition) error { *said = append(*said, c); return nil } - mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fpTest, Repository: "novox/resolved", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) - putModuleRaw(p.w, trunkBranch, "systemd-resolved", mod) - placeOn(t, p.w, "shanks", "systemd-resolved") - on := true - p.w.heardPresence("shanks", !on, time.Now()) - have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed}) - p.w.put(buckets.Have, "shanks.systemd-resolved", have) - hkey, _ := mwc.HeldKey("systemd-resolved", fpTest) - held, _ := mwc.EncodeHeld(mwc.Held{Node: "shanks", Words: "health failed"}) - p.w.put(buckets.Wants(), hkey, held) - return p, said - } - key := assign.NeitherCondition("shanks", "systemd-resolved", assign.Holding{}, false).Key - neithers := func(said []oc.Condition) (raised, cleared int) { - for _, c := range said { - if c.Key != key { - continue - } - if c.State == occ.StateRaised { - raised++ - } else { - cleared++ - } - } - return - } - // a standing neither this process raised - p, said := setup() - p.neither = map[string]neitherOne{"shanks.systemd-resolved": {node: "shanks", module: "systemd-resolved", holding: assign.Holding{Rule: mwc.RuleNotHeld}}} - p.wants(context.Background()) - p.wants(context.Background()) - if r, c := neithers(*said); r != 0 || c != 1 { - t.Errorf("standing in this process: %d raised, %d cleared, want 0 and 1: %+v", r, c, *said) - } - // one a previous process raised: the first pass after a start clears it - p, said = setup() - p.wants(context.Background()) - p.wants(context.Background()) - if r, c := neithers(*said); r != 0 || c != 1 { - t.Errorf("after a start: %d raised, %d cleared, want 0 and 1: %+v", r, c, *said) - } -} - // 1906 note 28: an ask outlives its fingerprint. Each pass takes away the hold, restore and record asks of a module whose // fingerprint (or composition) is not the trunk's now, or whose module left the trunk; a trunk not read takes nothing away; a // backup ask is taken away when its module's want on the node names another fingerprint or is removed, and left when the want @@ -1026,15 +611,10 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) { return map[string]meshcatalog.ModulesByModuleValue{"web": {Fp2: fp, Manifest: "sha256:" + strings.Repeat("1", 64), Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}} } type ask struct{ name, key string } - holdOld, _ := asks.HoldAskKey("web", fpOld) - holdNew, _ := asks.HoldAskKey("web", fpNew) - holdGone, _ := asks.HoldAskKey("gone", fpOld) - restOld, _ := asks.RestoreAskKey("novox", "web", fpOld) - restNew, _ := asks.RestoreAskKey("novox", "web", fpNew) recOld, _ := asks.RecordAskKey("novox", "web", digOld) recNew, _ := asks.RecordAskKey("novox", "web", digNew) backOld, _ := asks.BackupAskKey("novox", "web", fpOld) - all := []string{holdOld, holdNew, holdGone, restOld, restNew, recOld, recNew, backOld} + all := []string{recOld, recNew, backOld} for _, c := range []struct { name string trunk map[string]meshcatalog.ModulesByModuleValue @@ -1042,13 +622,12 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) { noDigit bool // the node's own composition is not read want *mwc.Want // the want of web on novox, nil when none is written }{ - {"the trunk moved on", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false, nil}, - {"the backup's want names another fingerprint", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld, backOld}, false, &mwc.Want{Fp: fpNew, Self: true}}, - {"the backup's want is removed", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld, backOld}, false, &mwc.Want{Removed: true}}, - {"the backup's want names the fingerprint in another composition (its have waits for a yes: a mid-move have is not past the hold, A-453a)", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false, &mwc.Want{Fp: fpOld, Composed: digNew}}, - {"the trunk is at the old fingerprint", trunkOf(fpOld), []string{holdNew, holdGone, restNew, recOld}, false, nil}, - {"the trunk's fingerprint is not read yet", trunkOf(""), []string{holdGone}, false, nil}, - {"the composition is not read yet", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, true, nil}, + {"the trunk moved on", trunkOf(fpNew), []string{recOld}, false, nil}, + {"the backup's want names another fingerprint", trunkOf(fpNew), []string{recOld, backOld}, false, &mwc.Want{Fp: fpNew, Self: true}}, + {"the backup's want is removed", trunkOf(fpNew), []string{recOld, backOld}, false, &mwc.Want{Removed: true}}, + {"the trunk is at the old fingerprint", trunkOf(fpOld), []string{recOld}, false, nil}, + {"the trunk's fingerprint is not read yet", trunkOf(""), nil, false, nil}, + {"the composition is not read yet", trunkOf(fpNew), nil, true, nil}, {"the trunk is not read at all", nil, nil, false, nil}, {"the trunk is not read at all, the backup's want moved", nil, nil, false, &mwc.Want{Fp: fpNew, Self: true}}, } { @@ -1061,14 +640,10 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) { k, _ := asks.ParseAnyAskKey(key) var a asks.Ask switch k.Kind { - case asks.AskKindHold: - a = assign.HoldAsk(key, k.Module, k.FP, "novox", "w", 1, time.Now()) - case asks.AskKindRestore: - a = assign.RestoreAsk(key, k.Node, k.Module, k.FP, "w", 2, time.Now()) case asks.AskKindRecord: a = assign.RecordAsk(key, k.Node, k.Module, fpNew, assign.RecordNeed{Node: k.Node, Digest: k.Composed}, 2, time.Now()) default: - a = assign.BackupAsk(key, k.Node, k.Module, k.FP, digOld, "w", 2, time.Now()) + a = assign.BackupAsk(key, k.Node, k.Module, k.FP, "w", 2, time.Now()) } v, err := assign.EncodeAsk(key, a) if err != nil { @@ -1095,20 +670,7 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) { } p.w.put(buckets.Wants(), wkey, raw) } - heldOld, _ := mwc.HeldKey("web", fpOld) - wantsKV.values[heldOld] = []byte("x") - p.w.put(buckets.Wants(), heldOld, []byte("x")) - firstAsk, _ := decodeAsk(holdOld, asksKV.values[holdOld]) p.staleAsks(context.Background(), c.trunk) - // the held record and condition of a fingerprint whose hold ask went are taken away with it - heldOld, _ = mwc.HeldKey("web", fpOld) - _, heldThere := wantsKV.values[heldOld] - cleared := slices.ContainsFunc(said, func(c oc.Condition) bool { - return c.Key == assign.HeldCondition("web", fpOld, "", "", false, false).Key && c.State == occ.StateCleared - }) - if gone := slices.Contains(c.gone, holdOld); !heldThere || cleared != gone { - t.Errorf("hold ask of the old fingerprint gone = %v: its held record stays = %v (must), its condition cleared = %v", gone, heldThere, cleared) - } // the failed backup's condition is cleared with its ask, and with no other (AC165-1) backCleared := slices.ContainsFunc(said, func(c oc.Condition) bool { return c.Key == assign.BackupCondition("novox", "web", fpOld, "", false).Key && c.State == occ.StateCleared @@ -1123,66 +685,9 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) { t.Errorf("%s: left in the bucket = %v, want %v", key, there, want) } } - if slices.Contains(c.gone, holdOld) { - // the trunk returns to the fingerprint (a revert): the held record still holds it, and it is asked and said again - p.now = func() time.Time { return time.Now().Add(time.Hour) } - p.holdAsk(context.Background(), "web", fpOld, nil) - p.heldCondition(context.Background(), "web", fpOld, nil, false) - again, err := decodeAsk(holdOld, asksKV.values[holdOld]) - raised := slices.ContainsFunc(said, func(c oc.Condition) bool { return c.State == occ.StateRaised }) - if err != nil || again.Ask.ID == firstAsk.Ask.ID || !raised { - t.Errorf("the trunk returned to the held fingerprint: ask %q (first %q, %v), condition raised again = %v", again.Ask.ID, firstAsk.Ask.ID, err, raised) - } - } }) } - t.Run("a clear that failed to be said is said at the next pass, and the ask goes after it", func(t *testing.T) { - asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}} - p := newPasser(kvSet{buckets.Asks(): asksKV, buckets.Wants(): wantsKV}, time.Now()) - var said []oc.Condition - fail := true - p.say = func(c oc.Condition) error { - if fail { - fail = false - return errors.New("router unreachable") - } - said = append(said, c) - return nil - } - a := assign.HoldAsk(holdOld, "web", fpOld, "novox", "w", 1, time.Now()) - v, err := assign.EncodeAsk(holdOld, a) - if err != nil { - t.Fatal(err) - } - asksKV.values[holdOld] = v - p.w.put(buckets.Asks(), holdOld, v) - p.staleAsks(context.Background(), trunkOf(fpNew)) - if _, there := asksKV.values[holdOld]; !there || len(said) != 0 { - t.Fatalf("after a failed clear: ask left = %v, conditions said = %d; want the ask kept and nothing said", there, len(said)) - } - p.staleAsks(context.Background(), trunkOf(fpNew)) - if _, there := asksKV.values[holdOld]; there || len(said) != 1 || said[0].State != occ.StateCleared { - t.Errorf("at the next pass: ask left = %v, conditions said = %v; want the ask gone and the clear said", there, said) - } - }) - - t.Run("a lift takes the restore asks of its fingerprint on every node", func(t *testing.T) { - asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}} - p := newPasser(kvSet{buckets.Asks(): asksKV, buckets.Wants(): wantsKV}, time.Now()) - keep, _ := asks.RestoreAskKey("ace", "web", fpNew) - keepOther, _ := asks.RestoreAskKey("ace", "other", fpOld) - for _, key := range []string{restOld, mustKey(asks.RestoreAskKey("ace", "web", fpOld)), keep, keepOther} { - asksKV.values[key] = []byte("x") - p.w.put(buckets.Asks(), key, []byte("x")) - } - p.lift(context.Background(), "web", fpOld) - for key, gone := range map[string]bool{restOld: true, mustKey(asks.RestoreAskKey("ace", "web", fpOld)): true, keep: false, keepOther: false} { - if _, there := asksKV.values[key]; there == gone { - t.Errorf("%s: left in the bucket = %v after the lift", key, there) - } - } - }) } func mustKey(key string, err error) string { @@ -1216,7 +721,7 @@ func aReleasesApproveAnswersTheMoveOfItsBuildAlone(t *testing.T) { have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fpOld, Composed: was, Phase: nodeengineclient.PhaseWaitingYes, Health: nodeengineclient.HealthOK, Judgement: assign.JudgedRecreates, Waits: waiting, Detail: "it recreates web.server", BuildAlone: true}) p.w.put(buckets.Have, "g14.web", have) - m := assign.Module{Name: "web", FP: fpTest, Nodes: []string{"g14"}, Self: func(string) bool { return true }} + m := assign.Module{Name: "web", Nodes: []string{"g14"}, Self: func(string) bool { return true }} out := assign.Outcome{Wants: map[string]mwc.Want{"g14": {Fp: fpTest, Self: true}}} p.yeses(context.Background(), "web", m, map[string]mwc.Want{"g14": {Fp: fpOld, Self: true}}, &out) if w := out.Wants["g14"]; w.Yes != "" || w.Release != "" { @@ -1254,7 +759,11 @@ func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) { p.logf = t.Logf put := func(bucket, key string, v []byte) { p.w.put(bucket, key, v) } mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) - putModuleRaw(p.w, trunkBranch, "web", mod) // the trunk's build is not done: rule 1 holds g14 at fpOld + putModuleRaw(p.w, trunkBranch, "web", mod) + // The newest version passed is fpOld's (design 47 H §750): the trunk's newer build passed no pipeline. + p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf) + p.ends.trunk = p.w.trunkName + _, _ = p.ends.take(ended(trunkBranch, 1, mpr.PipelineEndedOutcomePassed, mover("web", fpOld, "0.1.1", 1, mpr.StepResultOutcomePassed))) placeOn(t, p.w, "g14", "web") pl, _ := mmac.EncodeAssignments(mma.Assignments{Node: "g14", Modules: []mma.PlacedModule{{Module: "web"}}}) put(plc.AssignmentsBucket(), "assignments.g14", pl) @@ -1263,7 +772,7 @@ func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) { have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fpOld, Composed: was, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) put(buckets.Have, "g14.web", have) wkey, _ := mwc.WantKey("g14", "web") - want, _ := mwc.EncodeWant(mwc.Want{Fp: fpOld, Composed: was}) + want, _ := mwc.EncodeWant(mwc.Want{Fp: fpOld, Version: "0.1.1", Pipeline: trunkBranch + ".1", Rank: 1}) wantsKV.values[wkey] = want put(buckets.Wants(), wkey, want) sync := func() { @@ -1276,7 +785,7 @@ func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) { p.nodeWants(context.Background()) sync() got, err := mwc.DecodeWant(wantsKV.values[wkey]) - if err != nil || got.Fp != fpOld || got.Composed != "" || !got.Self { + if err != nil || got.Fp != fpOld || !got.Self { t.Fatalf("the placed node's want is %+v (%v), want it at %s, the node composing it", got, err, fpOld) } nkey, _ := mwc.WantNodeKey("g14") @@ -1392,92 +901,6 @@ func TestAnApprovedAssignmentsCallsNoControllerVerb(t *testing.T) { } } -// ADR 0614 §2, §3: of what a module invokes, the pass keeps as edges the verbs the graph does not leave out and another -// module serves; a verb is served once a running healthy build's manifest (the builder's build, the indexer's manifest) -// serves it, and a build whose manifest cannot be read counts what the graph says its module serves. No graph read: no wait. -func aWantWaitsOnTheVerbEdgesTheGraphKeeps(t *testing.T) { - p := newPasser(kvSet{}, time.Now()) - fp := func(c string) string { return meshcatalogclient.SourcePrefix2 + strings.Repeat(c, 64) } - if p.served(graph.Graph{}, assign.Nodes{}) != nil { - t.Error("with no graph read, a want waits on a verb") - } - for module, n := range map[string]graph.Node{ - "caller": {FP: fp("c"), Repository: "novox/caller", Invokes: []string{"seat:x.v", "loop.l", "none.t", "caller.own"}, LeftOut: []string{"loop.l"}, - Serves: []string{"caller.own"}}, - "server": {FP: fp("e"), Repository: "novox/server", Serves: []string{"seat:x.v", "seat:x.w"}}, - "loop": {FP: fp("1"), Repository: "novox/loop", Serves: []string{"loop.l"}}, - "late": {FP: fp("2"), Repository: "novox/late", Serves: []string{"late.t"}}, - } { - raw, err := graph.Encode(n) - if err != nil { - t.Fatal(err) - } - _ = raw - p.w.putModule(trunkBranch, module, meshcatalog.ModulesByModuleValue{}, n, nil) - } - g := p.moduleGraph(trunkBranch) - if got := keptInvokes(g, "caller"); !slices.Equal(got, []string{"seat:x.v"}) || keptInvokes(g, "unknown") != nil { - t.Errorf("the kept verb edges of caller = %v", got) - } - digest := strings.Repeat("d", 64) - parsedServer, err := catalogue.Decode([]byte(`{"module":"server","claims":[{"name":"x","scope":"mesh","serves":["w"]}]}`)) - if err != nil { - t.Fatal(err) - } - p.w.manifests[digest] = parsed{module: "server", manifest: parsedServer} - endDone(t, p.w, "server", fp("f"), digest) - running := nodeengine.HaveByNodeModuleValue{Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - at := func(f string) nodeengine.HaveByNodeModuleValue { h := running; h.Fp = f; return h } - served := p.served(g, assign.Nodes{Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{ - "g14": {"server": at(fp("f")), "late": at(fp("2"))}}}) - for v, want := range map[string]bool{"seat:x.w": true, "seat:x.v": false, "late.t": true} { - if served(v) != want { - t.Errorf("served(%s) = %v, want %v: the running server's manifest serves w alone, late's is not read", v, served(v), want) - } - } - // The modules whose running build's manifest is not read are said once per change of their set (G-23). - var said []string - p.logf = func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) } - lateOnly := assign.Nodes{Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"g14": {"server": at(fp("f")), "late": at(fp("2"))}}} - p.served(g, lateOnly) - p.served(g, lateOnly) - p.served(g, assign.Nodes{Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"g14": {"late": at(fp("2")), "loop": at(fp("1"))}}}) - p.served(g, lateOnly) - if len(said) != 2 || !strings.Contains(said[0], "[loop late]") && !strings.Contains(said[0], "[late loop]") || !strings.Contains(said[1], "[late]") { - t.Errorf("the unread manifests were said %q, want once per change of the set", said) - } - // A-426a: a healthy build held (no catalog entry of it) or waiting for a yes serves what it serves, said once; one held unhealthy, - // or whose running statement says it stopped, does not. - held := at(fp("2")) - held.Phase = nodeengineclient.PhaseHeld - served = p.served(g, assign.Nodes{Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"novox": {"late": held}}}) - if !served("late.t") { - t.Error("a healthy held build of late does not serve late.t") - } - stopped := assign.Nodes{Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"novox": {"late": held}}, - Running: map[string]map[string]nodeengine.Running{"novox": {"late": {Stopped: true}}}} - sick := held - sick.Health = "failed" - if p.served(g, stopped)("late.t") || p.served(g, assign.Nodes{Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"novox": {"late": sick}}})("late.t") { - t.Error("a held build stopped, or not healthy, serves late.t") - } - // ADR 0614 §5: bus-state counts as offered only from a healthy build on a node present; the bus module's build on an - // offline node offers nothing, and one whose manifest is not read counts its trunk manifest's offer. A-352: a have waiting - // for a person's yes on a newer build names the build that still runs, healthy, and it offers bus-state too. - bus := map[string]catalogue.Manifest{"bus": {Module: "bus", Provides: []catalogue.Offer{{Name: graph.BusState, Scope: catalogue.ScopeMesh}}}} - for _, online := range []bool{false, true} { - for _, phase := range []string{nodeengineclient.PhaseRunning, nodeengineclient.PhaseWaitingYes} { - h := at(fp("2")) - h.Phase = phase - offering := p.offering(bus, assign.Nodes{Online: map[string]bool{"n2": online}, - Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"n2": {"bus": h}}}) - if offering(graph.BusState) != online { - t.Errorf("with the bus module's node online %v in phase %s, bus-state offered %v", online, phase, offering(graph.BusState)) - } - } - } -} - // decodedAsk is an asks-state value with the router's ask it carries decoded (mesh-sdk keeps it as raw JSON). type decodedAsk struct { Asking int @@ -1567,20 +990,10 @@ func aRecordAskIsNotTakenAwayBeforeItsYesIsWrittenAndIsAskedAgainWhenWrittenAgai p.recordAsk(context.Background(), "web", fp, assign.RecordNeed{Node: "novox", Digest: dig}) return readAskID(t, p, rkey) } - rest, _ := asks.RestoreAskKey("novox", "web", fp) - restore := func(p *passer) string { - p.restoreAsk(context.Background(), "web", "novox", fp) - return readAskID(t, p, rest) - } - hold, _ := asks.HoldAskKey("web", fp) - holdIt := func(p *passer) string { - p.holdAsk(context.Background(), "web", fp, &mwc.Held{Node: "novox", Words: "w"}) - return readAskID(t, p, hold) - } for name, c := range map[string]struct { write func(*passer) string key string - }{"record": {rec, rkey}, "restore": {restore, rest}, "hold": {holdIt, hold}} { + }{"record": {rec, rkey}} { at := time.Now() asksKV := &memKV{values: map[string][]byte{}} p := newPasser(kvSet{buckets.Asks(): asksKV, buckets.Wants(): &memKV{values: map[string][]byte{}}}, at) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pins.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pins.go deleted file mode 100644 index 7b437b80..00000000 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pins.go +++ /dev/null @@ -1,539 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "slices" - "sort" - "strconv" - "strings" - "sync" - "time" - - "github.com/nats-io/nats.go/jetstream" - - "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" - - mdseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" - stdio "git.novox.be/novox/mesh-sdk/go" - "git.novox.be/novox/mesh-sdk/go/process" - - "mesh-delivery-assigner/internal/assign" - "mesh-delivery-assigner/internal/assigngraph" - "mesh-delivery-assigner/internal/assignment" -) - -// The pins of the assignment graph revisions (design 47 G §153, §178, §216, §579; A-169): the gate says on the seat -// mesh-delivery which revisions a delivery record starts (`pinned`) and stops (`freed`) pointing at, and once at each of its -// starts `pins-stated`. The assigner hears them on its durable consumer of the events stream and keeps, in its old `state` -// section `pins` (4 MiB), one key `trunk.` per assignment graph revision whose value is the list of the records pinning -// it, added or removed by compare-and-set before the event is acknowledged; a record heard twice pins once. Two more keys: -// `made`, written once when the state is first used, and `pins-stated`, the `at` of the last pins-stated applied. A revision -// is removed, its object and its key and the entry objects no kept revision lists and no want or running build names -// (§179), when its list is empty and it is not current (the newest the assigner wrote), only while the consumer is caught up -// and once a pins-stated said after `made` was applied; judged on a freed, on a revision ceasing to be current, on being -// caught up, and at start once both hold. Never on a timer, and never by reading the gate's state. - -const ( - pinsState = "pins" - pinsTool = "assigner_pins" - unpinTool = "assigner_unpin" - pinsMadeKey = "made" - pinsStatedKey = "pins-stated" - pinsCapBytes = 4 << 20 - pinMissingKind = "pin-missing" -) - -// PinsValue is one value of the state pins: the records that pin the revision, and whether the revision was no longer held -// when a record pinned it. -type PinsValue struct { - Records []string `json:"records"` - Missing bool `json:"missing,omitempty"` -} - -// pinKey is the key of an assignment graph revision, its object's own name (A-169). -func pinKey(revision int64) string { return assigngraph.TrunkObject(revision) } - -// pinRevision is the revision a key of the state names, and whether it names one. -func pinRevision(key string) (int64, bool) { - n, ok := strings.CutPrefix(key, "trunk.") - if !ok { - return 0, false - } - r, err := strconv.ParseInt(n, 10, 64) - return r, err == nil && r > 0 -} - -// pinObjects is the object bucket of the revisions as the pins remove from it. -type pinObjects interface { - assigngraph.Objects - Delete(ctx context.Context, name string) error -} - -// pinKeeper keeps the pins and removes what nothing pins. -type pinKeeper struct { - ctx context.Context - st endStore - objects pinObjects - // inUse says a want or a running build names the module's fingerprint (§179). - inUse func(module, fp string) bool - findings assign.Findings - now func() time.Time - logf func(string, ...any) - mu sync.Mutex - // caught is whether the last event applied left none of the consumer's messages pending. - caught bool -} - -// edit applies change to the value at key by compare-and-set, again while another write comes between. -func (k *pinKeeper) edit(key string, change func(v PinsValue, found bool) (PinsValue, bool)) error { - for { - raw, rev, err := k.st.get(key) - if err != nil { - return err - } - var v PinsValue - if raw != nil { - if err := json.Unmarshal(raw, &v); err != nil { - return fmt.Errorf("the pins of %s do not read: %w", key, err) - } - } - next, write := change(v, raw != nil) - if !write { - return nil - } - value, _ := json.Marshal(next) - if err := k.st.swap(key, value, rev); errors.Is(err, errSwapped) { - continue - } else { - return err - } - } -} - -// held says the revision's object is kept. -func (k *pinKeeper) held(revision int64) (bool, error) { - _, err := k.objects.Get(k.ctx, assigngraph.TrunkObject(revision)) - if errors.Is(err, assigngraph.ErrNoObject) { - return false, nil - } - return err == nil, err -} - -// pin adds record to each assignment graph revision of p; a revision no longer held is written missing, with the finding -// pin-missing naming the record. -func (k *pinKeeper) pin(p mdseat.Pins) error { - for _, r := range p.Assignment { - held, err := k.held(r) - if err != nil { - return err - } - err = k.edit(pinKey(r), func(v PinsValue, _ bool) (PinsValue, bool) { - if slices.Contains(v.Records, p.Record) && (held || v.Missing) { - return v, false - } - if !slices.Contains(v.Records, p.Record) { - v.Records = append(v.Records, p.Record) - slices.Sort(v.Records) - } - v.Missing = v.Missing || !held - return v, true - }) - if err != nil { - return err - } - if !held { - k.missing(r, p.Record) - } - } - return nil -} - -// free removes record from each assignment graph revision of p; a freed applied twice changes nothing. -func (k *pinKeeper) free(p mdseat.Pins) error { - for _, r := range p.Assignment { - err := k.edit(pinKey(r), func(v PinsValue, found bool) (PinsValue, bool) { - i := slices.Index(v.Records, p.Record) - if !found || i < 0 { - return v, false - } - v.Records = slices.Delete(v.Records, i, i+1) - return v, true - }) - if err != nil { - return err - } - } - return nil -} - -// missing writes the finding pin-missing, scope the assigner, id the revision. -func (k *pinKeeper) missing(revision int64, record string) { - if k.findings.Put == nil { - k.logf("the revision %d the record %s pins is not held, and no finding writer is wired to say so", revision, record) - return - } - o := assign.Observed{Scope: assignment.Module(), ID: strconv.FormatInt(revision, 10), Kind: pinMissingKind, - Summary: fmt.Sprintf("the delivery record %s pins the assignment graph revision %d, which the assigner no longer holds", record, revision), - Headline: "A delivery record pins a revision the assigner no longer holds", - Explanation: fmt.Sprintf("The record %s pinned revision %d after it was removed; %s clears it.", record, revision, unpinTool), - Source: assignment.Module()} - if err := k.findings.Put(o.Finding(k.now())); err != nil { - k.logf("the finding %s of revision %d is not written: %v", pinMissingKind, revision, err) - } -} - -// made writes the key made once, create-only; one there already stays. -func (k *pinKeeper) made() error { - raw, _, err := k.st.get(pinsMadeKey) - if err != nil || raw != nil { - return err - } - value, _ := json.Marshal(k.now().UTC()) - if err := k.st.swap(pinsMadeKey, value, 0); err != nil && !errors.Is(err, errSwapped) { - return err - } - return nil -} - -// timeAt is the time a key holds; false when it holds none. -func (k *pinKeeper) timeAt(key string) (time.Time, bool, error) { - raw, _, err := k.st.get(key) - if err != nil || raw == nil { - return time.Time{}, false, err - } - var t time.Time - if err := json.Unmarshal(raw, &t); err != nil { - return t, false, fmt.Errorf("the key %s does not read: %w", key, err) - } - return t, true, nil -} - -// stated keeps the at of a pins-stated applied. -func (k *pinKeeper) stated(at time.Time) error { - for { - raw, rev, err := k.st.get(pinsStatedKey) - if err != nil { - return err - } - var was time.Time - if raw != nil && json.Unmarshal(raw, &was) == nil && !at.After(was) { - return nil - } - value, _ := json.Marshal(at.UTC()) - if err := k.st.swap(pinsStatedKey, value, rev); errors.Is(err, errSwapped) { - continue - } else { - return err - } - } -} - -// statedSinceMade says a pins-stated said after the state was made was applied. -func (k *pinKeeper) statedSinceMade() (bool, error) { - made, ok, err := k.timeAt(pinsMadeKey) - if err != nil || !ok { - return false, err - } - at, ok, err := k.timeAt(pinsStatedKey) - if err != nil || !ok { - return false, err - } - return at.After(made), nil -} - -// hear applies one of the seat mesh-delivery's events and acknowledges it only once applied. -func (k *pinKeeper) hear(m jetstream.Msg) { - err := k.apply(m.Subject(), m.Data()) - if err != nil { - k.logf("%s: %v; it is heard again", m.Subject(), err) - _ = m.Nak() - return - } - _ = m.Ack() - caught := false - if meta, merr := m.Metadata(); merr == nil { - caught = meta.NumPending == 0 - } - k.mu.Lock() - was := k.caught - k.caught = caught - k.mu.Unlock() - if caught && (!was || m.Subject() != mdseat.SubjectPinned) { - k.judge() // on a freed, on a candidate's heads replaced, on pins-stated, and on being caught up - } -} - -// apply takes one event's body. -func (k *pinKeeper) apply(subject string, data []byte) error { - switch subject { - case mdseat.SubjectPinned, mdseat.SubjectFreed: - var p mdseat.Pins - if err := json.Unmarshal(data, &p); err != nil || p.Record == "" { - k.logf("%s holds no pins naming a record, and is not taken: %v", subject, err) - return nil - } - if subject == mdseat.SubjectPinned { - return k.pin(p) - } - return k.free(p) - case mdseat.SubjectPinsStated: - var s mdseat.PinsStatedEvent - if err := json.Unmarshal(data, &s); err != nil || s.At.IsZero() { - k.logf("%s holds no at, and is not taken: %v", subject, err) - return nil - } - return k.stated(s.At) - } - return nil // heads-replaced: a candidate's revision stops being current, judged after -} - -// judge removes every revision neither pinned nor current, when the rule allows a removal at all. -func (k *pinKeeper) judge() { - k.mu.Lock() - caught := k.caught - k.mu.Unlock() - if !caught { - return - } - ok, err := k.statedSinceMade() - if err != nil { - k.logf("the pins are not judged: %v", err) - return - } - if !ok { - return // no pins-stated applied since the state was made: a pin said before may not be heard yet - } - if err := k.remove(); err != nil { - k.logf("the revisions nothing pins are not removed: %v", err) - } -} - -// remove removes each kept revision that is not current and that no record pins, and then the entries left unlisted. -func (k *pinKeeper) remove() error { - names, err := k.objects.Names(k.ctx) - if err != nil { - return err - } - var kept []int64 - for _, n := range names { - if r, ok := pinRevision(n); ok { - kept = append(kept, r) - } - } - if len(kept) == 0 { - return nil - } - current := slices.Max(kept) - listed := map[string]bool{} - removed := 0 - for _, r := range kept { - raw, _, err := k.st.get(pinKey(r)) - if err != nil { - return err - } - var v PinsValue - if raw != nil { - if err := json.Unmarshal(raw, &v); err != nil { - return fmt.Errorf("the pins of %d do not read: %w", r, err) - } - } - if r == current || len(v.Records) > 0 { - rev, err := assigngraph.ReadRevision(k.ctx, k.objects, r) - if err != nil { - return err - } - for _, e := range rev.Entries { - listed[e.Digest] = true - } - continue - } - if err := k.objects.Delete(k.ctx, assigngraph.TrunkObject(r)); err != nil { - return fmt.Errorf("revision %d: %w", r, err) - } - if raw != nil { - if err := k.st.drop(pinKey(r)); err != nil { - return fmt.Errorf("the pins of %d: %w", r, err) - } - } - removed++ - } - if removed == 0 { - return nil - } - // A run's candidate kept now lists its entries too, until its group-ended drops it (A-201, A-235). - for _, n := range names { - group, heads, ok := meshwants.ParseAssignmentRevisionsByCandidateGroupHeadsKey(n) - if !ok { - continue - } - rev, err := assigngraph.ReadCandidate(k.ctx, k.objects, group, heads) - if err != nil { - return err - } - for _, e := range rev.Entries { - listed[e.Digest] = true - } - } - for _, n := range names { - digest, ok := strings.CutPrefix(n, "entry.") - if !ok || listed[digest] { - continue - } - e, err := assigngraph.ReadEntry(k.ctx, k.objects, digest) - if err != nil { - return err - } - if k.inUse != nil && k.inUse(e.Module, e.Fp) { - continue - } - if err := k.objects.Delete(k.ctx, n); err != nil { - return fmt.Errorf("the entry %s: %w", digest, err) - } - } - k.logf("%d assignment graph revisions no record pins and not current are removed", removed) - return nil -} - -// hearPins registers the seat mesh-delivery's four events on the assigner's hub. -func hearPins(hub *process.EventHub, k *pinKeeper) error { - for _, s := range []string{mdseat.SubjectPinned, mdseat.SubjectFreed, mdseat.SubjectHeadsReplaced, mdseat.SubjectPinsStated} { - if err := hub.Handle(s, k.hear); err != nil { - return fmt.Errorf("%s: %w", s, err) - } - } - return nil -} - -// pinsTools are assigner_pins and assigner_unpin over the runtime's states. -func pinsTools(st store, ask func(string, any) (json.RawMessage, error)) []stdio.Tool { - return []stdio.Tool{{Name: pinsTool, Description: "the assignment graph revisions the delivery records pin (design 47 G §579): per revision " + - "the records that pin it and whether it was missing when pinned, the newest revision kept (current, never removed), when the state " + - "was made, the last pins-stated applied, and the state's fill against its cap of 4 MiB.", - Run: func(map[string]any) (any, error) { return readPins(st) }}, - {Name: unpinTool, Description: "Remove one record from a revision's pins, for a freed that was lost (design 47 G §579); written in the " + - "hand-act log first. The removal rule then applies.", - Input: schema(map[string]string{"revision": "the assignment graph revision's number", "record": "the delivery record's id", - "why": "why it is removed by hand"}, []string{"revision", "record", "why"}), - Run: func(a map[string]any) (any, error) { - record, _ := a["record"].(string) - why, _ := a["why"].(string) - return unpin(st, ask, fmt.Sprint(a["revision"]), record, why, stdio.CallerOf(a)) - }}} -} - -// readPins is what assigner_pins answers. -func readPins(st store) (any, error) { - ks, err := st.keys(pinsState) - if err != nil { - return nil, err - } - sort.Strings(ks) - revisions := map[string]PinsValue{} - out := map[string]any{"revisions": revisions} - var newest int64 - bytes := 0 - for _, key := range ks { - raw, _, found, err := st.get(pinsState, key) - if err != nil || !found { - continue - } - bytes += len(key) + len(raw) - if r, ok := pinRevision(key); ok { - var v PinsValue - if json.Unmarshal(raw, &v) == nil { - revisions[key] = v - } - newest = max(newest, r) - continue - } - out[key] = json.RawMessage(raw) - } - out["fill"] = meshwants.Fill{Store: pinsState, Bytes: int64(bytes), Cap: pinsCapBytes} - if newest > 0 { - out["newest-pinned"] = newest - } - return out, nil -} - -// unpin removes one record from one revision's list, the hand act recorded first. -func unpin(st store, ask func(string, any) (json.RawMessage, error), revision, record, why, by string) (any, error) { - r, err := strconv.ParseInt(strings.TrimPrefix(revision, "trunk."), 10, 64) - switch { - case err != nil || r <= 0: - return nil, fmt.Errorf("revision %q is no assignment graph revision's number", revision) - case record == "" || why == "": - return nil, errors.New("name the record and say why") - case by == "" || by == stdio.Unproven: - return nil, errors.New("an unpin names who asks it, and the runtime proved no caller") - } - key := pinKey(r) - raw, _, found, err := st.get(pinsState, key) - if err != nil { - return nil, err - } - var v PinsValue - if found { - if err := json.Unmarshal(raw, &v); err != nil { - return nil, err - } - } - i := slices.Index(v.Records, record) - if i < 0 { - return nil, fmt.Errorf("the record %s does not pin revision %d: %s lists its pins", record, r, pinsTool) - } - taken, err := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: unpinTool + " " + key + " " + record, - Why: why + " (" + by + ")", Cause: "unpin"}) - if err != nil { - return nil, fmt.Errorf("the hand act is not recorded, so nothing is removed: %w", err) - } - v.Records = slices.Delete(v.Records, i, i+1) - value, _ := json.Marshal(v) - if err := st.put(pinsState, key, value); err != nil { - return nil, err - } - return map[string]any{"revision": r, "unpinned": record, "records": v.Records, "hand-act": taken.Log}, nil -} - -// watchUnpins judges again when a revision's pins change in the state itself, as assigner_unpin writes them from the tool -// process: the assigner's own state, heard as it changes, never on a timer. -func (k *pinKeeper) watchUnpins(kv jetstream.KeyValue) { - w, err := kv.WatchAll(k.ctx, jetstream.UpdatesOnly()) - if err != nil { - k.logf("the state %s is not watched, so a hand unpin is judged at the next freed or revision: %v", pinsState, err) - return - } - go func() { - defer func() { _ = w.Stop() }() - for e := range w.Updates() { - if e == nil { - continue - } - if _, ok := pinRevision(e.Key()); ok && e.Operation() == jetstream.KeyValuePut { - k.judge() - } - } - }() -} - -// inUseOf says a want or a running build names the module's fingerprint, as the world holds them now. -func inUseOf(w *world) func(module, fp string) bool { - return func(module, fp string) bool { - for _, want := range w.wantsOf()[module] { - if want.Fp == fp { - return true - } - } - for _, mods := range w.runningNow() { - if r, ok := mods[module]; ok && r.Fingerprint == fp { - return true - } - } - return false - } -} - -func pinsBucket() string { return assignment.Module() + "_" + pinsState } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pins_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pins_test.go deleted file mode 100644 index de2a17d8..00000000 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pins_test.go +++ /dev/null @@ -1,119 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "testing" - "time" - - mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" - mdseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" - - "mesh-delivery-assigner/internal/assign" - "mesh-delivery-assigner/internal/assigngraph" -) - -func (m memObjects) Delete(_ context.Context, name string) error { delete(m, name); return nil } - -// The pins of the assignment graph revisions (design 47 G §153, §178, §579; A-169): a revision two records pin stays until -// both free it; a record pinning twice pins once; the current revision is never removed; nothing is removed before a -// pins-stated said after the state was made, nor while the consumer is behind; a revision no longer held is pinned missing, -// with the finding pin-missing; an entry another kept revision lists, or a want or a running build names, stays. -func TestPinsKeepARevisionWhileARecordPinsIt(t *testing.T) { - objects := memObjects{} - entry := func(module, fp string) string { - e := meshwants.AssignmentEntry{Node: "n1", Module: module, Fp: fp} - d := assigngraph.EntryDigest(e) - raw, _ := json.Marshal(e) - objects[assigngraph.EntryObject(d)] = raw - return d - } - shared, old, running := entry("a", "fp-a"), entry("b", "fp-b1"), entry("c", "fp-c1") - revision := func(n int64, digests ...string) { - rev := meshwants.AssignmentRevision{Revision: n} - for _, d := range digests { - rev.Entries = append(rev.Entries, meshwants.NodeEntryRef{Node: "n1", Digest: d}) - } - raw, _ := json.Marshal(rev) - objects[assigngraph.TrunkObject(n)] = raw - } - // A run's candidate lists an entry no trunk revision lists (A-201): it stays while the candidate is kept. - cand := entry("d", "fp-d2") - candRaw, _ := json.Marshal(meshwants.AssignmentRevision{Revision: 1, Kind: "candidate", Group: "run-7", Heads: "h1", - Entries: []meshwants.NodeEntryRef{{Node: "n1", Module: "d", Digest: cand}}}) - objects[assigngraph.CandidateObject("run-7", "h1")] = candRaw - revision(1, shared, old, running) - revision(2, shared, old) - revision(3, shared) - var found []mcseat.Finding - clock := time.Date(2026, 11, 12, 10, 0, 0, 0, time.UTC) - k := &pinKeeper{ctx: context.Background(), st: newMemEnds(), objects: objects, now: func() time.Time { return clock }, - logf: t.Logf, - inUse: func(module, fp string) bool { return module == "c" && fp == "fp-c1" }, - findings: assign.Findings{Put: func(f mcseat.Finding) error { found = append(found, f); return nil }}} - if err := k.made(); err != nil { - t.Fatal(err) - } - say := func(subject string, body any) { - t.Helper() - raw, _ := json.Marshal(body) - if err := k.apply(subject, raw); err != nil { - t.Fatal(err) - } - k.judge() - } - pins := func(record string, revs ...int64) mdseat.Pins { return mdseat.Pins{Record: record, Assignment: revs} } - has := func(n int64) bool { _, ok := objects[assigngraph.TrunkObject(n)]; return ok } - - k.caught = true - say(mdseat.SubjectPinned, pins("r1", 1)) - say(mdseat.SubjectPinned, pins("r1", 1)) - say(mdseat.SubjectPinned, pins("r2", 1)) - say(mdseat.SubjectFreed, pins("r9", 2)) // nothing pins 2, but no pins-stated since made: nothing removed - if !has(2) { - t.Fatal("revision 2 was removed before a pins-stated said after the state was made") - } - k.caught = false - say(mdseat.SubjectPinsStated, mdseat.PinsStatedEvent{At: clock.Add(time.Minute)}) - if !has(2) { - t.Fatal("revision 2 was removed while the consumer was behind") - } - k.caught = true - say(mdseat.SubjectFreed, pins("r1", 1)) - if has(2) { - t.Error("revision 2, neither pinned nor current, is kept") - } - if !has(1) || !has(3) { - t.Error("revision 1 (pinned by r2) or 3 (current) was removed") - } - raw, _, _ := k.st.get(pinKey(1)) - if string(raw) != `{"records":["r2"]}` { - t.Errorf("the pins of 1 = %s, want r2 alone (r1 pinned twice pins once, and freed once)", raw) - } - say(mdseat.SubjectFreed, pins("r2", 1)) - if has(1) { - t.Error("revision 1 is kept once both records freed it") - } - if _, ok := objects[assigngraph.EntryObject(old)]; ok { - t.Error("the entry no kept revision lists and nothing names is kept") - } - if _, ok := objects[assigngraph.EntryObject(running)]; !ok { - t.Error("the entry of a running build is removed") - } - if _, ok := objects[assigngraph.EntryObject(shared)]; !ok { - t.Error("the entry the current revision lists is removed") - } - if _, ok := objects[assigngraph.EntryObject(cand)]; !ok { - t.Error("the entry a run's kept candidate lists is removed") - } - say(mdseat.SubjectHeadsReplaced, mdseat.HeadsReplacedEvent{}) - if _, ok := objects[assigngraph.CandidateObject("run-7", "h1")]; !ok { - t.Error("the gate's heads-replaced dropped a run's candidate") - } - say(mdseat.SubjectPinned, pins("r3", 2)) - raw, _, _ = k.st.get(pinKey(2)) - if string(raw) != `{"records":["r3"],"missing":true}` || len(found) != 1 || found[0].Kind != pinMissingKind || found[0].ID != "2" { - t.Errorf("a pin of a revision no longer held = %s, findings %+v", raw, found) - } -} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go index 5baf09ba..8e93cecb 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go @@ -16,18 +16,20 @@ import ( mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" mprc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient" - mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient" mpcat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/keys" "git.novox.be/novox/mesh-sdk/go/process" + "mesh-delivery-assigner/internal/assign" ) -// The wants follow a passed pipeline (design 47 G §29, §35, §325, §352): the runner says each pipeline's end as -// pipeline-ended, and a group's as group-ended. The assigner keeps, in its old state `written`, one key per module and -// watched branch: the source revision and end count of the newest end it heard for it, and whether the wants were written -// from it. A newer end replaces it by compare-and-set; an older one, or the same one again, writes nothing. +// The wants follow a passed pipeline (design 47 H §750): the runner says each pipeline's end once, per source revision, as +// pipeline-ended on `pipeline-ended..`, naming per mover its fingerprint, its released version and its tier +// (§746). The assigner keeps, in its state `written` (G §352), one key per module and watched branch: the newest released +// version of the module on that branch whose pipeline passed, with the pipeline and the mover's rank in it. A newer version +// replaces it by compare-and-set; an older one, the same one again, a failed pipeline and a run on request write nothing. // writtenState is the assigner's state of the pipeline ends its wants follow; writtenTool reads it. const ( @@ -35,23 +37,17 @@ const ( writtenTool = "assigner_written" ) -// pipelineEndedSubject and groupEndedSubject are the runner's events the assigner consumes. -const ( - pipelineEndedSubject = "mesh.seat." + mpr.Seat + ".event." + mpr.EventPipelineEnded + ".>" - groupEndedSubject = "mesh.seat." + mpr.Seat + ".event." + mpr.EventGroupEnded + ".>" -) +// pipelineEndedSubject is the runner's event the assigner consumes. +const pipelineEndedSubject = "mesh.seat." + mpr.Seat + ".event." + mpr.EventPipelineEnded + ".>" -// End is one value of the state written: the newest end of a module's pipeline on a watched branch. +// End is one value of the state written: the newest version of a module released on a watched branch whose pipeline passed. type End struct { + Pipeline string `json:"pipeline"` Source int64 `json:"source"` - EndCount int64 `json:"end-count"` Fp string `json:"fp"` - Commit string `json:"commit,omitempty"` - // Passed: the pipeline ended passed, every required step passed (and, for a group, the group ended passed). - Passed bool `json:"passed"` - // Written: the wants were written from this end. - Written bool `json:"written,omitempty"` - Why string `json:"why,omitempty"` + Version string `json:"version"` + // Rank is the mover's tier in the pipeline's delivery order, from 1 (§746, §750). + Rank int64 `json:"rank"` } // endKey is the key of a module on a watched branch, each name escaped as a key's name is. @@ -59,15 +55,10 @@ func endKey(module, branch string) string { return keys.EscapeName(module) + "." + keys.EscapeName(branch) } -// newer says whether an end of (source, count) replaces the one kept: a later source revision, or the same one ended again. -func (e End) newer(source, count int64) bool { - return source > e.Source || (source == e.Source && count > e.EndCount) -} - -// requiredPassed says whether no required step of a pipeline stands failed, unasked or unended (design 47 G §30: a failed -// optional step holds nothing). -func requiredPassed(p mpr.PipelineEnded) bool { - for _, s := range p.Steps { +// moverPassed says whether no required step of a mover stands failed, unasked or unended: a required step ending "nothing to +// run" counts as passed, as a bypassed one does (design 47 H §746, H-A-3); a failed optional step holds nothing (G §30). +func moverPassed(m mpr.MoverEnded) bool { + for _, s := range m.Steps { if s.Status != mpr.StepResultStatusRequired { continue } @@ -78,7 +69,7 @@ func requiredPassed(p mpr.PipelineEnded) bool { return false } } - return p.Outcome == mpr.PipelineEndedOutcomePassed + return true } // endStore is the kept state written: the bus bucket, or a test's. @@ -139,19 +130,19 @@ func isWrongSequence(err error) bool { return errors.As(err, &ae) && ae.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence } -// pipelineEnds follows the runner's ends. A nil *pipelineEnds is the trunk-only want, kept until the first registrations -// (design 47 G §95, §353). +// pipelineEnds follows the runner's ends; a nil *pipelineEnds keeps none, and no want is written from it. type pipelineEnds struct { st endStore wake func() logf func(string, ...any) - // on is whether the wants follow the ends: set by the first registration heard, never before (design 47 G §95, §353). + // on says the first registration was heard (design 47 G §353), said once in the journal. on atomic.Bool mu sync.Mutex // firsts is, by module, the first watched branch of its registration, the branch of a placement that names none (design // 47 G §14). firsts map[string]string - // runEnded is told a run on request's group-ended, its group `run-`, to drop the run's candidate (A-235); nil drops nothing. + // runEnded is told a run on request's pipeline-ended, its group `run-`, to drop the run's candidate (A-235, H-A-58); nil + // drops nothing. runEnded func(group string) error // left is told the key of a pipeline ended superseded or cancelled, whose composition steps leave the runs they read // (design 47 G §723, A-469n); none tells nothing. Set once the composition steps are served, after the hub starts. @@ -160,110 +151,121 @@ type pipelineEnds struct { onBranches func([]string) // trunk names the first watched branch, the trunk's (design 47 G §507), the branch of a module registered with none. trunk func() string - // runner is the module holding the seat mesh-pipeline-runner, as the last pass read it from the acting edges (A-334); nil - // until a pass read the graph. - runner atomic.Pointer[string] - // held are the group members' pipeline-ended heard before a pass read the graph, by their body: unacknowledged, so the - // bus keeps them, and taken once the runner is read (A-334), never dropped for the runner not being known yet. - held map[string]jetstream.Msg -} - -// setRunner keeps the module holding the seat mesh-pipeline-runner, read by the pass from the acting edges (A-334); the -// first one takes the group members' ends held until the graph was read. -func (e *pipelineEnds) setRunner(module string) { - e.mu.Lock() - first := e.runner.Swap(&module) == nil - held := e.held - e.held = nil - e.mu.Unlock() - if !first { - return - } - for _, m := range held { - e.hearPipeline(m) - } -} - -// holdUntilRead keeps m, a group member's pipeline-ended, until a pass reads the runner from the graph; false when the -// runner is read already, so m is taken now. -func (e *pipelineEnds) holdUntilRead(m jetstream.Msg) bool { - e.mu.Lock() - defer e.mu.Unlock() - if e.runner.Load() != nil { - return false - } - if e.held == nil { - e.held = map[string]jetstream.Msg{} - } - e.held[string(m.Data())] = m - e.logf("%s is a group member's end heard before the assignment graph is read: held until a pass reads which module is the runner (A-334)", m.Subject()) - return true -} - -// isRunner says whether module holds the seat mesh-pipeline-runner, as the last pass read it. -func (e *pipelineEnds) isRunner(module string) bool { - r := e.runner.Load() - return r != nil && *r != "" && *r == module } func newPipelineEnds(st endStore, wake func(), logf func(string, ...any)) *pipelineEnds { return &pipelineEnds{st: st, wake: wake, logf: logf, firsts: map[string]string{}} } -// take keeps one pipeline's end if it is newer than the one kept for its module and branch, by compare-and-set. -func (e *pipelineEnds) take(p mpr.PipelineEnded, passed bool) error { - key := endKey(p.Module, p.Branch) - for { - raw, rev, err := e.st.get(key) - if err != nil { - return err +// take keeps, for each mover of a passed pipeline that released a version, that version as the newest of its module on the +// pipeline's branch, by compare-and-set on a newer version (design 47 H §750): an older version, or the same one again, writes +// nothing. It says whether any key changed. +func (e *pipelineEnds) take(p mpr.PipelineEnded) (bool, error) { + if p.Outcome != mpr.PipelineEndedOutcomePassed || p.Run != "" { + return false, nil + } + changed := false + for _, m := range p.Movers { + if m.Version == "" || m.Fingerprint == "" || !moverPassed(m) { + continue } - if raw != nil { - var kept End - if err := json.Unmarshal(raw, &kept); err == nil && !kept.newer(p.Source, p.EndCount) { - return nil // an older end, or this one again + key := endKey(m.Module, p.Branch) + for { + raw, rev, err := e.st.get(key) + if err != nil { + return changed, err } + if raw != nil { + var kept End + if err := json.Unmarshal(raw, &kept); err == nil && compareVersion(m.Version, kept.Version) <= 0 { + break // an older version, or this one again + } + } + value, err := json.Marshal(End{Pipeline: p.Pipeline, Source: p.Source, Fp: m.Fingerprint, Version: m.Version, Rank: max(m.Tier, 1)}) + if err != nil { + return changed, err + } + if err := e.st.swap(key, value, rev); errors.Is(err, errSwapped) { + continue + } else if err != nil { + return changed, err + } + changed = true + break } - end := End{Source: p.Source, EndCount: p.EndCount, Fp: p.Fingerprint, Commit: p.Commit, Passed: passed, Why: p.Why} - value, err := json.Marshal(end) - if err != nil { - return err - } - if err := e.st.swap(key, value, rev); errors.Is(err, errSwapped) { - continue - } else if err != nil { - return err - } - e.wake() - return nil } + return changed, nil } -// markWritten says the wants of module on branch were written from the end it keeps, if that is still the end of fp. -func (e *pipelineEnds) markWritten(module, branch, fp string) { - key := endKey(module, branch) - for { - raw, rev, err := e.st.get(key) - var kept End - if err != nil || raw == nil || json.Unmarshal(raw, &kept) != nil || kept.Fp != fp || kept.Written { - return - } - kept.Written = true - value, _ := json.Marshal(kept) - if err := e.st.swap(key, value, rev); errors.Is(err, errSwapped) { - continue - } else if err != nil { - e.logf("the end of %s on %s is not marked written: %v", module, branch, err) - } - return +// endOf is the end kept for module on branch: the newest version released there whose pipeline passed. +func (e *pipelineEnds) endOf(module, branch string) (End, bool) { + if e == nil { + return End{}, false } -} - -// passedAt says whether the newest end kept for module on branch passed at fp. -func (e *pipelineEnds) passedAt(module, branch, fp string) bool { raw, _, err := e.st.get(endKey(module, branch)) var kept End - return err == nil && raw != nil && json.Unmarshal(raw, &kept) == nil && kept.Passed && kept.Fp == fp + if err != nil || raw == nil || json.Unmarshal(raw, &kept) != nil || kept.Fp == "" { + return End{}, false + } + return kept, true +} + +// target is the build module's wants on branch name (design 47 H §750): the newest version released there whose pipeline +// passed, with the pipeline and the mover's rank; false when none is kept. +func (e *pipelineEnds) target(module, branch string) (assign.Target, bool) { + end, ok := e.endOf(module, e.branchOf(module, branch)) + if !ok { + return assign.Target{}, false + } + return assign.Target{Fp: end.Fp, Version: end.Version, Pipeline: end.Pipeline, Rank: end.Rank}, true +} + +// hearPipeline takes a pipeline-ended: a passed one's released versions are kept (take), and a run on request's drops the +// run's candidate assignment graph (A-235, H-A-58); a run delivers nothing (design 47 G §29, §64). A pipeline ended superseded or +// cancelled has its composition steps leave the runs they read (G §723). +func (e *pipelineEnds) hearPipeline(m jetstream.Msg) { + p, err := mprc.PipelineEnded(m.Data()) + if err != nil { + e.logf("%s is not taken: %v", m.Subject(), err) + _ = m.Ack() + return + } + if left := e.left.Load(); left != nil && (p.Outcome == mpr.PipelineEndedOutcomeSuperseded || p.Outcome == mpr.PipelineEndedOutcomeCancelled) { + if err := (*left)(p.Pipeline); err != nil { + e.logf("%s: the composition steps of %s do not all leave their runs: %v; it is heard again", m.Subject(), p.Pipeline, err) + _ = m.Nak() + return + } + } + if p.Run != "" { + if e.runEnded != nil { + if err := e.runEnded(runGroupPrefix + p.Run); err != nil { + e.logf("%s: %v; it is heard again", m.Subject(), err) + _ = m.Nak() + return + } + } + _ = m.Ack() + return + } + changed, err := e.take(p) + if err != nil { + e.logf("%s: %v; it is heard again", m.Subject(), err) + _ = m.Nak() + return + } + if changed { + e.wake() + } + _ = m.Ack() +} + +// hearEnds registers pipeline-ended on the assigner's hub. +func hearEnds(hub *process.EventHub, e *pipelineEnds) error { + if err := hub.Handle(pipelineEndedSubject, e.hearPipeline); err != nil { + return fmt.Errorf("%s: %w", pipelineEndedSubject, err) + } + return nil } // branchOf is the watched branch a placement names, else the first of the module's registration, else the trunk's. @@ -282,76 +284,6 @@ func (e *pipelineEnds) branchOf(module, placed string) string { return "" } -// hearPipeline takes a pipeline-ended. A pipeline of a group writes nothing on its own end, but the runner's: in a group -// take-in that changes the runner it is the acting part the other members wait on, so it goes first, wanted on its own end -// once that passed, ahead of the group (A-334, as the node-engine does, §268). A run on request delivers nothing (design 47 G -// §29, §64). -func (e *pipelineEnds) hearPipeline(m jetstream.Msg) { - p, err := mprc.PipelineEnded(m.Data()) - if left := e.left.Load(); err == nil && left != nil && (p.Outcome == mpr.PipelineEndedOutcomeSuperseded || p.Outcome == mpr.PipelineEndedOutcomeCancelled) { - if err := (*left)(p.Pipeline); err != nil { - e.logf("%s: the composition steps of %s do not all leave their runs: %v; it is heard again", m.Subject(), p.Pipeline, err) - _ = m.Nak() - return - } - } - switch { - case err != nil: - e.logf("%s is not taken: %v", m.Subject(), err) - case p.Run == "" && p.Group != "" && e.holdUntilRead(m): - return - case p.Run != "" || (p.Group != "" && !e.isRunner(p.Module)): - default: - if err := e.take(p, requiredPassed(p)); err != nil { - e.logf("%s: %v; it is heard again", m.Subject(), err) - _ = m.Nak() - return - } - } - _ = m.Ack() -} - -// hearGroup takes a group-ended: each member's end, passed only if the group passed. A run on request's (group `run-`) delivers -// nothing: it drops the run's candidate assignment graph (A-235). -func (e *pipelineEnds) hearGroup(m jetstream.Msg) { - g, err := mprc.GroupEnded(m.Data()) - if err != nil { - e.logf("%s is not taken: %v", m.Subject(), err) - _ = m.Ack() - return - } - if strings.HasPrefix(g.Group, runGroupPrefix) { - if e.runEnded != nil { - if err := e.runEnded(g.Group); err != nil { - e.logf("%s: %v; it is heard again", m.Subject(), err) - _ = m.Nak() - return - } - } - _ = m.Ack() - return - } - for _, p := range g.Pipelines { - if err := e.take(p, g.Outcome == mpr.GroupEndedOutcomePassed && requiredPassed(p)); err != nil { - e.logf("%s: %v; it is heard again", m.Subject(), err) - _ = m.Nak() - return - } - } - _ = m.Ack() -} - -// hearEnds registers the two events on the assigner's hub. -func hearEnds(hub *process.EventHub, e *pipelineEnds) error { - if err := hub.Handle(pipelineEndedSubject, e.hearPipeline); err != nil { - return fmt.Errorf("%s: %w", pipelineEndedSubject, err) - } - if err := hub.Handle(groupEndedSubject, e.hearGroup); err != nil { - return fmt.Errorf("%s: %w", groupEndedSubject, err) - } - return nil -} - // writtenTools is the tool assigner_written: every key of the state written, with its end. func writtenTools(st store) []stdio.Tool { return []stdio.Tool{{Name: writtenTool, Description: "the pipeline ends the assigner's wants follow: per module and watched branch, the source revision and end count of the newest end heard, whether it passed and whether the wants were written from it; with why, why each want is what it is (as assigner_why).", @@ -383,56 +315,6 @@ func writtenTools(st store) []stdio.Tool { }}} } -// branchesOf is the watched branches the placements of module chose, each once; the default branch when none names one. -func (e *pipelineEnds) branchesOf(module string, placed map[string]map[string]string) []string { - if !e.following() { - return nil - } - seen := map[string]bool{} - var out []string - for _, b := range placed[module] { - b = e.branchOf(module, b) - if !seen[b] { - seen[b] = true - out = append(out, b) - } - } - if len(out) == 0 { - out = append(out, e.branchOf(module, "")) - } - sort.Strings(out) - return out -} - -// mayWant says whether module's wants may name fp: a pipeline passed at fp on every branch, or the wants name fp already -// (a node placed later, a module taken away). -func (e *pipelineEnds) mayWant(module, fp string, branches []string, wanted map[string]mwc.Want) bool { - if !e.following() { - return true - } - for _, w := range wanted { - if w.Fp == fp { - return true - } - } - for _, b := range branches { - if !e.passedAt(module, b, fp) { - return false - } - } - return true -} - -// wrote marks the end the wants of module were written from. -func (e *pipelineEnds) wrote(module, fp string, branches []string) { - if !e.following() { - return - } - for _, b := range branches { - e.markWritten(module, b, fp) - } -} - // following says whether the wants follow the pipeline ends yet; a nil follower never does. func (e *pipelineEnds) following() bool { return e != nil && e.on.Load() } @@ -476,7 +358,7 @@ func (e *pipelineEnds) registered(reg mpcat.Registration, gone bool) { // followRegistrations follows the registrations through the seat mesh-catalog (design 47 G §353): a first view of its verb // and each registration-changed on the hub. It returns when the hub's handler is registered; the first view is asked on its own -// goroutine, the wants keep the trunk-only form until a registration is heard. +// goroutine. func followRegistrations(ctx context.Context, hub *process.EventHub, call mcc.Call, e *pipelineEnds) error { regs := mcc.NewRegistrations(call) regs.Follower.Logf = e.logf diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go index 37533c01..9d7ff722 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go @@ -4,35 +4,24 @@ import ( "context" "encoding/json" "fmt" - mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient" - mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" - plc "mesh-delivery-assigner/internal/assignment" "slices" "strings" "testing" "time" + mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient" + mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" + oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" mpcat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" - - oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" - "git.novox.be/novox/mesh-sdk/go/catalogue" "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate" - - "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" - "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" - "mesh-delivery-assigner/internal/assigngraph" + plc "mesh-delivery-assigner/internal/assignment" "mesh-delivery-assigner/internal/buckets" - - mws "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" ) // memEnds is endStore over a map, with revisions. @@ -72,250 +61,148 @@ func (m *heardMsg) Subject() string { return m.subject } func (m *heardMsg) Ack() error { m.acked = true; return nil } func (m *heardMsg) Nak() error { m.naked = true; return nil } -func pipe(module, branch string, source, count int64, fp, outcome string, steps ...mpr.StepResult) mpr.PipelineEnded { - return mpr.PipelineEnded{Module: module, Branch: branch, Source: source, EndCount: count, Fingerprint: fp, Outcome: outcome, Steps: steps, - Pipeline: branch + "." + module, Commit: "c"} +// mover is one mover at its pipeline's end, its required unit step ending outcome. +func mover(module, fp, version string, tier int64, outcome string) mpr.MoverEnded { + return mpr.MoverEnded{Module: module, Fingerprint: fp, Version: version, Tier: tier, + Steps: []mpr.StepResult{{Step: "unit", Status: mpr.StepResultStatusRequired, Outcome: outcome, Module: module}}} } -func step(status, outcome string) mpr.StepResult { - return mpr.StepResult{Step: "unit", Status: status, Outcome: outcome} +// ended is the pipeline-ended of the take-in of source on branch, its movers given. +func ended(branch string, source int64, outcome string, movers ...mpr.MoverEnded) mpr.PipelineEnded { + return mpr.PipelineEnded{V: 1, Branch: branch, Source: source, EndCount: 1, Outcome: outcome, Movers: movers, + Pipeline: fmt.Sprintf("%s.%d", branch, source)} } -// Design 47 G §29, §35, §325, §352 (tests 66, 67, 89): a want follows a passed pipeline, by compare-and-set per module and branch. -func TestAWantFollowsAPassedPipeline(t *testing.T) { - const fp1, fp2 = "fp1", "fp2" +// A6/AC1 to AC3 (design 47 H §750, H-A-3): a passed pipeline keeps each mover's released version as the newest of its module on +// the branch, with the pipeline and the mover's tier; a failed pipeline, a mover whose required step failed, one that released +// no version, an older version, the same one again and a run on request keep nothing. +func TestThePassedVersionIsKept(t *testing.T) { pass, fail := mpr.PipelineEndedOutcomePassed, mpr.PipelineEndedOutcomeFailed - req, opt := mpr.StepResultStatusRequired, mpr.StepResultStatusOptional + ok, failed, nothing := mpr.StepResultOutcomePassed, mpr.StepResultOutcomeFailed, mpr.StepResultOutcomeNothingToRun for _, c := range []struct { - name string - ends []mpr.PipelineEnded - group *mpr.GroupEnded - module string - branch string - fp string - may bool + name string + ends []mpr.PipelineEnded + want *End // nil: none kept for web on main }{ - {"a failed required step writes no want", []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, fail, step(req, "failed"))}, nil, "web", "main", fp1, false}, - {"a pipeline passed writes", []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, pass, step(req, "passed"))}, nil, "web", "main", fp1, true}, - {"a failed optional step holds nothing", []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, pass, step(req, "passed"), step(opt, "failed"))}, nil, "web", "main", fp1, true}, - {"a pipeline passed at another fingerprint", []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, pass)}, nil, "web", "main", fp2, false}, - {"an older revision ending after a newer one writes nothing", []mpr.PipelineEnded{pipe("web", "main", 6, 1, fp2, fail), pipe("web", "main", 5, 1, fp1, pass)}, nil, "web", "main", fp1, false}, - {"another branch writes its own", []mpr.PipelineEnded{pipe("web", "main", 6, 1, fp2, fail), pipe("web", "beta", 5, 1, fp1, pass)}, nil, "web", "beta", fp1, true}, - {"a later end of the same revision writes", []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, fail, step(req, "failed")), pipe("web", "main", 5, 2, fp1, pass, step(req, "passed"))}, nil, "web", "main", fp1, true}, - {"the same end again after a later one keeps the later", []mpr.PipelineEnded{pipe("web", "main", 5, 2, fp1, pass), pipe("web", "main", 5, 1, fp1, fail)}, nil, "web", "main", fp1, true}, - {"a member's own end writes nothing", []mpr.PipelineEnded{func() mpr.PipelineEnded { p := pipe("web", "main", 5, 1, fp1, pass); p.Group = "g"; return p }()}, nil, "web", "main", fp1, false}, - {"a group ended passed writes its members", nil, &mpr.GroupEnded{Branch: "main", Group: "g", Outcome: mpr.GroupEndedOutcomePassed, Pipelines: []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, pass)}}, "web", "main", fp1, true}, - {"a group ended failed writes none", nil, &mpr.GroupEnded{Branch: "main", Group: "g", Outcome: mpr.GroupEndedOutcomeFailed, Pipelines: []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, pass)}}, "web", "main", fp1, false}, + {"a passed pipeline keeps its version", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "0.1.2", 2, ok))}, + &End{Pipeline: "main.5", Source: 5, Fp: "fp1", Version: "0.1.2", Rank: 2}}, + {"nothing to run counts as passed", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, nothing))}, + &End{Pipeline: "main.5", Source: 5, Fp: "fp1", Version: "0.1.2", Rank: 1}}, + {"a failed pipeline keeps nothing, a passed mover in it neither", []mpr.PipelineEnded{ended("main", 5, fail, mover("web", "fp1", "0.1.2", 1, ok), + mover("db", "fp9", "0.3.0", 1, failed))}, nil}, + {"a mover whose required step failed keeps nothing", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, failed))}, nil}, + {"no version released keeps nothing", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "", 1, ok))}, nil}, + {"an older end heard after a newer one keeps the newer", []mpr.PipelineEnded{ended("main", 6, pass, mover("web", "fp2", "0.1.3", 1, ok)), + ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok))}, &End{Pipeline: "main.6", Source: 6, Fp: "fp2", Version: "0.1.3", Rank: 1}}, + {"a newer version replaces", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok)), + ended("main", 6, pass, mover("web", "fp2", "0.1.10", 1, ok))}, &End{Pipeline: "main.6", Source: 6, Fp: "fp2", Version: "0.1.10", Rank: 1}}, + {"a run on request keeps nothing", []mpr.PipelineEnded{func() mpr.PipelineEnded { + p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok)) + p.Run = "r1" + return p + }()}, nil}, } { - woken := 0 - e := newPipelineEnds(newMemEnds(), func() { woken++ }, t.Logf) - e.on.Store(true) + e := newPipelineEnds(newMemEnds(), func() {}, t.Logf) for _, p := range c.ends { - if p.Group != "" { - continue // heard and ignored, as hearPipeline does - } - if err := e.take(p, requiredPassed(p)); err != nil { - t.Fatal(err) + if _, err := e.take(p); err != nil { + t.Fatalf("%s: %v", c.name, err) } } - if g := c.group; g != nil { - for _, p := range g.Pipelines { - if err := e.take(p, g.Outcome == mpr.GroupEndedOutcomePassed && requiredPassed(p)); err != nil { - t.Fatal(err) - } - } - } - if got := e.mayWant(c.module, c.fp, []string{c.branch}, nil); got != c.may { - t.Errorf("%s: may want = %v, want %v", c.name, got, c.may) + got, kept := e.endOf("web", "main") + switch { + case c.want == nil && kept: + t.Errorf("%s: kept %+v", c.name, got) + case c.want != nil && (!kept || got != *c.want): + t.Errorf("%s: kept %+v (%v), want %+v", c.name, got, kept, *c.want) } } - // A run on request's group-ended (group run-, A-235) writes no want and drops the run's candidate; a drop that fails is - // heard again. - { - e := newPipelineEnds(newMemEnds(), func() {}, t.Logf) - e.on.Store(true) - var dropped []string - fail := false - e.runEnded = func(group string) error { - dropped = append(dropped, group) - if fail { - return fmt.Errorf("the bucket is not open") - } - return nil - } - raw, _ := json.Marshal(mpr.GroupEnded{V: 1, Branch: "main", Group: "run-7", Outcome: mpr.GroupEndedOutcomePassed, - Pipelines: []mpr.PipelineEnded{func() mpr.PipelineEnded { p := pipe("web", "main", 5, 1, fp1, pass); p.Run = "7"; return p }()}}) - m := &heardMsg{data: raw, subject: "mesh.seat.mesh-pipeline-runner.event.group-ended.x"} - e.hearGroup(m) - if e.mayWant("web", fp1, []string{"main"}, nil) || !slices.Equal(dropped, []string{"run-7"}) || !m.acked { - t.Errorf("a run's group-ended: may want %v, dropped %v, acked %v; want no want, run-7 dropped, acked", - e.mayWant("web", fp1, []string{"main"}, nil), dropped, m.acked) - } - fail = true - m = &heardMsg{data: raw, subject: m.subject} - e.hearGroup(m) - if !m.naked { - t.Error("a run's candidate not dropped is not heard again") + // The same end heard again changes nothing and wakes nothing; a run on request's end drops its candidate. + woken := 0 + e := newPipelineEnds(newMemEnds(), func() { woken++ }, t.Logf) + var dropped []string + e.runEnded = func(group string) error { dropped = append(dropped, group); return nil } + p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok)) + for range 2 { + raw, _ := json.Marshal(p) + m := &heardMsg{data: raw, subject: "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.main.5"} + e.hearPipeline(m) + if !m.acked { + t.Error("the end was not acknowledged") } } - // A-334 (§29, §65, §268): in a group take-in that changes the runner, the runner's member (the holder of the seat - // mesh-pipeline-runner, read from the acting edges) is wanted on its own pipeline-ended once it passed, ahead of the group; - // the other members wait for group-ended. - { - e := newPipelineEnds(newMemEnds(), func() {}, t.Logf) - e.on.Store(true) - heard := func(p mpr.PipelineEnded) *heardMsg { - p.Group = "g" - raw, _ := json.Marshal(p) - m := &heardMsg{data: raw, subject: "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.x"} - e.hearPipeline(m) - return m - } - hear := func(p mpr.PipelineEnded) { - if m := heard(p); !m.acked { - t.Errorf("the pipeline-ended of %s is not acknowledged", p.Module) - } - } - // The runner's end heard before a pass read the graph is held, unacknowledged, and taken once the runner is read. - early := heard(pipe("runner", "main", 5, 1, "fp-runner", pass, step(req, "passed"))) - if early.acked || early.naked || e.mayWant("runner", "fp-runner", []string{"main"}, nil) { - t.Fatalf("the runner's end before the graph is read: acked %v, naked %v, wanted %v; want it held", early.acked, early.naked, - e.mayWant("runner", "fp-runner", []string{"main"}, nil)) - } - e.setRunner(assigngraph.Runner([]mws.AssignmentEntry{{Node: "ace", Module: "web", Edges: []mws.Edge{ - {Kind: assigngraph.EdgeActing, Node: "ace", Module: "runner", Seat: mpr.Seat, Step: "pipeline"}}}})) - if !early.acked { - t.Error("the runner's end held until the graph was read is not taken once it is") - } - hear(pipe("web", "main", 5, 1, fp1, pass, step(req, "passed"))) - if !e.mayWant("runner", "fp-runner", []string{"main"}, nil) || e.mayWant("web", fp1, []string{"main"}, nil) { - t.Errorf("a group changing the runner: may want the runner %v, web %v; want the runner ahead of the group and web not yet", - e.mayWant("runner", "fp-runner", []string{"main"}, nil), e.mayWant("web", fp1, []string{"main"}, nil)) - } - hear(pipe("runner", "main", 6, 1, "fp-runner-2", fail, step(req, "failed"))) - if e.mayWant("runner", "fp-runner-2", []string{"main"}, nil) { - t.Error("the runner's member that failed is wanted") - } - } - // Design 47 G A-428: the runner's first start skips the gate while no module claiming the seat runs; another module never - // does, and a runner running anywhere (stopped ones aside) closes it; with no running heard the gate stands. - manifests := map[string]catalogue.Manifest{"runner": {Claims: []catalogue.Claim{{Name: mpr.Seat}}}, "web": {}} - none := map[string]map[string]nodeengine.Running{"ace": {"web": {Fingerprint: fp1}}} - if !runnerFirstStart("runner", manifests, none) || runnerFirstStart("web", manifests, none) || runnerFirstStart("runner", manifests, nil) { - t.Error("the runner's first start: the runner skips the gate, another module or an unknown world does not") - } - if runnerFirstStart("runner", manifests, map[string]map[string]nodeengine.Running{"novox": {"runner": {Fingerprint: "fp-runner"}}}) || - !runnerFirstStart("runner", manifests, map[string]map[string]nodeengine.Running{"novox": {"runner": {Stopped: true}}}) { - t.Error("a runner running closes the gate's opening; one stopped does not") - } - // A want that names the fingerprint already is not held by a later failure (a node placed afterwards gets it). - e := newPipelineEnds(newMemEnds(), func() {}, t.Logf) - e.on.Store(true) - if !e.mayWant("web", fp1, []string{"main"}, map[string]mwc.Want{"g14": {Fp: fp1}}) { - t.Error("a want naming the fingerprint was held") - } - // markWritten keeps the end and says it was written from; a newer end replaces it unwritten. - st := newMemEnds() - e = newPipelineEnds(st, func() {}, t.Logf) - e.on.Store(true) - _ = e.take(pipe("web", "main", 5, 1, fp1, pass), true) - e.wrote("web", fp1, []string{"main"}) - var end End - _ = json.Unmarshal(st.vals[endKey("web", "main")], &end) - if !end.Written || end.Source != 5 { - t.Errorf("end = %+v, want written at source 5", end) - } - _ = e.take(pipe("web", "main", 6, 1, fp2, pass), true) - end = End{} - _ = json.Unmarshal(st.vals[endKey("web", "main")], &end) - if end.Written || end.Source != 6 { - t.Errorf("a newer end = %+v, want unwritten at source 6", end) + run := p + run.Run = "r1" + raw, _ := json.Marshal(run) + e.hearPipeline(&heardMsg{data: raw, subject: "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.main.5"}) + if woken != 1 || !slices.Equal(dropped, []string{runGroupPrefix + "r1"}) { + t.Errorf("woken %d (want 1), dropped %v (want the run's candidate)", woken, dropped) } } -// Design 47 G §29: the wants of a pass follow a pipeline: with the follower on, a module whose fingerprint no pipeline passed at -// gets no want and one a pipeline passed at gets it; with it off (nil, the trunk-only want) the pass is as before. -func TestAPassWritesNoWantWithoutAPassedPipeline(t *testing.T) { +// A6/AC1, A5/AC1 (design 47 H §750): a pass writes, on every node the module is assigned to, the want naming the newest version +// released on its branch whose pipeline passed, with the pipeline and the rank; with none passed it writes none. No grant, +// canary, provider or composition is asked for. +func TestAPassWritesTheWantOfThePassedVersion(t *testing.T) { fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64) - for _, c := range []struct { - name string - follow bool - passed bool - want bool - }{{"the trunk-only want", false, false, true}, {"no pipeline passed", true, false, false}, {"a pipeline passed", true, true, true}} { + for _, passed := range []bool{false, true} { wantsKV := &memKV{values: map[string][]byte{}} p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, time.Now()) p.say = func(oc.Condition) error { return nil } mdigest := strings.Repeat("e", 64) mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: mdigest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) putModuleRaw(p.w, trunkBranch, "web", mod) - p.w.manifests[mdigest] = parsed{module: "web", manifest: catalogue.Manifest{Module: "web"}} - endDone(t, p.w, "web", fp, mdigest) - gkey, _ := grantsstate.PartGrantKey("web", fp) - grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{}) - p.w.put(buckets.Grants, gkey, grant) - digest := "sha256:" + strings.Repeat("c", 64) - have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Composed: digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) - p.w.put(buckets.Have, "g14.web", have) - composes(t, p.w, "g14", digest) + p.w.manifests[mdigest] = parsed{module: "web", manifest: catalogue.Manifest{Module: "web", Requires: []string{"database"}}} placeOn(t, p.w, "g14", "web") - on := true - p.w.heardPresence("g14", !on, time.Now()) - if c.follow { - p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf) - p.ends.trunk = p.w.trunkName - p.ends.on.Store(true) - if c.passed { - _ = p.ends.take(pipe("web", trunkBranch, 5, 1, fp, mpr.PipelineEndedOutcomePassed), true) - } + placeOn(t, p.w, "ace", "web") + p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf) + p.ends.trunk = p.w.trunkName + if passed { + _, _ = p.ends.take(ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("web", fp, "0.1.2", 2, mpr.StepResultOutcomePassed))) } p.wants(context.Background()) - key, _ := mwc.WantKey("g14", "web") - if _, got := wantsKV.values[key]; got != c.want { - t.Errorf("%s: a want written = %v, want %v", c.name, got, c.want) - } - if c.follow && c.passed { - raw, _, _ := p.ends.st.get(endKey("web", trunkBranch)) - var end End - _ = json.Unmarshal(raw, &end) - if !end.Written { - t.Errorf("%s: the end was not marked written", c.name) + for _, node := range []string{"g14", "ace"} { + key, _ := mwc.WantKey(node, "web") + raw, got := wantsKV.values[key] + if got != passed { + t.Errorf("passed %v: a want on %s written = %v", passed, node, got) + continue + } + if !passed { + continue + } + if w, err := mwc.DecodeWant(raw); err != nil || w.Fp != fp || w.Version != "0.1.2" || w.Pipeline != trunkBranch+".5" || w.Rank != 2 { + t.Errorf("the want on %s = %+v (%v)", node, w, err) } } } } -// Design 47 G §95, §130, §353, §352 (test 100): the wants keep the trunk-only form until the first registration is heard and -// switch on it; a branch a registration no longer watches loses its end, and its first branch is a placement's default. -func TestTheWantsSwitchOnTheFirstRegistration(t *testing.T) { +// Design 47 G §353, §352: a registration's first branch is a placement's default, and a branch it no longer watches loses its +// end; a registration gone loses them all. +func TestARegistrationNamesTheDefaultBranch(t *testing.T) { st := newMemEnds() woken := 0 e := newPipelineEnds(st, func() { woken++ }, t.Logf) e.trunk = func() string { return trunkBranch } - _ = e.take(pipe("web", "main", 5, 1, "fp1", mpr.PipelineEndedOutcomeFailed), false) - _ = e.take(pipe("web", "beta", 5, 1, "fp1", mpr.PipelineEndedOutcomePassed), true) - if e.following() || !e.mayWant("web", "fp1", []string{"main"}, nil) { - t.Fatal("the follower held a want before any registration") - } + pass := mpr.PipelineEndedOutcomePassed + _, _ = e.take(ended("main", 5, pass, mover("web", "fp1", "0.1.1", 1, mpr.StepResultOutcomePassed))) + _, _ = e.take(ended("beta", 5, pass, mover("web", "fp2", "0.1.2", 1, mpr.StepResultOutcomePassed))) e.registered(mpcat.Registration{Module: "web", Branches: []string{"beta"}}, false) - if !e.following() || woken == 0 { - t.Error("the first registration did not switch the wants on") - } - if e.mayWant("web", "fp1", []string{"main"}, nil) || !e.mayWant("web", "fp1", []string{"beta"}, nil) { - t.Error("after the switch the wants did not follow the ends") - } - if len(st.vals) != 1 || e.branchOf("web", "") != "beta" || e.branchOf("other", "") != trunkBranch { + if woken == 0 || len(st.vals) != 1 || e.branchOf("web", "") != "beta" || e.branchOf("other", "") != trunkBranch { t.Errorf("ends kept %d, default branch %q: an unwatched branch's end stayed, or the default is wrong", len(st.vals), e.branchOf("web", "")) } + if tg, ok := e.target("web", ""); !ok || tg.Fp != "fp2" { + t.Errorf("the default branch's target = %+v %v", tg, ok) + } e.registered(mpcat.Registration{Module: "web"}, true) if len(st.vals) != 0 { t.Errorf("a registration gone left %d ends", len(st.vals)) } } -// Design 47 G §14, §184 (test 68): a node's entry is computed from the catalog half of the branch its placement names, so a -// change on `beta` moves only the want of the node whose placement chose `beta`. -func TestACommitOnBetaMovesOnlyTheNodeThatChoseBeta(t *testing.T) { +// Design 47 G §14, §184 (test 68): a node's want follows the branch its placement names, so a version passed on `beta` moves only +// the want of the node whose placement chose `beta`. +func TestAVersionOnBetaMovesOnlyTheNodeThatChoseBeta(t *testing.T) { fp := func(c string) string { return meshcatalogclient.SourcePrefix2 + strings.Repeat(c, 64) } wantsKV := &memKV{values: map[string][]byte{}} p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, time.Now()) @@ -323,15 +210,8 @@ func TestACommitOnBetaMovesOnlyTheNodeThatChoseBeta(t *testing.T) { p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf) p.ends.trunk = p.w.trunkName p.ends.registered(mpcat.Registration{Module: "web", Branches: []string{"main", "beta"}}, false) - digest := "sha256:" + strings.Repeat("c", 64) - on := true - for _, node := range []string{"g14", "ace"} { - composes(t, p.w, node, digest) - p.w.heardPresence(node, !on, time.Now()) - } for node, branch := range map[string]string{"g14": "", "ace": "beta"} { - pl := mma.Assignments{Node: node, Modules: []mma.PlacedModule{{Module: "web", Branch: branch}}} - values, err := mmac.AssignmentsValues(pl) + values, err := mmac.AssignmentsValues(mma.Assignments{Node: node, Modules: []mma.PlacedModule{{Module: "web", Branch: branch}}}) if err != nil { t.Fatal(err) } @@ -339,24 +219,10 @@ func TestACommitOnBetaMovesOnlyTheNodeThatChoseBeta(t *testing.T) { p.w.put(plc.AssignmentsBucket(), key, raw) } } - // publish makes the half of branch name web at fingerprint f: built, granted and passed by its pipeline. - publish := func(branch, f string, source int64) { - mdigest := strings.Repeat("e", 62) + fmt.Sprintf("%02d", source) - if branch == "beta" { - mdigest = strings.Repeat("f", 62) + fmt.Sprintf("%02d", source) - } - mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: f, Manifest: mdigest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) - putModuleRaw(p.w, branch, "web", mod) - p.w.manifests[mdigest] = parsed{module: "web", manifest: catalogue.Manifest{Module: "web"}} - endDone(t, p.w, "web", f, mdigest) - gkey, _ := grantsstate.PartGrantKey("web", f) - grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{}) - p.w.put(buckets.Grants, gkey, grant) - _ = p.ends.take(pipe("web", branch, source, 1, f, mpr.PipelineEndedOutcomePassed), true) - } - run := func(node, f string) { - have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: f, Composed: digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) - p.w.put(buckets.Have, node+".web", have) + mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp("a"), Manifest: strings.Repeat("e", 64), Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) + putModuleRaw(p.w, "main", "web", mod) + passed := func(branch, f, version string, source int64) { + _, _ = p.ends.take(ended(branch, source, mpr.PipelineEndedOutcomePassed, mover("web", f, version, 1, mpr.StepResultOutcomePassed))) } fpWant := func(node string) string { key, _ := mwc.WantKey(node, "web") @@ -366,18 +232,16 @@ func TestACommitOnBetaMovesOnlyTheNodeThatChoseBeta(t *testing.T) { } return w.Fp } - publish("main", fp("a"), 5) - publish("beta", fp("b"), 5) - run("g14", fp("a")) - run("ace", fp("b")) + passed("main", fp("a"), "0.1.1", 5) + passed("beta", fp("b"), "0.2.1", 5) p.wants(context.Background()) if fpWant("g14") != fp("a") || fpWant("ace") != fp("b") { t.Fatalf("each node is wanted at its branch's build: g14 %s, ace %s", fpWant("g14"), fpWant("ace")) } - publish("beta", fp("d"), 6) + passed("beta", fp("d"), "0.2.2", 6) p.wants(context.Background()) if fpWant("g14") != fp("a") || fpWant("ace") != fp("d") { - t.Errorf("a commit on beta moves only ace: g14 %s (want main's), ace %s (want beta's new)", fpWant("g14"), fpWant("ace")) + t.Errorf("a version on beta moves only ace: g14 %s (want main's), ace %s (want beta's new)", fpWant("g14"), fpWant("ace")) } } @@ -411,40 +275,3 @@ func TestTheCatalogHalfIsTakenInPerBranch(t *testing.T) { t.Error("a module taken out of beta's half leaves main's") } } - -// Design 47 G §379, §526: a want held at an older build, the trunk's newer fingerprint having passed no pipeline, is rewritten -// with that build's released version (and parts for an engine that reads them), the rest of the want kept; no want names the -// unpassed fingerprint (novox 2026-11-16, mesh-hand-acts 0.1.13 at 4eb3cf48). -func TestAWantHeldAtAnOlderBuildNamesItsReleaseWithoutAPassedPipeline(t *testing.T) { - old := meshcatalogclient.SourcePrefix2 + strings.Repeat("4", 64) - fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("9", 64) - wantsKV := &memKV{values: map[string][]byte{}} - p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, time.Now()) - p.say = func(oc.Condition) error { return nil } - mdigest := strings.Repeat("e", 64) - mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: mdigest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) - putModuleRaw(p.w, trunkBranch, "web", mod) - p.w.manifests[mdigest] = parsed{module: "web", manifest: catalogue.Manifest{Module: "web"}} - endRaw, _ := json.Marshal(builds.BuildEnded{Module: "web", Fingerprint: old, Outcome: builds.BuildDone, Manifest: mdigest, Made: map[string]string{"tools": "sha256:1c"}}) - if err := p.w.ends.put(endRaw); err != nil { - t.Fatal(err) - } - rel, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: "web", Source: 962, Version: "0.1.13", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "tools", Digest: "sha256:1c"}}}) - if err := p.w.released.put(rel); err != nil { - t.Fatal(err) - } - placeOn(t, p.w, "g14", "web") - key, _ := mwc.WantKey("g14", "web") - held, _ := mwc.EncodeWant(mwc.Want{Fp: old, Self: true}) - wantsKV.values[key] = held - p.w.put(buckets.Wants(), key, held) - p.w.heardPresence("g14", false, time.Now()) - p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf) - p.ends.trunk = p.w.trunkName - p.ends.on.Store(true) - p.wants(context.Background()) - got, err := mwc.DecodeWant(wantsKV.values[key]) - if err != nil || got.Fp != old || got.Version != "0.1.13" || !got.Self { - t.Errorf("the held want = %+v, %v; want %s at 0.1.13, self kept", got, err, old) - } -} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/placeorder_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/placeorder_test.go index 761b7271..c2202904 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/placeorder_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/placeorder_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" "strings" "testing" "time" @@ -67,6 +68,8 @@ func newOrderFixture(t *testing.T, mods []orderMod, history ...map[string]string return nil } digest := "sha256:" + strings.Repeat("c", 64) + p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf) + p.ends.trunk = p.w.trunkName pl := mma.Assignments{Node: "ace"} forms := map[string]int{} // by fingerprint: each module's own modules := []string{} @@ -92,6 +95,8 @@ func newOrderFixture(t *testing.T, mods []orderMod, history ...map[string]string grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{}) p.w.put(buckets.Grants, gkey, grant) f.runs(m.name, m.run) + // The trunk's build is released with a passed pipeline (design 47 H §750). + _, _ = p.ends.take(ended(trunkBranch, 1, mpr.PipelineEndedOutcomePassed, mover(m.name, m.trunk, "0.1.1", 1, mpr.StepResultOutcomePassed))) wkey, _ := mwc.WantKey("ace", m.name) old, _ := mwc.EncodeWant(mwc.Want{Fp: m.run, Self: true}) p.w.put(buckets.Wants(), wkey, old) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go index dc4b3db9..64415132 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go @@ -136,14 +136,6 @@ func (p *passer) puts(ctx context.Context) { world := p.putWorld(node, module, put, manifest, assigned[node]) world.Snapshot, world.Snapshotting, world.SnapshotFailed = p.acts.state(id) next := assign.StepPut(world) - // A held bus whose user list the rules moved on (ADR 0539 §7): the put follows the composition its want names. - if w, ok := wanted[module][node]; ok && w.Put && w.Fp == next.Put.FP && w.Composed != "" && w.Composed != next.Put.Composed && - (next.Put.Step == puts.Wanted || next.Put.Step == puts.Holding) { - if d, _ := p.composedFor(node, module, w.Fp); d != nil && d.UserListOnly && d.Digest == w.Composed { - next.Put = next.Put.Said(world.Now, next.Put.Step, fmt.Sprintf("the bus's user list changed: the want names composition %s", shortHex(w.Composed))) - next.Put.Composed = w.Composed - } - } // The bus's want is written only for the act the operator's yes bound (ADR 0546 clause 8). if next.Put.Step == puts.Wanted && put.Step != puts.Wanted && world.Bus && next.Put.Ask != "" { if err := p.performs(next.Put.Ask, assign.PutAct(node, module, next.Put.FP, next.Put.Composed)); err != nil { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/revisions.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/revisions.go new file mode 100644 index 00000000..57c2d085 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/revisions.go @@ -0,0 +1,140 @@ +package main + +import ( + "context" + "fmt" + "slices" + "strconv" + "strings" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + + "mesh-delivery-assigner/internal/assigngraph" +) + +// The assignment graph revisions are removed when they are no longer current (design 47 H §764, H-A-86): the gate's pins of +// them went with its pinned, freed, heads-replaced and pins-stated (H-A-48), and with them the assigner's state pins and its +// tools assigner_pins and assigner_unpin. A revision that is not current (the newest the assigner wrote) is removed, its object +// and the entry objects no kept revision or candidate lists and no want or running build names (G §179), when a revision ceases +// to be current and once at the start. Never on a timer. + +// revisionObjects is the object bucket of the revisions as the remover removes from it. +type revisionObjects interface { + assigngraph.Objects + Delete(ctx context.Context, name string) error +} + +// revisionRemover removes the revisions that are not current. +type revisionRemover struct { + ctx context.Context + objects revisionObjects + // inUse says a want or a running build names the module's fingerprint (§179). + inUse func(module, fp string) bool + logf func(string, ...any) +} + +// judge removes every revision that is not current; a removal that fails is said and made at the next judging. +func (k *revisionRemover) judge() { + if err := k.remove(); err != nil { + k.logf("the revisions that are not current are not removed: %v", err) + } +} + +// trunkRevision is the revision an object of a trunk revision names, and whether it names one. +func trunkRevision(name string) (int64, bool) { + n, ok := strings.CutPrefix(name, "trunk.") + if !ok { + return 0, false + } + r, err := strconv.ParseInt(n, 10, 64) + return r, err == nil && r > 0 +} + +// remove removes each kept revision that is not current, and then the entries no kept revision or candidate lists and no want or +// running build names (design 47 G §179). +func (k *revisionRemover) remove() error { + names, err := k.objects.Names(k.ctx) + if err != nil { + return err + } + var kept []int64 + for _, n := range names { + if r, ok := trunkRevision(n); ok { + kept = append(kept, r) + } + } + if len(kept) == 0 { + return nil + } + current := slices.Max(kept) + listed := map[string]bool{} + removed := 0 + for _, r := range kept { + if r == current { + rev, err := assigngraph.ReadRevision(k.ctx, k.objects, r) + if err != nil { + return err + } + for _, e := range rev.Entries { + listed[e.Digest] = true + } + continue + } + if err := k.objects.Delete(k.ctx, assigngraph.TrunkObject(r)); err != nil { + return fmt.Errorf("revision %d: %w", r, err) + } + removed++ + } + if removed == 0 { + return nil + } + // A run's candidate kept now lists its entries too, until its run's pipeline-ended drops it (A-201, A-235). + for _, n := range names { + group, heads, ok := meshwants.ParseAssignmentRevisionsByCandidateGroupHeadsKey(n) + if !ok { + continue + } + rev, err := assigngraph.ReadCandidate(k.ctx, k.objects, group, heads) + if err != nil { + return err + } + for _, e := range rev.Entries { + listed[e.Digest] = true + } + } + for _, n := range names { + digest, ok := strings.CutPrefix(n, "entry.") + if !ok || listed[digest] { + continue + } + e, err := assigngraph.ReadEntry(k.ctx, k.objects, digest) + if err != nil { + return err + } + if k.inUse != nil && k.inUse(e.Module, e.Fp) { + continue + } + if err := k.objects.Delete(k.ctx, n); err != nil { + return fmt.Errorf("the entry %s: %w", digest, err) + } + } + k.logf("%d assignment graph revisions not current are removed", removed) + return nil +} + +// inUseOf says a want or a running build names the module's fingerprint, as the world holds them now. +func inUseOf(w *world) func(module, fp string) bool { + return func(module, fp string) bool { + for _, want := range w.wantsOf()[module] { + if want.Fp == fp { + return true + } + } + for _, mods := range w.runningNow() { + if r, ok := mods[module]; ok && r.Fingerprint == fp { + return true + } + } + return false + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/revisions_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/revisions_test.go new file mode 100644 index 00000000..5b38752d --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/revisions_test.go @@ -0,0 +1,53 @@ +package main + +import ( + "context" + "encoding/json" + "testing" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + + "mesh-delivery-assigner/internal/assigngraph" +) + +func (m memObjects) Delete(_ context.Context, name string) error { delete(m, name); return nil } + +// Design 47 H §764, H-A-86: with the gate's pins gone, an assignment graph revision that is not current is removed, with the +// entries no kept revision or candidate lists and no want or running build names; the current revision, the entries it lists, +// a running build's entry and a run's kept candidate with its entries stay. +func TestARevisionNotCurrentIsRemoved(t *testing.T) { + objects := memObjects{} + entry := func(module, fp string) string { + e := meshwants.AssignmentEntry{Node: "n1", Module: module, Fp: fp} + d := assigngraph.EntryDigest(e) + raw, _ := json.Marshal(e) + objects[assigngraph.EntryObject(d)] = raw + return d + } + shared, old, running, cand := entry("a", "fp-a"), entry("b", "fp-b1"), entry("c", "fp-c1"), entry("d", "fp-d2") + revision := func(n int64, digests ...string) { + rev := meshwants.AssignmentRevision{Revision: n} + for _, d := range digests { + rev.Entries = append(rev.Entries, meshwants.NodeEntryRef{Node: "n1", Digest: d}) + } + raw, _ := json.Marshal(rev) + objects[assigngraph.TrunkObject(n)] = raw + } + candRaw, _ := json.Marshal(meshwants.AssignmentRevision{Revision: 1, Kind: "candidate", Group: "run-7", Heads: "h1", + Entries: []meshwants.NodeEntryRef{{Node: "n1", Module: "d", Digest: cand}}}) + objects[assigngraph.CandidateObject("run-7", "h1")] = candRaw + revision(1, shared, old, running) + revision(2, shared, old) + revision(3, shared) + k := &revisionRemover{ctx: context.Background(), objects: objects, logf: t.Logf, + inUse: func(module, fp string) bool { return module == "c" && fp == "fp-c1" }} + k.judge() + has := func(name string) bool { _, ok := objects[name]; return ok } + for name, want := range map[string]bool{assigngraph.TrunkObject(1): false, assigngraph.TrunkObject(2): false, assigngraph.TrunkObject(3): true, + assigngraph.EntryObject(old): false, assigngraph.EntryObject(running): true, assigngraph.EntryObject(shared): true, + assigngraph.EntryObject(cand): true, assigngraph.CandidateObject("run-7", "h1"): true} { + if has(name) != want { + t.Errorf("%s kept = %v, want %v", name, has(name), want) + } + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go index 8d4b894f..f037ba4f 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go @@ -312,10 +312,10 @@ func warrantOf(v any) (*mma.Warrant, error) { // mesh-node-enroller when the proven caller is that module's (its bus user `.`, never a person's // `person.`), else the caller as proven, which the refusal names. func moduleOfCaller(by string, enrollers []string) string { - for _, module := range enrollers { - if (stdio.Callers{module}).Allows(by) { - return module - } + // The enrolment's caller stays (design 47 H-A-41): a module's bus user is `.`, a person's `person.`. + _, module, ok := strings.Cut(by, ".") + if ok && !strings.HasPrefix(by, "person.") && slices.Contains(enrollers, module) { + return module } return by } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go index fbe233ca..8b18c9f0 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go @@ -20,11 +20,9 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey" "git.novox.be/novox/mesh-sdk/go/keys" - "git.novox.be/novox/mesh-sdk/go/seatlist" mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient" mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" - "git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/views" @@ -270,19 +268,6 @@ func TestTheSeatsVerbs(t *testing.T) { }) } s.seatList = nil - t.Run("1966: the seat list's again tool answers an operator's session through the mesh MCP server", func(t *testing.T) { - // Such a session is proved as its machine's tool runner (issue 627); the healers too, no other module. The list names - // the seats, resolved from the assigner's own seat-holders (A-295, A-298): a module holding neither is refused. - held := map[string][]byte{"node-tool-runner.shanks": []byte(`{"holders":[{"module":"node-tools","node":"shanks"}]}`), - "mesh-healers": []byte(`{"holders":[{"module":"mesh-delivery-watchman","node":"ace"}]}`)} - holders := seatlist.HoldersIn(func() (map[string][]byte, error) { return held, nil }) - for caller, refused := range map[string]bool{"person.jochen": false, "shanks.node-tools": false, "ace.mesh-delivery-watchman": false, - "ace.mesh-healers": true, "ace.messenger": true} { - if err := seatlist.RefusalWith("assigner_seat_list_again", seatListAgainCallers, caller, holders); (err != nil) != refused { - t.Errorf("%s: refused=%v, err %v", caller, refused, err) - } - } - }) t.Run("1953/D4-D6: the seat mesh-wants' upgrade and bus", wantsVerbs) t.Run("1955/L7 (ADR 0571 §6): rule 7 reads the node's own composition; an unplaced node is said", func(t *testing.T) { w := newWorld() @@ -533,51 +518,6 @@ func TestTheSeatsVerbs(t *testing.T) { }) } }) - t.Run("1954/D1: a failed provision is the assigner's finding until it is provided", func(t *testing.T) { - findings := map[string][]byte{} - p := &passer{w: newWorld(), logf: func(string, ...any) {}, kvs: kvSet{buckets.Provisions: nil}, findings: assign.Findings{ - Put: putInto(findings), - Delete: func(k string) error { delete(findings, k); return nil }}} - failed, _ := meshprovisionerclient.Encode(meshprovisionerclient.Provision{Provision: "db", Consumer: meshprovisionerclient.End{Module: "photos", Node: "ace"}, - Provider: meshprovisionerclient.End{Module: "postgres", Node: "novox"}, State: meshprovisionerclient.Failed, Why: "the role exists already"}) - p.w.putEntry(buckets.Provisions, "ace.photos.db", failed, 1, time.Now()) - p.kvs[buckets.Provisions] = &memKV{} - of := p.provisionsOf() - if got := of("photos")("ace"); got["ace.photos.db"].State != meshprovisionerclient.Failed { - t.Fatalf("provisions = %+v", got) - } - if _, ok := findings[assign.ProvisionFailedKey("ace.photos.db")]; !ok || len(findings) != 1 { - t.Fatalf("findings = %v", findings) - } - provided, _ := meshprovisionerclient.Encode(meshprovisionerclient.Provision{Provision: "db", Consumer: meshprovisionerclient.End{Module: "photos", Node: "ace"}, - Provider: meshprovisionerclient.End{Module: "postgres", Node: "novox"}, State: meshprovisionerclient.Provided}) - p.w.putEntry(buckets.Provisions, "ace.photos.db", provided, 2, time.Now()) - p.provisionsOf() - if len(findings) != 0 { - t.Errorf("findings after provided = %v", findings) - } - // A required provision with no key: pulsed with the bound first, said once the pulse expired, ended once keyed. - ctx := context.Background() - p.kvs[plc.PulsesBucket()] = &memKV{values: map[string][]byte{}} - gone := map[string]bool{"ace.photos.s3": true} - p.provisionsMissing(ctx, gone) - if !p.pulsing("provision-missing.ace.photos.s3") || len(findings) != 0 { - t.Fatalf("first seen: pulsing %v, findings %v", p.pulsing("provision-missing.ace.photos.s3"), findings) - } - p.provisionsMissing(ctx, gone) - if len(findings) != 0 { - t.Fatal("said before its bound") - } - p.w.put(plc.PulsesBucket(), "provision-missing.ace.photos.s3", nil) // the server expired it - p.provisionsMissing(ctx, gone) - if _, ok := findings[assign.ProvisionMissingKey("ace.photos.s3")]; !ok { - t.Fatalf("past its bound: findings %v", findings) - } - p.provisionsMissing(ctx, map[string]bool{}) - if len(findings) != 0 { - t.Errorf("keyed again: findings %v", findings) - } - }) // A whole state is read in pages where the store has them, never a key at a time (27 s for the assignments live, // 2026-11-14): the paged read answers what the key-by-key read answers, and asks no key alone. direct, err := readAll(st, haveState) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seatlist.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seatlist.go index cf2da054..1b78b133 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seatlist.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seatlist.go @@ -1,16 +1,7 @@ package main -import ( - stdio "git.novox.be/novox/mesh-sdk/go" -) - // The assigner reads the seat list as every part does, through mesh-sdk's one wiring (go/seatlist.Part; design 47 G §204): its // reader over the seat mesh-catalog's verb seat-definitions and its two events, its tools `assigner_seat_list` and // `assigner_seat_list_again`. What the reader holds is given to catalogue at every change, which the check of a proposal and the // seat-holders state read; until the list is read, a mesh seat the bootstrapped definitions do not name is never judged held once // (plc.CheckAssignments), and seat-holders is not written. - -// A call through the mesh MCP server is proved only as its machine's tool runner (issue 627), so the holders of the seat -// node-tool-runner stand for an operator's session until a session is proved (issue 1966). The list names seats, never a -// module, resolved from the assigner's own seat-holders (design 47 G A-210, A-295, A-298). -var seatListAgainCallers = []string{stdio.AnyPerson, stdio.CallerSeatPrefix + "node-tool-runner", stdio.CallerSeatPrefix + "mesh-healers"} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seatlist_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seatlist_test.go index 992599c2..64c3ecc4 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seatlist_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seatlist_test.go @@ -60,7 +60,7 @@ func gateOver(t *testing.T, silent bool) *seatlist.Part { // that waits for it to stay unread keeps short. t.Cleanup(busretry.Waits(20*time.Millisecond, time.Millisecond, 5*time.Millisecond)) } - g := seatlist.NewPart("assigner", seatListAgainCallers) + g := seatlist.NewPart("assigner") r := g.Follow(standInCatalog{silent: silent}, nil, t.Logf) if r == nil { t.Fatal("the reader did not start") @@ -98,7 +98,7 @@ func TestTheSeatListIsWiredByTheSDKAlone(t *testing.T) { // A-179, A-186: the definition the verb input's callers are checked against is the seat list's, none before the reader starts // or while the seat mesh-catalog has not answered, and none for a seat the list does not hold. func TestTheAnsweredSeatsDefinitionIsTheSeatListsOnceRead(t *testing.T) { - if _, ok := seatlist.Definition(seatlist.NewPart("assigner", seatListAgainCallers).Reader, "mesh-wants")(); ok { + if _, ok := seatlist.Definition(seatlist.NewPart("assigner").Reader, "mesh-wants")(); ok { t.Error("a definition is answered before the seat list's reader started") } if _, ok := seatlist.Definition(gateOver(t, true).Reader, "mesh-wants")(); ok { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go index 8caf8846..c7123cb6 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go @@ -1,7 +1,6 @@ package main import ( - mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" plc "mesh-delivery-assigner/internal/assignment" "encoding/json" @@ -394,7 +393,7 @@ func serveTools() error { } }() // The seat list (seatlist.go), read through the runtime once Serve has its channel: the seat's verbs judge by it. - gate := seatlist.NewPart("assigner", seatListAgainCallers) + gate := seatlist.NewPart("assigner") go func() { <-stdio.Served() gate.Follow(seatlist.StdioBus(), nil, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: "+f+"\n", a...) }) @@ -408,8 +407,6 @@ func serveTools() error { return stdio.SeatPublish(wantsEventSubject(e), b, id) }) wants.watched = stdio.NewWatchedStates() // the large states read whole from one watched copy (ADR 0580 clause 3) - // The callers' seat entries are resolved from the assigner's own seat-holders, its watched copy (A-298). - gate.Holders(seatlist.HoldersIn(func() (map[string][]byte, error) { return wants.watched.Values(mma.StateSeatHolders) })) // The verb assignments with seats answers from the same watched copy, so every caller of it is fast too (issue 1982). seatVerbs.watched = wants.watched wants.revisions = runtimeObjects{busstate.ObjectsOf(assigngraph.RevisionsObjects)} @@ -422,7 +419,6 @@ func serveTools() error { tools = append(tools, enginereads.Tool(engineReadsTool, "the assigner", enginereads.KeptState(stdio.State(engineReadsState)), false, time.Now)) tools = append(tools, writtenTools(runtimeStore{})...) tools = append(tools, whyTools(runtimeStore{})...) - tools = append(tools, pinsTools(runtimeStore{}, stdio.Ask)...) tools = append(tools, askAgainTools(runtimeStore{}, stdio.Ask, time.Now)...) return stdio.Serve("", tools) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/verbs.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/verbs.go index 9a4dec05..9f867b0c 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/verbs.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/verbs.go @@ -24,26 +24,6 @@ import ( // moduleGraph is the module graph of the catalog half of branch, from the entries' invokes and serves (design 47 G §184, §195). func (p *passer) moduleGraph(branch string) graph.Graph { return p.w.graphOf(branch) } -// keptInvokes are the verbs of module the graph keeps as edges: each it invokes and does not leave out, that a module -// other than itself serves. A verb no module of the graph serves is no edge, so it holds nothing; nil when the graph holds -// no node of module. -func keptInvokes(g graph.Graph, module string) []string { - n, ok := g[module] - if !ok { - return nil - } - var out []string - for _, v := range n.Invokes { - if slices.Contains(n.LeftOut, v) { - continue - } - if slices.ContainsFunc(g.Servers(v), func(s string) bool { return s != module }) { - out = append(out, v) - } - } - return out -} - // served is, for this pass, whether a build serving a verb runs healthy on some node (graph.Served): every running healthy // have's build (the builder's builds key of its module and fingerprint) names its manifest, and the verbs that manifest // serves are served. A healthy build whose manifest cannot be read counts what the graph says its module serves, said in diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/views.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/views.go index 92eeecb7..2aec6e8d 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/views.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/views.go @@ -73,7 +73,7 @@ func viewTools(getenv func(string) string, state func(string) views.Source, now marks := state(views.MarksState) var out []stdio.Tool for _, t := range firstViewTools { - out = append(out, views.FirstViewTool(assignment.Module(), t.tool, t.state, t.words, views.ParseCallers(getenv(views.CallersEnv(t.tool))), state(t.state), marks, now)) + out = append(out, views.FirstViewTool(assignment.Module(), t.tool, t.state, t.words, state(t.state), marks, now)) } for _, t := range readTools { out = append(out, views.ReadTool(assignment.Module(), t.tool, t.state, t.words, state(t.state), marks, now)) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/why.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/why.go index ce6e58d4..94d73776 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/why.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/why.go @@ -26,32 +26,36 @@ const whyTool = "assigner_why" // The steps of the pass that decide a want. const ( - whyNoPipeline = "no-passed-pipeline" // the trunk's fingerprint passed no pipeline on the branch, and no want names it - whyRules = "rules" // the six rules judged it + whyNoPipeline = "no-passed-pipeline" // no version of the module released on the branch passed its pipeline: the want written stays + whyPassed = "passed-pipeline" // the want names the newest version released on the branch whose pipeline passed (design 47 H §750) ) +// whyStep is the step that decided a want: a passed pipeline's version, or none yet. +func whyStep(passed bool) string { + if passed { + return whyPassed + } + return whyNoPipeline +} + // wantWhy is one key's value. type wantWhy struct { At time.Time `json:"at"` Branch string `json:"branch,omitempty"` - // Fp is the fingerprint the trunk's entry names, the one the pass judged. - Fp string `json:"fp,omitempty"` - Step string `json:"step"` + // Fp, Version and Pipeline are the build the want names and the pipeline whose passed end released it (design 47 H §750). + Fp string `json:"fp,omitempty"` + Version string `json:"version,omitempty"` + Pipeline string `json:"pipeline,omitempty"` + Step string `json:"step"` // Before is the want that stood before the pass, After the one the pass wrote (none: it kept Before). Before *mwc.Want `json:"before,omitempty"` After *mwc.Want `json:"after,omitempty"` - // Rule is the rule holding the node back from Fp, with its words. - Rule string `json:"rule,omitempty"` - // What the rules read of the module. - Built, BuildFailed, Held, Granted, Released, Kept bool `json:",omitempty"` - BuildError string `json:"build-error,omitempty"` - Words string `json:"words,omitempty"` } func whyKey(node, module string) string { return whyPrefix + node + "." + module } // sayWhy writes the why of module on each of nodes, once per change (the time aside): every put is a change its watchers hear. -func (p *passer) sayWhy(ctx context.Context, module string, nodes []string, w wantWhy, before map[string]mwc.Want, out *assign.Outcome, m *assign.Module) { +func (p *passer) sayWhy(ctx context.Context, module string, nodes []string, w wantWhy, before map[string]mwc.Want, out *assign.Outcome) { kv := p.kvs[writtenBucket()] if kv == nil { return @@ -69,12 +73,6 @@ func (p *passer) sayWhy(ctx context.Context, module string, nodes []string, w wa if a, ok := out.Wants[node]; ok { one.After = &a } - if h, ok := out.Rules[node]; ok { - one.Rule = h.Reason() - } - } - if m != nil { - one.Built, one.BuildFailed, one.BuildError, one.Held, one.Granted, one.Released, one.Kept = m.Built, m.BuildFailed, m.BuildError, m.Held, m.Granted, m.Released, m.Kept } same := one same.At = time.Time{} @@ -97,7 +95,7 @@ func (p *passer) sayWhy(ctx context.Context, module string, nodes []string, w wa // whyTools is the tool assigner_why: every want's why, or those of one module or one node. func whyTools(st store) []stdio.Tool { - return []stdio.Tool{{Name: whyTool, Description: "why a want is what it is: per node and module, the step of the assigner's last pass that decided the want (no passed pipeline, or the six rules with the rule holding it back), the fingerprint the trunk names, the want before and the one written, and what the rules read (built, held, granted). A time that stands still says the pass no longer judges that module. Arguments: module and node narrow it.", + return []stdio.Tool{{Name: whyTool, Description: "why a want is what it is: per node and module, the step of the assigner's last pass that decided the want (no passed pipeline yet, or a passed pipeline's version), the build, version and pipeline it names, the want before and the one written. A time that stands still says the pass no longer judges that module. Arguments: module and node narrow it.", Input: schema(map[string]string{"module": "a module's name", "node": "a node's name"}, nil), Run: func(args map[string]any) (any, error) { ks, err := st.keys(writtenState) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go index 947569cf..1a9d9b1b 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go @@ -12,7 +12,6 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" mdc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient" mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" @@ -135,23 +134,6 @@ func (w *world) put(bucket, key string, value []byte) { } } -// lastHealthy is the newest fingerprint other than fp the node ran the module healthy at, or empty. -func (w *world) lastHealthy(node, module, fp string) string { - key, err := nodeengineclient.HaveKey(node, module) - if err != nil { - return "" - } - w.mu.Lock() - defer w.mu.Unlock() - list := w.healthy[key] - for i := len(list) - 1; i >= 0; i-- { - if list[i] != fp { - return list[i] - } - } - return "" -} - // read is a copy of what a bucket holds now. func (w *world) read(bucket string) map[string][]byte { w.mu.Lock() @@ -460,30 +442,6 @@ func (w *world) has(bucket, key string) bool { return ok } -// built says whether the builder's build of a module at a fingerprint is done, from its `build.ended` heard (design 47 G A-181). -func (w *world) built(module, fp string) bool { - e, ok := w.ends.endOf(module, fp) - return ok && e.Outcome == builds.BuildDone -} - -// buildStanding says whether the builder's `build.ended` of module at fp said the build failed, and its error whole (design 47 G -// §575, A-162): the words of a hold under rule 1. -func (w *world) buildStanding(module, fp string) (failed bool, words string) { - if e, ok := w.ends.endOf(module, fp); ok && e.Outcome == builds.BuildFailed { - return true, e.Error - } - return false, "" -} - -// buildNeeds are the reference forms the builder's done build of module at fp is written in (the build's end says them, ADR -// 0618 §1, A-181): nil when it is not done or names none. -func (w *world) buildNeeds(module, fp string) []string { - if e, ok := w.ends.endOf(module, fp); ok && e.Outcome == builds.BuildDone { - return e.Needs - } - return nil -} - // recordReads is what node's record `.node` states its engine reads (ADR 0614 §4), and whether the assigner holds // a record of it: the fallback where the engine does not answer the verb manifest-reads (enginereads.FromRecord). func (w *world) recordReads(node string) ([]string, bool) { diff --git a/modules/mesh-delivery-assigner/internal/assign/asks.go b/modules/mesh-delivery-assigner/internal/assign/asks.go index f88290fe..77ac64f9 100644 --- a/modules/mesh-delivery-assigner/internal/assign/asks.go +++ b/modules/mesh-delivery-assigner/internal/assign/asks.go @@ -3,7 +3,6 @@ package assign import ( "fmt" "mesh-delivery-assigner/internal/assignment" - "regexp" "strings" "time" @@ -16,56 +15,12 @@ import ( oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" ) -// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (asks.HoldAskKey): keep it held or -// release it. Releasing writes the held fingerprint's want again by the rules (ADR 0539 §19). node and words are the held -// record's: the machine whose health check failed and the hold's words (issue 1906). asking numbers the writing: 0 the first, a -// larger number an ask written again after a hold lifted by itself, so the router takes it as new. -func HoldAsk(key, module, fp, node, words string, asking int, now time.Time) asks.Ask { - return assignerAsk(asking, oc.Ask{ - ID: meshdeliveryclient.AskingID(key, asking), - Headline: fmt.Sprintf("%s is held at %s", module, short(fp)), - Explanation: fmt.Sprintf("The health check of %s failed on %s at %s (%s), so the assigner holds that version and wants "+ - "no node to move to it. It lifts the hold by itself once every node that runs %s runs it healthy; releasing it "+ - "lets the wants move to it now.", module, node, short(fp), words, module), - Options: []oc.Option{ - {ID: "keep", Label: "Keep it held", Does: "the version stays held", Level: occ.Acknowledge}, - {ID: "release", Label: "Release it", Does: "the version is released and the wants move to it by the rules again", Level: occ.Acknowledge}, - }, - Who: occ.Operator, - Expires: occ.FormatTime(now.Add(AskLasts)), - OnExpiry: "the version stays held", - About: key, - }) -} - // The options of the restore ask (ADR 0537 §3, ADR 0539 §19), and what each does on the node. const ( OptionRestore = "restore" OptionLeaveStopped = "leave-stopped" ) -// RestoreAsk is the ask about a module holding data whose fingerprint failed its health check on a node (ADR 0429 §6): it is -// not set back by itself, since a set-back would lose the writes made since. words is the check's words. asking is the writing's -// number (AskingSeconds): an ask taken away as stale and written again is another asking under an id of its own, since the -// router drops a second ask under an id it keeps and the teller took the first back (issue 1966). Restore sets the -// node back to the last healthy fingerprint and loses the newer writes; leaving it stopped keeps the data as it is. -func RestoreAsk(key, node, module, fp, words string, asking int, now time.Time) asks.Ask { - return assignerAsk(asking, oc.Ask{ - ID: meshdeliveryclient.AskingID(key, asking), - Headline: fmt.Sprintf("%s failed its health check on %s", module, node), - Explanation: fmt.Sprintf("%s holds data, and its health check failed on %s at %s: %s. Setting it back to the version that ran healthy "+ - "would lose the writes made since; leaving it stopped keeps the data as it is.", module, node, short(fp), words), - Options: []oc.Option{ - {ID: OptionRestore, Label: "Restore it", Does: "the node is set back to the last healthy version and the newer writes are lost", Level: occ.Acknowledge}, - {ID: OptionLeaveStopped, Label: "Leave it stopped", Does: "the module stays as it is on the node and the data is kept", Level: occ.Acknowledge}, - }, - Who: occ.Operator, - Expires: occ.FormatTime(now.Add(AskLasts)), - OnExpiry: "the module stays as it is on the node", - About: key, - }) -} - // The options of the outside ask (design 47 G §706). const ( OptionGoOn = "go-on" @@ -106,92 +61,6 @@ func WaitsOutsideFinding(node, module, causeKey, on string, at time.Time) Observ Needs: "Nothing for you to do.", Source: "assigner.delivery-waits-outside"} } -// HeldCondition is the condition the assigner raises when it holds a fingerprint of a module (ADR 0429 §6, ADR 0539 §19), or -// clears (raised false) when the hold is released. It names the module, the node whose health check failed and the check's -// words, under the assigner's own name (ADR 0437 clause 9), keyed by the held fingerprint. needsYou is true for a module holding -// data, whose restore ask waits for the operator. -func HeldCondition(module, fp, node, words string, raised, needsYou bool) oc.Condition { - key := assignment.Module() + ".held." + module + "." + strings.TrimPrefix(fp, meshcatalogclient.SourcePrefix2) - subject := oc.ConditionSubject{Scope: "module", ID: module, Machine: node} - if !raised { - return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared} - } - c := oc.Condition{Key: key, Severity: occ.SeverityWarning, Subject: subject, State: occ.StateRaised, - Summary: fmt.Sprintf("%s is held at %s", module, short(fp)), - Headline: fmt.Sprintf("%s is held at %s", module, short(fp))} - what := fmt.Sprintf("The health check of %s failed on %s at %s: %s. The assigner holds that version and wants no node to move to it.", module, node, short(fp), words) - if needsYou { - c.Needs = "answer whether to restore " + module + " on " + node - c.Explanation = occ.NeedsYou + c.Needs + ". " + what - } else { - c.Explanation = occ.NothingToDo + " " + what + " Nodes that wanted it are set back to the version that ran healthy." - } - return c -} - -// NodeFailedCondition is the condition the assigner raises for a module whose newest fingerprint fp failed its health check on -// node while other nodes run it healthy (ADR 0557 §1, issue 1496): the failure is the node's, so nothing is held or set back and -// the node keeps its want. It names the node, the module, fp and the have's words, keyed by node and module, and is cleared -// (raised false) once the node no longer fails alone. -func NodeFailedCondition(node, module, fp string, f NodeFailure, raised bool) oc.Condition { - key := assignment.Module() + ".failed-alone." + node + "." + module - subject := oc.ConditionSubject{Scope: "module", ID: module, Machine: node} - if !raised { - return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared, - Resolved: fmt.Sprintf("%s on %s no longer fails its health check alone.", module, node)} - } - headline := fmt.Sprintf("%s fails on %s alone", module, node) - if n := len([]rune(headline)); n > occ.HeadlineLength { - headline = string([]rune(headline)[:occ.HeadlineLength]) - } - return oc.Condition{Key: key, Severity: occ.SeverityWarning, Subject: subject, State: occ.StateRaised, - Summary: headline, Headline: headline, - Explanation: fmt.Sprintf("%s The health check of %s at %s failed on %s (%s), while it runs healthy on %s. The failure is %s's, "+ - "so the assigner holds nothing and sets no node back, and %s keeps its want. This clears once %s runs it healthy or "+ - "another version is the newest.", occ.NothingToDo, module, short(fp), node, f.Words, strings.Join(f.Others, ", "), node, - node, node)} -} - -// NeitherCondition is the condition of a module assigned to node that holds no want for it (issue 1945): the node's own composition -// gives up every module with a fingerprint of the new form, so nothing applies it there and its files are gone. It is raised (urgent) -// naming the module, the node and the rule holding its first want back with the rule's words, under the assigner's own name keyed by -// node and module, and cleared (raised false) when the node has a want for it again. -func NeitherCondition(node, module string, h Holding, raised bool) oc.Condition { - key := assignment.Module() + ".neither." + node + "." + module - subject := oc.ConditionSubject{Scope: "module", ID: module, Machine: node} - if !raised { - return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared, - Resolved: fmt.Sprintf("%s on %s has a want again, so its own composition is applied there.", module, node)} - } - headline := fmt.Sprintf("%s on %s is on no list", module, node) - if n := len([]rune(headline)); n > occ.HeadlineLength { - headline = string([]rune(headline)[:occ.HeadlineLength]) - } - return oc.Condition{Key: key, Severity: occ.SeverityUrgent, Subject: subject, State: occ.StateRaised, - Summary: headline, Headline: headline, - Explanation: fmt.Sprintf("%s %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+ - "hold it either, so nothing applies it there. The assigner's rule %d holds its first want back: %s. The assigner writes the want once the "+ - "rule holds, and clears this.", occ.NothingToDo, module, node, h.Rule, h.Words)} -} - -// NeitherLeftCondition clears the neither condition of module on node when it is no longer on neither list: why is "" when the -// node has a live want for it again, else why it has none (it is held, its move waits for an ask, the node no longer runs it), so -// the clear never says a want the node does not hold (issue 1945: "nats on novox has a want again" while it had none). -func NeitherLeftCondition(node, module, why string) oc.Condition { - c := NeitherCondition(node, module, Holding{}, false) - if why != "" { - c.Resolved = fmt.Sprintf("%s on %s is no longer on neither list and has no want: %s.", module, node, why) - } - return c -} - -// LiftedCondition clears the condition of a held fingerprint the assigner lifted by itself, saying why (issue 1906). -func LiftedCondition(module, fp string) oc.Condition { - c := HeldCondition(module, fp, "", "", false, false) - c.Resolved = LiftedWords(module, fp) + "." - return c -} - // The options of the record ask (ADR 0539 §14) and of the backup ask (§16). const ( OptionYes = "yes" @@ -288,27 +157,6 @@ func NextBackupAsking(after int) int { return max(after+1, 2) } -var backupDigest = regexp.MustCompile(`\(composition (sha256:[0-9a-f]{12})\)`) - -// BackupAskDigest is the digest of the composition a backup ask was made about, from its explanation, cut to its first 12 digits as -// the ask says it (ShortDigest); empty for an ask that names -// none. An answer is taken for the composition of the ask it answers, and a new revision at the same fingerprint is asked again -// under a new number (ADR 0539 §16). -func BackupAskDigest(a asks.Ask) string { - ask, err := AskOf(a) - if err != nil { - return "" - } - if m := backupDigest.FindStringSubmatch(ask.Explanation); m != nil { - return m[1] - } - return "" -} - -// ShortDigest is a composition's digest as a backup ask names it, its first 12 digits (said): the ask stays bound to its -// composition, and a new composition is asked again. -func ShortDigest(digest string) string { return said(digest) } - // BackupAnswerWords are the words of a want carrying the answer to the asking-th ask: each asking's answer is another want // value, so a backup that fails again after "back up again" and is answered "again" again changes the key and the node retries. func BackupAnswerWords(option string, asking int) string { @@ -316,14 +164,14 @@ func BackupAnswerWords(option string, asking int) string { } // BackupAsk is the ask about a backup of a module on node that failed before an apply (ADR 0539 §16): back up again, or go on -// without a backup. digest is the composition about to be applied and asking the number of the asking, 0 the first; words are the +// without a backup, at the build fp the want names. asking is the number of the asking, 0 the first; words are the // backup's own words, from the node's have. -func BackupAsk(key, node, module, fp, digest, words string, asking int, now time.Time) asks.Ask { +func BackupAsk(key, node, module, fp, words string, asking int, now time.Time) asks.Ask { return assignerAsk(asking, oc.Ask{ ID: meshdeliveryclient.AskingID(key, asking), Headline: fmt.Sprintf("The backup of %s failed on %s", module, node), - Explanation: fmt.Sprintf("The backup before applying %s at %s on %s failed: %s. %s keeps running what it has until you answer. (composition %s)", - module, short(fp), node, words, module, digest), + Explanation: fmt.Sprintf("The backup before applying %s at %s on %s failed: %s. %s keeps running what it has until you answer.", + module, short(fp), node, words, module), Options: []oc.Option{ {ID: OptionBackupAgain, Label: "Back up again", Does: "the node tries the backup again before it applies", Level: occ.Acknowledge}, {ID: OptionBackupSkip, Label: "Go on without a backup", Does: "the node applies the new version without a backup", Level: occ.Acknowledge}, diff --git a/modules/mesh-delivery-assigner/internal/assign/asks_test.go b/modules/mesh-delivery-assigner/internal/assign/asks_test.go index d6ff0f23..78a9d0f6 100644 --- a/modules/mesh-delivery-assigner/internal/assign/asks_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/asks_test.go @@ -23,11 +23,10 @@ import ( "git.novox.be/novox/mesh-sdk/go/views" ) -// 1297/B9 (the assigner's half): the ask about a held fingerprint is a valid ask of the router and of mesh-sdk's layout at -// its key, of a kind the view may answer. -func TestTheHoldAskIsValid(t *testing.T) { - t.Run("hold", theHoldAskIsValid) - t.Run("restore", theRestoreAskIsValid) +// 1297/B9 (the assigner's half): the backup ask is a valid ask of the router and of mesh-sdk's layout at its key, of a kind the +// view may answer; the hold and restore asks went with the hold of a fingerprint (design 47 H §750, H-A-55). +func TestTheAssignersAsks(t *testing.T) { + t.Run("backup", theBackupAskIsValid) t.Run("ended at the channel", anAskEndedAtTheChannelIsWrittenAgain) t.Run("ended at the channel, read at bind", anAskEndedAtTheChannelIsReadAtBind) } @@ -40,8 +39,6 @@ func anAskEndedAtTheChannelIsWrittenAgain(t *testing.T) { asked := time.Date(2026, 11, 9, 22, 0, 0, 0, time.UTC) now := asked.Add(8 * 24 * time.Hour) digest := "sha256:" + strings.Repeat("e", 64) - holdKey, _ := asks.HoldAskKey("nats", fNew) - restoreKey, _ := asks.RestoreAskKey("novox", "nats", fNew) backupKey, _ := asks.BackupAskKey("novox", "nats", fNew) revertKey, _ := asks.RevertAskKey("novox", "nats", fNew) recordKey, _ := asks.RecordAskKey("novox", "nats", digest) @@ -63,9 +60,7 @@ func anAskEndedAtTheChannelIsWrittenAgain(t *testing.T) { put *puts.Put why string // "" when it is written again }{ - {name: "hold", key: holdKey, a: ptr(HoldAsk(holdKey, "nats", fNew, "novox", "w", AskingSeconds(asked), asked)), outcome: occ.OutcomeExpired}, - {name: "restore", key: restoreKey, a: ptr(RestoreAsk(restoreKey, "novox", "nats", fNew, "w", AskingSeconds(asked), asked)), outcome: occ.OutcomeCancelled}, - {name: "backup", key: backupKey, a: ptr(BackupAsk(backupKey, "novox", "nats", fNew, digest, "w", 0, asked)), outcome: occ.OutcomeCancelled}, + {name: "backup", key: backupKey, a: ptr(BackupAsk(backupKey, "novox", "nats", fNew, "w", 0, asked)), outcome: occ.OutcomeCancelled}, {name: "revert", key: revertKey, a: ptr(assignerAsk(AskingSeconds(asked), oc.Ask{ID: meshdeliveryclient.AskingID(revertKey, AskingSeconds(asked)), Headline: "h", Explanation: "e", Who: occ.Operator, Expires: occ.FormatTime(asked.Add(AskLasts)), OnExpiry: "o", Options: []oc.Option{{ID: "hold", Label: "Hold", Does: "d", Level: occ.Acknowledge}}})), outcome: occ.OutcomeExpired}, @@ -73,11 +68,11 @@ func anAskEndedAtTheChannelIsWrittenAgain(t *testing.T) { {name: "record of another put", key: recordKey, a: &record, outcome: occ.OutcomeExpired, put: &other}, {name: "record a put asked", key: recordKey, a: &byPut, outcome: occ.OutcomeExpired, put: &put, why: "put"}, {name: "assignments", key: assignmentsKey, a: &assignments, outcome: occ.OutcomeExpired, why: "ADR 0567"}, - {name: "another asking stands", key: holdKey, a: ptr(HoldAsk(holdKey, "nats", fNew, "novox", "w", AskingSeconds(now), now)), - ended: meshdeliveryclient.AskingID(holdKey, AskingSeconds(asked)), outcome: occ.OutcomeExpired, why: "holds the asking"}, - {name: "taken away", key: holdKey, ended: "hold-x", outcome: occ.OutcomeExpired, why: "taken away"}, - {name: "refused", key: holdKey, a: ptr(HoldAsk(holdKey, "nats", fNew, "novox", "w", 0, asked)), outcome: occ.OutcomeRefused, why: "refused"}, - {name: "replaced", key: holdKey, a: ptr(HoldAsk(holdKey, "nats", fNew, "novox", "w", 0, asked)), outcome: occ.OutcomeReplaced, why: "replaced"}, + {name: "another asking stands", key: backupKey, a: ptr(BackupAsk(backupKey, "novox", "nats", fNew, "w", AskingSeconds(now), now)), + ended: meshdeliveryclient.AskingID(backupKey, AskingSeconds(asked)), outcome: occ.OutcomeExpired, why: "holds the asking"}, + {name: "taken away", key: backupKey, ended: "backup-x", outcome: occ.OutcomeExpired, why: "taken away"}, + {name: "refused", key: backupKey, a: ptr(BackupAsk(backupKey, "novox", "nats", fNew, "w", 0, asked)), outcome: occ.OutcomeRefused, why: "refused"}, + {name: "replaced", key: backupKey, a: ptr(BackupAsk(backupKey, "novox", "nats", fNew, "w", 0, asked)), outcome: occ.OutcomeReplaced, why: "replaced"}, } { t.Run(c.name, func(t *testing.T) { var current []byte @@ -134,12 +129,12 @@ func anAskEndedAtTheChannelIsWrittenAgain(t *testing.T) { // answered or another asker's ask, and one the key no longer holds, are not. func anAskEndedAtTheChannelIsReadAtBind(t *testing.T) { asked := time.Date(2026, 11, 9, 22, 0, 0, 0, time.UTC) - holdKey, _ := asks.HoldAskKey("nats", fNew) - restoreKey, _ := asks.RestoreAskKey("novox", "nats", fNew) backupKey, _ := asks.BackupAskKey("novox", "nats", fNew) - hold := HoldAsk(holdKey, "nats", fNew, "novox", "w", AskingSeconds(asked), asked) - restore := RestoreAsk(restoreKey, "novox", "nats", fNew, "w", AskingSeconds(asked.Add(time.Hour)), asked.Add(time.Hour)) - backup := BackupAsk(backupKey, "novox", "nats", fNew, "sha256:"+strings.Repeat("e", 64), "w", 0, asked.Add(-time.Hour)) // no asking + holdKey, _ := asks.BackupAskKey("g14", "nats", fNew) + restoreKey, _ := asks.BackupAskKey("ace", "nats", fNew) + hold := BackupAsk(holdKey, "g14", "nats", fNew, "w", AskingSeconds(asked), asked) + restore := BackupAsk(restoreKey, "ace", "nats", fNew, "w", AskingSeconds(asked.Add(time.Hour)), asked.Add(time.Hour)) + backup := BackupAsk(backupKey, "novox", "nats", fNew, "w", 0, asked.Add(-time.Hour)) // no asking held := map[string][]byte{} for key, a := range map[string]asks.Ask{holdKey: hold, restoreKey: restore, backupKey: backup} { v, err := EncodeAsk(key, a) @@ -174,27 +169,27 @@ func anAskEndedAtTheChannelIsReadAtBind(t *testing.T) { func ptr(a asks.Ask) *asks.Ask { return &a } -func theHoldAskIsValid(t *testing.T) { - key, err := asks.HoldAskKey("mail", fNew) +func theBackupAskIsValid(t *testing.T) { + key, err := asks.BackupAskKey("ace", "mail", fNew) if err != nil { t.Fatal(err) } now := time.Now() - a := HoldAsk(key, "mail", fNew, "ace", "health failed, phase running: connection refused", 0, now) + a := BackupAsk(key, "ace", "mail", fNew, "health failed, phase running: connection refused", 0, now) if err := occ.Check(mustAsk(t, a), now); err != nil { t.Errorf("the router refuses the ask: %v", err) } if _, err := EncodeAsk(key, a); err != nil { t.Errorf("the ask cannot be written: %v", err) } - // 1906/H3: the ask names the machine whose check failed and the hold's words. + // The ask names the machine whose backup failed and the backup's words. if !strings.Contains(mustAsk(t, a).Explanation, "on ace") || !strings.Contains(mustAsk(t, a).Explanation, "connection refused") { t.Errorf("the ask does not name the node and the hold's words: %q", mustAsk(t, a).Explanation) } - // 1906/H2: an ask written again after a lift is another asking, under an id of its own. - again := HoldAsk(key, "mail", fNew, "ace", "x", 7, now) + // An ask written again is another asking, under an id of its own. + again := BackupAsk(key, "ace", "mail", fNew, "x", 7, now) if again.Asking != 7 || mustAsk(t, again).ID != meshdeliveryclient.AskingID(key, 7) || mustAsk(t, again).ID == mustAsk(t, a).ID { - t.Errorf("the asking after a lift = %d %s", again.Asking, mustAsk(t, again).ID) + t.Errorf("the asking again = %d %s", again.Asking, mustAsk(t, again).ID) } if _, err := EncodeAsk(key, again); err != nil { t.Errorf("the ask asked again cannot be written: %v", err) @@ -204,7 +199,7 @@ func theHoldAskIsValid(t *testing.T) { } // The join (A1d ruling 1): mesh-sdk no longer refuses another id, no option or no ask in an assigner's ask; EncodeAsk does. t.Run("EncodeAsk refuses what mesh-sdk no longer does", func(t *testing.T) { - good := HoldAsk(key, "mail", fNew, "ace", "x", 0, time.Unix(100, 0)) + good := BackupAsk(key, "ace", "mail", fNew, "x", 0, time.Unix(100, 0)) other := good other.Asking = 3 // its id is the asking 0's empty := good @@ -219,68 +214,11 @@ func theHoldAskIsValid(t *testing.T) { }) } -// 1906/D3 (ADR 0539 §19): the ask about a data module held on a node is a valid ask of the router and of mesh-sdk's layout at -// its restore key, of a kind the view may answer; it offers restoring (losing the newer writes) and leaving it stopped. -func theRestoreAskIsValid(t *testing.T) { - key, err := asks.RestoreAskKey("ace", "mail", fNew) - if err != nil { - t.Fatal(err) - } - now := time.Now() - a := RestoreAsk(key, "ace", "mail", fNew, "connection refused", 2, now) - if err := occ.Check(mustAsk(t, a), now); err != nil { - t.Errorf("the router refuses the ask: %v", err) - } - if _, err := EncodeAsk(key, a); err != nil { - t.Errorf("the ask cannot be written: %v", err) - } - if _, err := viewstate.ViewAnswerKey(key); err != nil { - t.Errorf("the view may not answer it: %v", err) - } - var ids []string - for _, o := range mustAsk(t, a).Options { - ids = append(ids, o.ID) - } - if len(ids) != 2 || ids[0] != OptionRestore || ids[1] != OptionLeaveStopped { - t.Errorf("options = %v", ids) - } - if !strings.Contains(mustAsk(t, a).Explanation, "connection refused") || !strings.Contains(mustAsk(t, a).Explanation, "ace") { - t.Errorf("the ask does not name the node and the check's words: %q", mustAsk(t, a).Explanation) - } -} - -// 1906/D3: the condition for a held fingerprint names the module, the node and the check's words, under the assigner's own -// name (ADR 0437 clause 9), and is cleared under the same key. +// 1952/WM6: the assigner's conditions are findings. func TestTheAssignersConditions(t *testing.T) { - t.Run("the held condition names module, node and words", heldConditionNamesModuleNodeAndWords) - t.Run("a lifted hold clears its condition saying why", liftedHoldClearsItsConditionSayingWhy) t.Run("1952/WM6: each said condition is a finding, its clear deletes it, said straight only with no findings bucket", saidConditionsAreFindings) } -func heldConditionNamesModuleNodeAndWords(t *testing.T) { - c := HeldCondition("mail", fNew, "ace", "connection refused", true, true) - if _, err := occ.EncodeCondition(c); err != nil { - t.Fatalf("the condition is refused: %v", err) - } - if c.State != occ.StateRaised || c.Subject.ID != "mail" || c.Subject.Machine != "ace" || !strings.Contains(c.Explanation, "connection refused") || !strings.Contains(c.Key, "mail") { - t.Errorf("condition = %+v", c) - } - if got := HeldCondition("mail", fNew, "", "", false, false); got.State != occ.StateCleared || got.Key != c.Key { - t.Errorf("cleared = %+v", got) - } -} - -// 1906/H2: a hold lifted by itself clears the held condition under its key, saying why. -func liftedHoldClearsItsConditionSayingWhy(t *testing.T) { - c := LiftedCondition("mail", fNew) - if _, err := occ.EncodeCondition(c); err != nil { - t.Fatalf("the condition is refused: %v", err) - } - if c.State != occ.StateCleared || c.Key != HeldCondition("mail", fNew, "ace", "x", true, false).Key || !strings.Contains(c.Resolved, "lifted its hold by itself") { - t.Errorf("condition = %+v", c) - } -} - // 1952/WM6 (ADR 0564 §2, issue 1952 note 16): every condition the assigner says is a finding in its own bucket, under // `..`, written at the raise and deleted at the clear, and only as a finding: the keeper is the one // source since the controller's end (novox/hq ADR 0590 §3, issue 1126 B8). Said straight to the router only where the @@ -291,9 +229,6 @@ func saidConditionsAreFindings(t *testing.T) { raised, cleared oc.Condition key string }{ - {HeldCondition("mail", fNew, "ace", "refused", true, true), LiftedCondition("mail", fNew), "module.mail." + strings.TrimPrefix(fNew, meshcatalogclient.SourcePrefix2) + ".held"}, - {NodeFailedCondition("ace", "mail", fNew, NodeFailure{Words: "x", Others: []string{"g14"}}, true), NodeFailedCondition("ace", "mail", fNew, NodeFailure{}, false), "module.ace.mail.failed-alone"}, - {NeitherCondition("ace", "mail", Holding{Rule: 3, Words: "w"}, true), NeitherCondition("ace", "mail", Holding{}, false), "module.ace.mail.neither"}, {SeatWaitsCondition([]string{"assignments"}), SeatWaitsCondition(nil), "module.mesh-delivery-assigner.seat-states"}, {BackupCondition("ace", "mail", fNew, "disk full", true), BackupCondition("ace", "mail", fNew, "", false), "module.ace.mail." + strings.TrimPrefix(fNew, meshcatalogclient.SourcePrefix2) + ".backup"}, } { @@ -336,8 +271,7 @@ func TestTheRecordAndBackupAsksAreValid(t *testing.T) { if err != nil { t.Fatal(err) } - dgst := "sha256:" + strings.Repeat("b", 64) - b := BackupAsk(bkey, "ace", "mail", fNew, dgst, "no space left", 0, now) + b := BackupAsk(bkey, "ace", "mail", fNew, "no space left", 0, now) vkey, err := asks.RevertAskKey("ace", "mail", fNew) if err != nil { t.Fatal(err) @@ -360,18 +294,15 @@ func TestTheRecordAndBackupAsksAreValid(t *testing.T) { if !strings.Contains(mustAsk(t, b).Explanation, "no space left") || mustAsk(t, b).Options[0].ID != mwc.BackupAgain || mustAsk(t, b).Options[1].ID != mwc.BackupSkip { t.Errorf("the backup ask = %+v", mustAsk(t, b)) } - // the id binds the ask to the composition and to its number (review round 1, items 3 and 4) - if again := BackupAsk(bkey, "ace", "mail", fNew, dgst, "x", NextBackupAsking(0), now); mustAsk(t, again).ID == mustAsk(t, b).ID { + // the id binds the ask to its number (review round 1, item 4); its key binds it to the build (design 47 H §750: a want names no composition) + if again := BackupAsk(bkey, "ace", "mail", fNew, "x", NextBackupAsking(0), now); mustAsk(t, again).ID == mustAsk(t, b).ID { t.Errorf("asked again under the same id") } if NextBackupAsking(-1) != 0 || NextBackupAsking(0) != 2 || NextBackupAsking(2) != 3 { t.Errorf("the numbers of the askings: %d %d %d", NextBackupAsking(-1), NextBackupAsking(0), NextBackupAsking(2)) } - if BackupAskDigest(b) != ShortDigest(dgst) || ShortDigest(dgst) == dgst { - t.Errorf("the ask names composition %q, want %q", BackupAskDigest(b), ShortDigest(dgst)) - } // novox 2026-11-16: the router refuses a long hexadecimal string, so no word of an ask carries a digest or fingerprint whole. - held := BackupAsk(bkey, "novox", "mail", fNew, dgst, "the kept data between "+fNew+" and "+fNew+" is not known", 0, now) + held := BackupAsk(bkey, "novox", "mail", fNew, "the kept data between "+fNew+" and "+fNew+" is not known", 0, now) if a := mustAsk(t, held); regexp.MustCompile(`[0-9a-f]{20,}`).MatchString(a.Headline + a.Explanation + a.Details) { t.Errorf("the ask carries a long hexadecimal string: %q", a.Explanation) } @@ -401,6 +332,7 @@ func TestTheRecordAndBackupAsksAreValid(t *testing.T) { // ADR 0546 clauses 7 and 8: a put's ask is valid at the record ask's key; for the bus its yes approves and binds the act, // so the view cannot answer it, and it lasts no longer than an approving ask may. put := puts.Put{FP: fNew, Why: "the bus's user list", Hold: true} + dgst := "sha256:" + strings.Repeat("b", 64) d := ComposedDigest{Digest: dgst, Judgement: JudgedCannot, Words: "the have names no composition"} for _, bus := range []bool{false, true} { a, err := PutAsk(rkey, "ace", "mail", put, d, bus, now) @@ -445,22 +377,6 @@ func TestTheRecordAndBackupAsksAreValid(t *testing.T) { } } -// 1945/AC21: the condition of a module on neither list is urgent, names the module, the node and the rule holding its want back, -// under the assigner's own name keyed by node and module, and is cleared under the same key. -func TestTheNeitherConditionNamesModuleNodeAndRule(t *testing.T) { - c := NeitherCondition("g14", "i3status-rust", Holding{Rule: mwc.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}, true) - if _, err := occ.EncodeCondition(c); err != nil { - t.Fatalf("the condition is refused: %v", err) - } - if c.State != occ.StateRaised || c.Severity != occ.SeverityUrgent || c.Subject.ID != "i3status-rust" || c.Subject.Machine != "g14" || - !strings.Contains(c.Explanation, "x11-display") || !strings.Contains(c.Explanation, "rule 5") || c.Key != "mesh-delivery-assigner.neither.g14.i3status-rust" { - t.Errorf("condition = %+v", c) - } - if got := NeitherCondition("g14", "i3status-rust", Holding{}, false); got.State != occ.StateCleared || got.Key != c.Key { - t.Errorf("cleared = %+v", got) - } -} - // mustAsk is the router's ask an asks-state value carries. func mustAsk(t *testing.T, a asks.Ask) oc.Ask { t.Helper() diff --git a/modules/mesh-delivery-assigner/internal/assign/policy.go b/modules/mesh-delivery-assigner/internal/assign/policy.go index 248346ae..38911e1a 100644 --- a/modules/mesh-delivery-assigner/internal/assign/policy.go +++ b/modules/mesh-delivery-assigner/internal/assign/policy.go @@ -6,13 +6,11 @@ import ( "mesh-delivery-assigner/internal/assignment" "mesh-delivery-assigner/internal/buckets" "sort" - "strings" "time" mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" - "git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient" "git.novox.be/novox/mesh-sdk/go/catalogue" ) @@ -48,66 +46,15 @@ func PolicyOf(m catalogue.Manifest, chosen map[string]catalogue.PersonPolicy) (p return catalogue.Upgrade(m, c) } -func contains(list []string, s string) bool { - for _, v := range list { - if v == s { - return true - } - } - return false -} - // KindProvisionFailed is the assigner's finding of a provision its module's want waits for that failed (ADR 0570 §4). const KindProvisionFailed = "provision-failed" -// ProvisionFailedKey is a failed provision's finding's key, `provider...[.].provision-failed`. -func ProvisionFailedKey(key string) string { - return mcc.Key(mcc.ScopeProvider, key, KindProvisionFailed) -} - -// ProvisionFailedFinding is the finding of a failed provision, naming its consumer, its provider and why. -func ProvisionFailedFinding(key string, p meshprovisionerclient.Provision, at time.Time) (mcseat.Finding, error) { - summary := fmt.Sprintf("%s for %s on %s failed at %s on %s: %s, so %s is not moved there until it is provided", p.Provision, - p.Consumer.Module, p.Consumer.Node, p.Provider.Module, p.Provider.Node, p.Why, p.Consumer.Module) - return encode(Observed{Scope: mcc.ScopeProvider, ID: key, Kind: KindProvisionFailed, Machine: p.Consumer.Node, Also: also(p.Provider.Node, p.Consumer.Node), - Summary: summary, Said: p.Why, Headline: clipHeadline(p.Consumer.Module + " waits for a provision that failed"), - Explanation: occ.NothingToDo + " " + summary + ". It clears once the provider provides it.", - Source: "assigner." + KindProvisionFailed}, at) -} - // encode is the observation as the assigner's finding at at. func encode(o Observed, at time.Time) (mcseat.Finding, error) { return o.Finding(at), nil } -func also(provider, consumer string) []string { - if provider == "" || provider == consumer { - return nil - } - return []string{provider} -} - // KindProvisionMissing is the assigner's finding of a required provision that has had no key for longer than its bound. const KindProvisionMissing = "provision-missing" -// ProvisionMissingKey is its key, `provider....provision-missing`. -func ProvisionMissingKey(key string) string { - return mcc.Key(mcc.ScopeProvider, key, KindProvisionMissing) -} - -// ProvisionMissingFinding is the finding of a required provision with no key past bound (key `..`). -func ProvisionMissingFinding(key string, bound time.Duration, at time.Time) (mcseat.Finding, error) { - parts := strings.SplitN(key, ".", 3) - if len(parts) != 3 { - return mcseat.Finding{}, fmt.Errorf("%q is not ..", key) - } - node, module, provision := parts[0], parts[1], parts[2] - summary := fmt.Sprintf("%s on %s requires %s and mesh-provisioner has composed no key for it in %s, so %s is not moved there: "+ - "no provider of it may be placed", module, node, provision, bound, module) - return encode(Observed{Scope: mcc.ScopeProvider, ID: key, Kind: KindProvisionMissing, Machine: node, - Summary: summary, Headline: clipHeadline(module + " on " + node + " lacks " + provision), - Explanation: occ.NothingToDo + " " + summary + ". It clears once the provision has its key.", - Source: "assigner." + KindProvisionMissing}, at) -} - // KindUnplaced is, from the lease write, a node that runs but no assignments names (ADR 0571 §6). const KindUnplaced = "unplaced" diff --git a/modules/mesh-delivery-assigner/internal/assign/put.go b/modules/mesh-delivery-assigner/internal/assign/put.go index da2515f2..7a9f9635 100644 --- a/modules/mesh-delivery-assigner/internal/assign/put.go +++ b/modules/mesh-delivery-assigner/internal/assign/put.go @@ -212,9 +212,9 @@ func needsPerson(w PutWorld, d ComposedDigest) bool { return d.Judgement == JudgedRecreates || d.Judgement == JudgedMovesData || d.Judgement == JudgedCannot } -// PutWant is the want a put writes once it is wanted: the fingerprint and composition, marked put, and held when it holds. +// PutWant is the want a put writes once it is wanted: the fingerprint, marked put, and held when it holds. func PutWant(p puts.Put) mwc.Want { - return mwc.Want{Fp: p.FP, Composed: p.Composed, Put: true, Hold: p.Hold, Words: "put: " + p.Why} + return mwc.Want{Fp: p.FP, Put: true, Hold: p.Hold, Words: "put: " + p.Why} } // PutOwns says what a put does to the node's want: owns says the rules pass over it, want the want it writes (nil: what diff --git a/modules/mesh-delivery-assigner/internal/assign/verbs.go b/modules/mesh-delivery-assigner/internal/assign/verbs.go deleted file mode 100644 index a6d806f5..00000000 --- a/modules/mesh-delivery-assigner/internal/assign/verbs.go +++ /dev/null @@ -1,38 +0,0 @@ -package assign - -import ( - "slices" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph" -) - -// verbWaiting says why the module's want on node waits for a verb it invokes (novox/hq ADR 0614 §2): the first seat verb or -// module tool of Invokes (the module graph's verb edges, those it leaves out on a cycle not among them, §3) that no -// running healthy build serves. Nil Served (nothing read of what runs) waits for none, and a verb a running build serves -// already adds no wait. It holds under the provisions' rule (RuleProvisions): no reader knows a rule number of its own -// yet, so the words name the edge (graph.VerbWaitWords, which the delivery view reads). -func (m Module) verbWaiting(node string) (string, bool) { - if m.Served == nil { - return "", false - } - for _, v := range slices.Sorted(slices.Values(m.Invokes)) { - if !m.Served(v) { - return graph.VerbWaitWords(v, m.Name, node), true - } - } - return "", false -} - -// busStateWaiting says why the module's want on node waits for bus state (ADR 0614 §5): its manifest asks bus state (a -// bus-state entry, as the module graph reads it) and no build whose manifest offers bus-state runs healthy on a node -// present. The provision bus-state is answered with no credential of a consumer's own, so ADR 0603 §5 counts the -// provider's own have and no provisioner key is waited for, handed or not. Nil Offering (nothing read of what runs) waits -// for none, and neither does a module that offers bus-state itself. It holds under the provisions' rule, its words naming bus state (graph.BusStateWaitWords). -func (m Module) busStateWaiting(node string) (string, bool) { - if len(m.Manifest.BusState) == 0 || m.Offering == nil || m.Offering(graph.BusState) || - slices.Contains(m.Manifest.Offers(), graph.BusState) { // the bus module offering it waits on nobody for it - - return "", false - } - return graph.BusStateWaitWords(m.Name, node), true -} diff --git a/modules/mesh-delivery-assigner/internal/assign/wants.go b/modules/mesh-delivery-assigner/internal/assign/wants.go index e1e2acc7..e4fca6f8 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants.go @@ -2,324 +2,75 @@ package assign import ( "fmt" + "slices" + "strings" + "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey" - "git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate" - "maps" - "mesh-delivery-assigner/internal/puts" - "slices" - "sort" - "strings" - "time" - - "git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient" "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds/forms" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph" + "mesh-delivery-assigner/internal/puts" ) -// Module is what the six rules read of one module (ADR 0429 §6, ADR 0528 clause 3): its newest fingerprint F at the -// trunk (the indexer's modules), whether the builder's build of F is done, whether F is held, whether the controller -// issued its credential for F, its manifest at F, and the nodes it is assigned to. +// A want names the newest released version whose pipeline passed (design 47 H §750): for every node whose assignment places +// a module on a watched branch, the want names the version, the pipeline and the mover's rank in it, written tier by tier +// (§751). It is held for nothing else: the rules of ADR 0429 §6 (the build done, not held, the canary and step, every +// provider healthy, the grant, the composition and the reader capabilities, the provisions, the served verbs and the bus +// state), the network hold and A-428's exceptions end (H-A-55). A put owns its node's want as before (ADR 0546), and the +// operator's yes on a module of policy record is the node-engine's have and the asker's (the pass's yeses). + +// Target is the build a node's want names: its fingerprint and released version, with the pipeline whose passed end released +// it and the mover's rank in that pipeline; a placement's build names no pipeline and no rank (design 47 H §750, H-A-37). +type Target struct { + Fp string + Version string + Pipeline string + Rank int64 +} + +// Module is what the want rule reads of one module: the nodes it is assigned to, the build each node's want names, the puts +// that own a node's want and the nodes that compose it themselves. type Module struct { - Name string - FP string - Built bool - // BuildFailed and BuildError say a build that ended failed, with the error as the builder's `build.ended` carries it whole - // (design 47 G §575, A-162): the words of a hold under rule 1. - BuildFailed bool - BuildError string - Held bool - Granted bool - // GrantedAt says whether the controller issued the module's credential for another fingerprint than FP: the first want of a node - // that runs an older one names it (issue 1945). Nil: none. - GrantedAt func(fp string) bool - // GrantedOn says whether the controller's grant of FP names the module's accounts on a node, when the grant names any: - // a grant written for the module's other nodes says nothing of a node whose account is not issued and sealed (issue - // 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus). Nil: the grant alone. - GrantedOn func(node string) bool - Manifest catalogue.Manifest - // Provisions is the seat mesh-provisioner's provisions this module consumes on a node, by key (ADR 0570 §4): the want - // waits until each is provided. Nil: the seat's state is not read (not delivered yet), and nothing waits on it. - Provisions func(node string) map[string]meshprovisionerclient.Provision - // Answering is the manifests of the modules placed somewhere that provide a provision: what catalogue.Awaited - // judges a want's provisions by (issue 774). Nil: none known, and no key is waited for that is not there. - Answering func(provision string) []catalogue.Manifest - // Offered says whether any module offers a provision by name or by its own name (catalogue.Manifest.Offers) among every - // manifest the trunk holds and the modules the mesh provides itself, placed or not. Nil: not known, and no requirement - // is waited on for want of an offer. - Offered func(provision string) bool - // Invokes are the seat verbs and module tools the module graph says the module invokes and keeps as edges to the - // modules serving them, those it leaves out on a cycle not among them (ADR 0614 §2, §3): the want waits until each is - // served (Served). Nil: no graph node is read, and no want waits on a verb. - Invokes []string - // Served says whether a build serving a seat verb or module tool runs healthy, counted from its holder's have (ADR 0614 - // §2, as ADR 0603 §5 counts a provider's). Nil: nothing is read of what runs, and no want waits on a verb. - Served func(invoke string) bool - // Offering says whether a build whose manifest offers a provision runs healthy on a node present, counted from the - // provider's own have (ADR 0603 §5): what a want asking bus state waits for (ADR 0614 §5). Nil: nothing is read of what - // runs, and no want waits on bus state. - Offering func(provision string) bool + Name string + Manifest catalogue.Manifest // Chosen is a person's choice of the module's upgrade policy (the assigner's `policies`, ADR 0565 §7); nil: none. Chosen *catalogue.PersonPolicy Nodes []string - // Composed answers the composition of the module for a node at a fingerprint: the digest the node's own have names (a - // placed node composes its own, ADR 0571 §6), and whether it names one. Nil: none exists (ADR 0539 §1, §6). - Composed func(node, fp string) (ComposedDigest, bool) + // Target is the build node's want names: the newest version of the module released on the branch the node's placement + // names whose pipeline passed and whose tier is written (§750, §751), or the placement's build (H-A-5); false: none yet, + // and the want written stays. + Target func(node string) (Target, bool) // Allowed says whether the operator answered yes to the record ask about the move of this module on node to the composition // with this digest (ADR 0539 §14). Nil: none did. Allowed func(node, digest string) bool - // Released is a hold on FP the operator released (ADR 0539 §19): FP is not held again by a health check that failed. - Released bool - // Restored says whether the operator answered "restore" to the restore ask of this data module on node: it is then set back - // as any module is, losing the writes made since (ADR 0539 §19). Nil: none did. - Restored func(node string) bool - // Kept says the operator answered "keep" to the hold ask of FP: a hold kept so is not lifted by itself. - Kept bool - // Now is when this pass reads the haves: the held record names it as when the health check was seen to fail. Zero: not said. - Now time.Time - // Puts are the puts of the module, by node (ADR 0546): while one owns a node's want (PutOwns), the rules pass over it. + // Puts are the puts of the module, by node (ADR 0546): while one owns a node's want (PutOwns), the rule passes over it. Puts map[string]puts.Put - // Needs are the reader capabilities the manifest at FP needs (the indexer's modules value, catalogue.ManifestNeeds, ADR - // 0614 §4): a node's want moves only while that node's engine states each of them. Nil: none. - Needs []string - // FormNeeds are the reference forms the module's build at FP is written in (`form:`, the builder's build key's - // needs, ADR 0618 §1): a node's want is held only while that node's engine reads none of them (ADR 0624 §2). Nil: the - // build names none, and holds nothing. - FormNeeds []string - // Self says whether node composes the module itself: the catalogue's assignments names the node (ADR 0547 §2). It is the one - // source of a want's Self: every want the rules write or keep is derived from it (selfDerived), so a want written Self while - // the node was placed does not outlive the assignments (issue 1126). Nil: no node does. + // Self says whether node composes the module itself: the catalogue's assignments names the node (ADR 0547 §2). Nil: no + // node does. Self func(node string) bool - // KeptBuilds answers what is known of the build at a fingerprint, its released version among it (design 47 G §379, A-96). Nil: - // none is read. - KeptBuilds func(fp string) (KeptBuild, bool) - // MachineHeld says whether the node's own network condition is open, which holds the machine as a whole: no want moves to it - // until that clears (design 47 G §706 *M's own network*). Nil: none is held. - MachineHeld func(node string) bool } -// KeptBuild is what the assigner knows of a build: the digest of each of its parts by `:` and the build node that -// keeps it, as the builder's `build.ended` says them, by which its released version is found; for a trunk build the released version -// the catalog's `version-released` says (design 47 G §379, A-96). A want names the version alone (design 47 H §750). -type KeptBuild struct { - Parts map[string]string - KeptOn string - Version string -} - -// withKept is w naming the released version of the build at its fingerprint; a want names a version, never a part's digest or a -// keeping node (design 47 H §750, §764: A-436's digests end, G §379's go), so a build with no version known is named by its -// fingerprint alone. -func (n Nodes) withKept(m Module, node string, w mwc.Want) mwc.Want { - if m.KeptBuilds == nil { - return w - } - if k, ok := m.KeptBuilds(w.Fp); ok && k.Version != "" { - w.Version = k.Version - } - return w -} - -// NameKept is w naming the build at its fingerprint as the want of a module does (withKept), for a want held as it stands. -func (n Nodes) NameKept(m Module, node string, w mwc.Want) mwc.Want { return n.withKept(m, node, w) } - func (m Module) selfOn(node string) bool { return m.Self != nil && m.Self(node) } -func (m Module) composition(node, fp string) (ComposedDigest, bool) { - if m.Composed == nil { - return ComposedDigest{}, false - } - return m.Composed(node, fp) -} - -// Node is what the rules read of the nodes: whether each is online (the watchman's presence) and each module's have. +// Nodes is what the pass reads of the nodes: whether each is online (the watchman's presence), each module's have and the +// node-engine seat's last `running`. type Nodes struct { Online map[string]bool Haves map[string]map[string]nodeengine.HaveByNodeModuleValue - // Running is, by node and module, the last `running` the node-engine seat said (design 47 G §309): a provider runs healthy - // while the build it runs does, as this says it, whatever a newer want of it waits for (rule 5, main file *A provider - // before its consumers*). Nil: the seat's events are not heard, and the have says it. + // Running is, by node and module, the last `running` the node-engine seat said (design 47 G §309). Nil: the seat's events + // are not heard. Running map[string]map[string]nodeengine.Running // Providers is, by provision, the modules that provide it, with the nodes each is assigned to. Providers map[string]map[string][]string - // Reads is, by node, the reader capabilities its node-engine states in its node record (nodeengine.NodeRecord.ManifestReads, - // ADR 0614 §4). A node with none, or none read, is read as stating the baseline (catalogue.StatedManifestReads). + // Reads is, by node, the reader capabilities its node-engine states (nodeengine.NodeRecord.ManifestReads, ADR 0614 §4). Reads map[string][]string } -// healthy says whether a have runs fp healthy. -func healthy(h nodeengine.HaveByNodeModuleValue, fp string) bool { - return h.Fp == fp && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK -} - // Failed says whether a have's health check failed once the node is done moving. func Failed(h nodeengine.HaveByNodeModuleValue) bool { return failed(h) } -// Canary is the module's canary: the first of its nodes, by name, that is online (ADR 0435 §3; one ring, ordered by -// name, ADR 0528 clause 3). Empty when none is online. -func Canary(nodes []string, online map[string]bool) string { - sorted := slices.Sorted(slices.Values(nodes)) - for _, n := range sorted { - if online[n] { - return n - } - } - return "" -} - -// Holds is whether every rule holds for moving node's want for m to m.FP, and if not, the first rule that does not, -// with why (ADR 0429 §6). Rule 4, at most `step` others ahead, always holds: the mesh keeps no step, and step 0 is no -// limit (ADR 0528 clause 3). Rule 5, every provider healthy, holds no want (issue 1960): a module whose provider is down -// is wanted all the same, and said as not reached (Unreached), since a site runs while the proxy in front of it is -// down and one module never blocks another. -func Holds(m Module, node string, n Nodes) (bool, int, string) { - return holds(m, node, n, true) -} - -// holds is Holds; with ordering false it leaves out the canary (3), which orders a move among nodes, and keeps those -// without which there is nothing to want: the build done, not held, the grant and the composition. -func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) { - switch { - case !m.Built: - if m.BuildFailed { - return false, mwc.RuleBuildDone, fmt.Sprintf("the build of %s at %s failed: %s", m.Name, short(m.FP), m.BuildError) - } - return false, mwc.RuleBuildDone, fmt.Sprintf("the build of %s at %s is being built", m.Name, short(m.FP)) - case m.Held: - return false, mwc.RuleNotHeld, fmt.Sprintf("%s is held at %s", m.Name, short(m.FP)) - case m.MachineHeld != nil && m.MachineHeld(node): - return false, mwc.RuleNotHeld, fmt.Sprintf("the condition machine.%s.network is open: no want moves to %s until its own network is healthy", node, node) - } - if ordering { - canary := Canary(m.Nodes, n.Online) - switch { - case canary == "": - return false, mwc.RuleCanary, fmt.Sprintf("none of the nodes of %s is online, so it has no canary", m.Name) - case node != canary && !n.RunsHealthy(canary, m.Name, m.FP): - return false, mwc.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP)) - } - } - if !m.Granted { - return false, mwc.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s for %s", m.Name, short(m.FP)) - } - if m.GrantedOn != nil && !m.GrantedOn(node) { - return false, mwc.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s on %s for %s", m.Name, node, - short(m.FP)) - } - if words, lacking := m.readerLacking(node, n); lacking { - return false, mwc.RuleComposed, words - } - if _, ok := m.composition(node, m.FP); !ok { - return false, mwc.RuleComposed, fmt.Sprintf("%s has named no composition of its own for %s at %s yet", node, m.Name, short(m.FP)) - } - if words, waiting := m.provisionWaiting(node); waiting { - return false, mwc.RuleProvisions, words - } - if words, waiting := m.verbWaiting(node); waiting { - return false, mwc.RuleProvisions, words - } - if words, waiting := m.busStateWaiting(node); waiting { - return false, mwc.RuleProvisions, words - } - return true, 0, "" -} - -// readerLacking says why the module's want on node is held for the node's engine (ADR 0614 §4): a reader capability its -// manifest needs that the engine does not state, read as the baseline when it states none, a bootstrapped seat counting as -// read by every engine (graph.MissingReads, ADR 0636 §2). It is held under rule 7 (the -// composition), since no rule of its own is numbered in this release, with words naming the capability; it moves once the -// engine states it. The node-engine's own new build is held only by a word of its own manifest no engine states. -// -// A build's reference forms hold the want only when the engine reads none of them (ADR 0624 §2): every engine reads a -// first form, so a build written in one is never held by its forms; the words name the forms it reads none of. -func (m Module) readerLacking(node string, n Nodes) (string, bool) { - missing := graph.MissingReads(m.Needs, n.Reads[node]) - if len(missing) == 0 && forms.Current.Held(m.FormNeeds, n.Reads[node]) { - missing = slices.Clone(m.FormNeeds) - } - if len(missing) == 0 { - return "", false - } - return graph.ReaderWaitWords(node, missing, m.Name, short(m.FP)), true // the words the delivery view names the edge by -} - -// provisionWaiting says why the module's want on node waits for a provision (ADR 0570 §4): a provision it requires that has -// no key yet, or the first of its keys that is not provided. Nil Provisions (the seat's state not on the bus) waits for none. -func (m Module) provisionWaiting(node string) (string, bool) { - if m.Provisions == nil { - return "", false - } - if missing := m.MissingProvisions(node); len(missing) > 0 { - return fmt.Sprintf("provision %s has no key yet for %s on %s", missing[0], m.Name, node), true - } - keyed := m.Provisions(node) - // Only a provision the newest manifest still requires is waited on (ADR 0603 §9): a key of an older revision's edge, - // asked or retiring for the operator's answer (ADR 0570 §5), holds nothing. A requirement catalogue.Awaited leaves out is - // not waited on when something offers it (a module's own name, a module the mesh provides itself, a provision the - // composition does not key: optional, a toolchain on the module's own machine, a login the controller makes). One that - // nothing offers is waited on (ADR 0603 §5: an unknown provider has provided nothing), so a consumer never moves ahead of - // a provider whose module the indexer has not yet taken in; it holds loudly, since the watchman raises the hold past its - // bound (ADR 0603 §8). - needed := m.Manifest.Requires - if m.Answering != nil { - needed = catalogue.Awaited(m.Manifest, m.Answering) - have := map[string]bool{} - for _, p := range keyed { - have[p.Provision] = true - } - for _, r := range slices.Sorted(slices.Values(m.Manifest.Requires)) { - if have[r] && !m.Manifest.OptionalRequires[r] && !slices.Contains(needed, r) { - needed = append(needed, r) // the provisioner keys it from the manifests, whoever is placed yet - continue - } - if m.Offered == nil || m.Manifest.OptionalRequires[r] || len(m.Answering(r)) > 0 || m.Offered(r) || slices.Contains(needed, r) { - continue - } - needed = append(needed, r) - if !have[r] { - return fmt.Sprintf("no module offers %s yet for %s on %s: an unknown provider has provided nothing, so the want waits for one, and the watchman raises it past its bound", r, m.Name, node), true - } - } - } - for _, key := range slices.Sorted(maps.Keys(keyed)) { - if p := keyed[key]; p.State != meshprovisionerclient.Provided && slices.Contains(needed, p.Provision) { - words := fmt.Sprintf("the provision %s of %s on %s is %s, not provided", key, m.Name, node, p.State) - if p.Why != "" { - words += ": " + p.Why - } - return words, true - } - } - return "", false -} - -// MissingProvisions are the provisions a want of the module waits for (catalogue.Awaited, the one rule the plan -// composition and the seat mesh-provisioner key by, issue 774) that have no key on node yet, sorted; none while the -// seat's state is not read. A requirement the composition keys nothing for (a toolchain on the module's own machine, a -// login the controller still makes for a module not handed) is not waited for: on 2026-10-30 every new build of -// grafana, dnsmasq, mesh-knowledge, mesh-mail and the builders waited here for a key nobody would make. -func (m Module) MissingProvisions(node string) []string { - if m.Provisions == nil || m.Answering == nil { - return nil - } - have := map[string]bool{} - for _, p := range m.Provisions(node) { - have[p.Provision] = true - } - var out []string - for _, r := range catalogue.Awaited(m.Manifest, m.Answering) { - if !have[r] { - out = append(out, r) - } - } - return out -} - // runsHealthy says whether module runs healthy on node: the node-engine seat's last `running` of it names a build, does not say it // stopped and does not say `healthy` false (design 47 G §520); nothing heard of it counts as not healthy. func (n Nodes) runsHealthy(node, module string) bool { @@ -328,317 +79,49 @@ func (n Nodes) runsHealthy(node, module string) bool { } // RunsHealthy says whether the build of module at fp runs healthy on node, from the node-engine seat's last `running` of it alone -// (design 47 G §520): a `running` with no `healthy`, said by a node-engine of before G, reads as healthy while it stands, and a -// module of which nothing was heard counts as not healthy. The have is never read for it. +// (design 47 G §520). The have is never read for it. func (n Nodes) RunsHealthy(node, module, fp string) bool { r, ok := n.Running[node][module] return ok && n.runsHealthy(node, module) && r.Fingerprint == fp } -// providedHealthy is rule 5 for one provision: some module providing it runs healthy on the node, or, assigned -// elsewhere only, at its canary. A provision no module provides is not a module's to judge (the mesh or the node -// offers it) and holds. -func providedHealthy(provision, node string, n Nodes) (bool, string) { - providers := n.Providers[provision] - if len(providers) == 0 { - return true, "" - } - names := make([]string, 0, len(providers)) - for p := range providers { - names = append(names, p) - } - sort.Strings(names) - for _, p := range names { - nodes := providers[p] - at := node - if !slices.Contains(nodes, node) { - at = Canary(nodes, n.Online) - } - if at != "" && n.runsHealthy(at, p) { - return true, "" - } - } - return false, fmt.Sprintf("no provider of %s runs healthy on %s or at its canary", provision, node) -} - -// unreached is why m on node may be wanted and not reached: the first provision it requires that no provider runs -// healthy on node or at its canary (rule 5, said and never holding a want; issue 1960); empty when all do. -func unreached(m Module, node string, n Nodes) string { - for _, provision := range m.Manifest.Requires { - if ok, why := providedHealthy(provision, node, n); !ok { - return why - } - } - return "" -} - -// keepsData says whether a module holds data a set-back would lose: an own item of a class that keeps (irreplaceable, -// valuable or rebuildable; cache and none do not). -func keepsData(m catalogue.Manifest) bool { - for _, it := range m.DataItems() { - if it.Class == catalogue.ClassIrreplaceable || it.Class == catalogue.ClassValuable || it.Class == catalogue.ClassRebuildable { - return true - } - } - return false -} - -// Outcome is what one module's pass decides: each node's want to write (only those that change), and a hold to write. +// Outcome is what one module's pass decides: each node's want to write (only those that change). type Outcome struct { Wants map[string]mwc.Want - // Held is the hold of m.FP to write, when a node's health check failed on it and it is not held yet. - Held *mwc.Held - // Lifted is the hold of m.FP lifted by itself: every node that runs the module runs m.FP healthy (issue 1906). The held - // record and its ask are taken away, and the wants move by the rules again in this pass. - Lifted bool - // NodeFailed is, by node, a failed health check of m.FP on a node while another node runs m.FP healthy: the failure is - // that node's, so m.FP is not held and nothing is set back (ADR 0557 §1, issue 1496). The pass raises its condition once. - NodeFailed map[string]NodeFailure - // Record is the moves a person must allow before the want names them: each is asked once (ADR 0539 §14). - Record []RecordNeed - // Restore is the nodes of a module holding data whose held fingerprint failed its health check there and that are not set - // back: the operator is asked whether to restore each (ADR 0539 §19). - Restore []string - // Rules is, by node, the rule holding the node back from m.FP and why. - Rules map[string]Holding - // Unreached is, by node, why the module may be wanted there and not reached: no provider of a provision it requires runs - // healthy (rule 5, issue 1960). It holds no want back; the pass says it. - Unreached map[string]string - // Neither is, by node, the nodes the module is assigned to that run it (or whose have names it), hold no want for it after this - // pass and are not held (a held module is on the held list), with the rule holding the first want back: the node's own composition does not hold a module with a fingerprint of the new form, so such a module is on - // neither list and nothing applies it on that node (issue 1945). A module with no fingerprint of the new form goes with the node's - // own composition and is never here. - Neither map[string]Holding } -// RecordNeed is a move of a module of policy record on a node that waits for the operator's yes: the composition's digest, the -// controller's judgement of the move and its words. -type RecordNeed struct { - Node string - Digest string - Judgement Judgement - Words string -} - -// Holding is a rule holding a node back, and why. The bus module is held back by the rules alone, as every module (ADR 0601 §1, -// §2): no bar of its own. -type Holding struct { - Rule int - Words string -} - -// Reason is the holding in words: the rule and its words. -func (h Holding) Reason() string { - return fmt.Sprintf("rule %d: %s", h.Rule, h.Words) -} - -// Wants is the pass of one module m: for each of its nodes, the want by the six rules, starting from the want written -// before (wants, by node); lastHealthy is, by node, the newest fingerprint other than m.FP the node ran healthy. A failed -// health check of m.FP on a node holds it: every node wanting m.FP is set back to its last healthy fingerprint, but a -// module holding data is not set back but asked about (Restore). A -// node that wants m but is no longer among its nodes gets its want written removed, never deleted (ADR 0408 §4). -func Wants(m Module, n Nodes, wants map[string]mwc.Want, lastHealthy map[string]string) Outcome { - out := Outcome{Wants: map[string]mwc.Want{}, Rules: map[string]Holding{}, Neither: map[string]Holding{}} - held := m.Held - if held && !m.Kept && m.FP != "" && allRunHealthy(m, n) { - out.Lifted, held = true, false - } - if !held && !out.Lifted && !m.Released && m.FP != "" { - for _, node := range slices.Sorted(slices.Values(m.Nodes)) { - h, ok := n.Haves[node][m.Name] - if !ok || h.Fp != m.FP || !failed(h) { - continue - } - if others := healthyElsewhere(m, n, node); len(others) > 0 && !failedElsewhere(m, n, node) { - // F runs healthy on another node, so its failure here is this node's, not F's: F is not held, no node is set back, - // and the failing node keeps its want (ADR 0557 §1; issue 1496: on ace a newly assigned mesh-build-node failed its - // first check and the hold moved shanks, g14 and novox, healthy on F for an hour, back). The pass raises the node's - // condition. With no online healthy witness, or with F failed on a second node, the hold and the set-back stand (ADR 0557 - // §2, ADR 0429 §6). - if out.NodeFailed == nil { - out.NodeFailed = map[string]NodeFailure{} - } - out.NodeFailed[node] = NodeFailure{Others: others, Words: HeldWords(h, time.Time{})} // said once: no clock in it - continue - } - out.Held = &mwc.Held{Node: node, Words: HeldWords(h, m.Now)} - held = true - break - } - } - mm := m - mm.Held = held +// Wants is the pass of one module m: for each of its nodes, the want naming its Target, starting from the want written before +// (wants, by node). A put that owns a node's want writes the put's; a node with no target keeps the want written. A node that +// wants m but is no longer among its nodes gets its want written removed, never deleted (ADR 0408 §4). +func Wants(m Module, wants map[string]mwc.Want) Outcome { + out := Outcome{Wants: map[string]mwc.Want{}} for _, node := range m.Nodes { - if m.FP == "" { - break - } before, had := wants[node] - var next mwc.Want p, put := m.Puts[node] if !put && had && before.Put && before.Hold { - // The want is the truth (ADR 0546 clause 5): a want marked put and held stands while its put is not heard (the - // puts not open, or not yet handed after a start); only a put heard released gives it back to the rules. A put - // without hold is not kept this way: its put takes it further once heard. - p, put = puts.Put{FP: before.Fp, Composed: before.Composed, Hold: true, Why: strings.TrimPrefix(before.Words, "put: "), - Step: puts.Holding}, true + // The want is the truth (ADR 0546 clause 5): a want marked put and held stands while its put is not heard; only a put + // heard released gives it back to the rule. + p, put = puts.Put{FP: before.Fp, Hold: true, Why: strings.TrimPrefix(before.Words, "put: "), Step: puts.Holding}, true } if put { if want, owns := PutOwns(p); owns { - if want != nil { - w := *want - // A held bus still takes a revision of its composition that changes only its user list (ADR 0539 §7). - if ul, ok := m.userListRevision(node, before, had, n); ok && ul.Fp == w.Fp { - w.Composed = ul.Composed - } - if !had || !mwc.SameWant(normal(before), normal(w)) { - out.Wants[node] = w - } + if want != nil && (!had || !mwc.SameWant(normal(before), normal(*want))) { + out.Wants[node] = *want } continue } } - // A revision of the bus module's composition that differs only in its user list is written at once, at the fingerprint the - // node runs, outside the rules, the record hold and the bar on a new fingerprint: a new module's credential must reach the - // bus before the module can connect (ADR 0539 §7). The controller marks it; whatever it judges of the file's mount stands. - if w, ok := m.userListRevision(node, before, had, n); ok { - if !had || !mwc.SameWant(normal(before), normal(w)) { - out.Wants[node] = w - } + t, ok := Target{}, false + if m.Target != nil { + t, ok = m.Target(node) + } + if !ok || t.Fp == "" { continue } - ok, rule, words := Holds(mm, node, n) - target := m.FP - if have, ran := n.Haves[node][m.Name]; !ok && (!had || before.Removed) && rule == mwc.RuleCanary { - // The canary orders a move, never the first want of a node that does not run the module (issue 1960: de-spiegel, - // link2pay and photos on novox were left with no want). A node that runs an older fingerprint of the new form, - // composed and granted, gets its first want at what it runs, and rule 3 then orders its move to F as from any want. - // A node that runs an old-form build, which no want can name, stays under the canary (issue 1945, a design limit): - // its first want names F once the canary runs F healthy. Any other node that runs it (F itself, or an older - // fingerprint with no composition or grant) gets F by the rules without which there is nothing to want. - older := ran && runs(have) && have.Fp != m.FP && strings.HasPrefix(have.Fp, meshcatalogclient.SourcePrefix2) - if older { - _, composed := m.composition(node, have.Fp) - older = composed && m.GrantedAt != nil && m.GrantedAt(have.Fp) - } - oldForm := ran && runs(have) && have.Fp != m.FP && !strings.HasPrefix(have.Fp, meshcatalogclient.SourcePrefix2) - switch { - case older: - ok, target = true, have.Fp - case oldForm: - // held back by rule 3 as written: on neither list until the canary runs F healthy - default: - ok, rule, words = holds(mm, node, n, false) - } - } - if why := unreached(m, node, n); why != "" { - if out.Unreached == nil { - out.Unreached = map[string]string{} - } - out.Unreached[node] = why - } - if ok && target == m.FP && len(out.NodeFailed) > 0 && !(had && !before.Removed && before.Fp == m.FP) && - n.Haves[node][m.Name].Fp != m.FP { - // While a node fails F alone, F reaches no further node: a node that neither wants nor runs F gets no new want - // naming it, and wants already written stay (ADR 0557 §2). A node that runs F with no live want gets its want at F, or - // it would be on neither list and its node-engine would give the module up (issue 1945). - ok, rule, words = false, mwc.RuleNotHeld, FailedAloneWords(m.Name, m.FP, slices.Sorted(maps.Keys(out.NodeFailed))) - } - switch { - case ok: - d, _ := m.composition(node, target) - next = n.withKept(m, node, mwc.Want{Fp: target, Composed: d.Digest}) - at := m - if target != m.FP { - // the first want at the older fingerprint the node runs: held back from F by the rule, and judged as a want at it - next.Newest, next.Rule, next.Words = m.FP, int64(rule), words - at.FP = target - } - // A placed node's want names the build alone: it is the same target while the build is. - if had && !before.Removed && !before.Put && before.Fp == target && (before.Composed == d.Digest || (before.Self && m.selfOn(node))) { - next.Backup, next.Words = before.Backup, before.Words // the answer to a failed backup stays while its target stays (ADR 0539 §16) - } - running, ran := n.Haves[node][m.Name] - // A removed want is no want: a first want after it goes through the record ask as a first want does. - asked, hadLive := before, had - if before.Removed { - asked, hadLive = mwc.Want{}, false - } - if need, wait := at.mustAsk(node, asked, hadLive, ran, running.Fp, running.Composed == "", d); wait { - if need != nil { - out.Record = append(out.Record, *need) - } - continue - } - case held && had && before.Fp == m.FP: - out.Rules[node] = Holding{Rule: rule, Words: words} - last := lastHealthy[node] - switch { - case keepsData(m.Manifest) && (m.Restored == nil || !m.Restored(node)): - next = before - next.Words = fmt.Sprintf("%s holds data, so it is not set back from %s: the operator is asked whether to restore it", m.Name, short(m.FP)) - out.Restore = append(out.Restore, node) - case last == "": - next = before - next.Words = fmt.Sprintf("%s is held at %s and %s ran no other fingerprint healthy to set it back to", m.Name, short(m.FP), node) - default: - d, there := m.composition(node, last) - if !there { - next = before - next.Words = fmt.Sprintf("%s is held at %s, and the composition of %s at %s is gone, so %s is not set back", m.Name, short(m.FP), m.Name, short(last), node) - break - } - next = mwc.Want{Fp: last, Composed: d.Digest, Newest: m.FP, Rule: int64(rule), Words: words} - } - default: - out.Rules[node] = Holding{Rule: rule, Words: words} - if !had || before.Removed { - // The node runs the module, or its have names it, and gets no want for it: the module is on neither list (issue 1945), and - // the pass says so. A new assignment, with no have, waits for the rules and is on no list yet by design. A held module (rule 2, - // held before or by this pass) is on the held list instead, whose hold is said and answered on its own: it is never on neither - // list (issue 1945: systemd-resolved, held on shanks by the operator, was said as urgent every hour). - h, there := n.Haves[node][m.Name] - switch { - case there && h.Phase != nodeengineclient.PhaseRemoved && !held: - out.Neither[node] = Holding{Rule: rule, Words: words} - case had && (!there || h.Phase == nodeengineclient.PhaseRemoved): - // A removed want of a node that does not run the module says why no want follows it (issue 1945), so the view's - // assigner_want answers it; it stays removed. A node with no key gets none: a removed want written where none was - // would have its node agent remove the module and stop holding its composition's resources back. - next = mwc.Want{Removed: true, Words: NoWantWords(rule, words)} - if !mwc.SameWant(normal(before), normal(next)) { - out.Wants[node] = next - } - } - continue - } - next = mwc.Want{Fp: before.Fp, Composed: before.Composed, Newest: m.FP, Rule: int64(rule), Words: words} - if before.Fp == m.FP { - // Held at F by the rules: a new revision of F's composition waits for them, as a new fingerprint does (ADR 0539 §5). - next = mwc.Want{Fp: m.FP, Composed: before.Composed} - } else if d, there := m.composition(node, before.Fp); there && d.Digest != "" && d.Digest != before.Composed { - // Held back at an earlier fingerprint whose kept composition the controller composed again with today's config - // (novox/hq ADR 0545): the want follows its new revision, so the node does not run that build with the config of - // the moment it was made (on g14 a capacity of 0 after a rollback). Asked as any move of a record module is. - at := m - at.FP = before.Fp - running, ran := n.Haves[node][m.Name] - need, wait := at.mustAsk(node, before, true, ran, running.Fp, running.Composed == "", d) - if need != nil { - out.Record = append(out.Record, *need) - } - if !wait { - next.Composed = d.Digest - } - } - } - next, write := m.selfDerived(node, before, had, next, n, &out) - if !write { - continue - } - // Every want that names a build names its version (A-430a), whichever rule made or kept it: a want held at its fingerprint - // by the rules, a node's own (self) want too. - if !next.Removed && next.Fp != "" && next.Version == "" { - next = n.withKept(m, node, next) + next := mwc.Want{Fp: t.Fp, Version: t.Version, Pipeline: t.Pipeline, Rank: t.Rank, Self: m.selfOn(node)} + if had && !before.Removed && !before.Put && before.Fp == t.Fp { + // The answers the want carries stay while its build does: a failed backup's (ADR 0539 §16), a held revert's, the yes. + next.Backup, next.Words, next.Revert, next.Yes, next.Release = before.Backup, before.Words, before.Revert, before.Yes, before.Release } if !had || !mwc.SameWant(normal(before), normal(next)) { out.Wants[node] = next @@ -652,143 +135,17 @@ func Wants(m Module, n Nodes, wants map[string]mwc.Want, lastHealthy map[string] return out } -// selfDerived is next with its Self derived from the assignments now (Module.Self), whichever rule made it, and whether it is -// written. A placed node's want names the build alone (ADR 0547 §2). An unplaced node's want that a rule kept from the want -// written while the node was placed (held at F, held back by rule 3 or 5, a data module asked about) names the controller's -// composition of the same fingerprint, asked as any move of a record module is (ADR 0539 §14): on 2026-10-27, after the -// assignments seed's revert, g14's systemd-resolved (held) and forticlient (rule 5), and shanks's forticlient (rule 3), kept -// their Self wants, which their node agents, no longer placed, could not compose (issue 1126). With no such composition, or -// while its ask waits, the want written stays and the node's holding says why. -func (m Module) selfDerived(node string, before mwc.Want, had bool, next mwc.Want, n Nodes, out *Outcome) (mwc.Want, bool) { - switch { - case next.Removed: - return next, true - case m.selfOn(node): - next.Self, next.Composed = true, "" - return next, true - case !next.Self && next.Composed != "": - return next, true - } - next.Self, next.Yes = false, "" - d, there := m.composition(node, next.Fp) - if !there || d.Digest == "" { - out.Rules[node] = Holding{Rule: mwc.RuleComposed, Words: fmt.Sprintf("%s on %s was composed by the node while it was placed, "+ - "and no composition of its own is named at %s now, so its want stays", m.Name, node, short(next.Fp))} - return next, false - } - at := m - at.FP = next.Fp - running, ran := n.Haves[node][m.Name] - if need, wait := at.mustAsk(node, before, had, ran, running.Fp, running.Composed == "", d); wait { - if need != nil && !slices.Contains(out.Record, *need) { // the branch held back at an older fingerprint may have asked it - out.Record = append(out.Record, *need) - } - return next, false - } - next.Composed = d.Digest - return next, true -} - -// NoWantWords say why a node gets no want for a module: the rule that holds its first want back and why (issue 1945). -func NoWantWords(rule int, words string) string { - return fmt.Sprintf("no want: rule %d: %s", rule, words) -} - // failed says whether a have's health check failed once the node is done moving: phase running or failed (never removed, held, or -// moving). A failed health while it -// backs up, applies or checks is a step of the move, not its outcome (on novox at 15:27:36 on 2026-10-25, "health failed, phase -// applying" held the controller's canary two seconds before the new build was healthy). Every healthy read (healthy, providedHealthy, -// the world's lastHealthy) already asks for phase running. +// moving). func failed(h nodeengine.HaveByNodeModuleValue) bool { return h.Health == nodeengineclient.HealthFailed && (h.Phase == nodeengineclient.PhaseRunning || h.Phase == nodeengineclient.PhaseFailed) } -// healthyElsewhere is every online node of m but node that runs m.FP healthy, by name. -func healthyElsewhere(m Module, n Nodes, node string) []string { - var out []string - for _, other := range slices.Sorted(slices.Values(m.Nodes)) { - // A witness is online: a node that sleeps keeps the have it last wrote, and a stale have is no evidence that F runs - // healthy now. - if other != node && n.Online[other] && n.RunsHealthy(other, m.Name, m.FP) { - out = append(out, other) - } - } - return out -} - -// failedElsewhere says whether another node of m has a have naming m.FP failed: then the failure is not one node's alone, and -// the hold and the set-back stand at the second failure (ADR 0557 §2). -func failedElsewhere(m Module, n Nodes, node string) bool { - for _, other := range m.Nodes { - if h, ok := n.Haves[other][m.Name]; ok && other != node && h.Fp == m.FP && failed(h) { - return true - } - } - return false -} - -// FailedAloneWords say why a node gets no new want naming fp: fp fails its health check on the nodes named, alone. -func FailedAloneWords(module, fp string, failing []string) string { - return fmt.Sprintf("%s at %s fails its health check on %s, so no further node moves to it until that is over", module, short(fp), - strings.Join(failing, ", ")) -} - -// NodeFailure is a failed health check of a module's newest fingerprint on one node while it runs healthy on Others: the have's -// health, phase and words (HeldWords without a time), and the healthy witnesses by name. -type NodeFailure struct { - Others []string - Words string -} - -// runs says whether a have is of a module the node runs: running, whatever its health. -func runs(h nodeengine.HaveByNodeModuleValue) bool { - return h.Phase == nodeengineclient.PhaseRunning -} - -// allRunHealthy says whether every node of m that runs the module (its have's phase is running, or failed) runs m.FP healthy, and -// at least one does: a hold whose fingerprint runs healthy wherever the module runs holds nothing back any more (issue 1906: seven -// modules stayed held on every machine after one transient failed check). A node set back to another fingerprint does not run -// m.FP, and a failed one does not run it healthy, so the hold stays; a have that is removed, held for its backup or still moving -// (backing up, applying, checking) is not counted, or it would keep the hold up for ever. -func allRunHealthy(m Module, n Nodes) bool { - ran := 0 - for _, node := range m.Nodes { - h, ok := n.Haves[node][m.Name] - if !ok || (h.Phase != nodeengineclient.PhaseRunning && h.Phase != nodeengineclient.PhaseFailed) { - continue - } - if !n.RunsHealthy(node, m.Name, m.FP) { - return false - } - ran++ - } - return ran > 0 -} - -// HeldWords are the words a hold keeps of its cause (issue 1906): the health and phase the node reported, when the assigner saw -// it, and the node's own words. The node is the held record's own field. -func HeldWords(h nodeengine.HaveByNodeModuleValue, at time.Time) string { - words := fmt.Sprintf("health %s, phase %s", h.Health, h.Phase) - if !at.IsZero() { - words += ", seen at " + at.UTC().Format(time.RFC3339) - } - if d := strings.TrimSpace(h.Detail); d != "" { - return words + ": " + d - } - return words + ": the node gave no words" -} - -// LiftedWords say why a hold was lifted by itself. -func LiftedWords(module, fp string) string { - return fmt.Sprintf("every node that runs %s runs %s healthy, so the assigner lifted its hold by itself", module, short(fp)) -} - // normal is a want without its version, as a decoded and a composed one are compared. func normal(w mwc.Want) mwc.Want { w.V = 0 return w } - func short(fp string) string { part, err := catalogkey.KeyPart(fp) if err != nil || len(part) < 12 { @@ -797,69 +154,20 @@ func short(fp string) string { return part[:12] } -// mustAsk says whether the move to the composition d on node waits for the operator's yes (need names the ask, ADR 0539 §14). A -// module the node never ran (it has no want and no have), a move judged neither, a module that is not of policy record and a want -// that stays are written as any want. The bus module is no exception: it moves by the rules (ADR 0601 §1), its node backing up its -// streams before it applies (§2); the assigner asks no snapshot for it. -func (m Module) mustAsk(node string, before mwc.Want, had, ran bool, running string, haveNamesNoComposition bool, d ComposedDigest) (*RecordNeed, bool) { - if (!had && !ran) || before.Removed || (before.Fp == m.FP && before.Composed == d.Digest) { - return nil, false - } - // Right after the switch no node has a readable want, and a have from the old node-engine names a fingerprint and no composition - // (ADR 0539 §9, §11): a node running this fingerprint is given its want at it with no hold when the controller judges the composition - // unchanged, equal to the declaration the node last applied (ADR 0540 §9), since the node agent then applies nothing. A composition that - // differs from it (a changed setting or secret), or that the controller could not judge, waits for the ask. A new fingerprint still waits. - if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == JudgedUnchanged { - return nil, false - } - if !m.recorded(d) || d.Judgement == JudgedNeither || d.Judgement == JudgedUnchanged { - return nil, false - } - if m.Allowed != nil && m.Allowed(node, d.Digest) { - return nil, false - } - return &RecordNeed{Node: node, Digest: d.Digest, Judgement: d.Judgement, Words: d.Words}, true +// RecordNeed is a move of a module of policy record on a node that waits for the operator's yes: the digest the node-engine's have +// waits on, its judgement of the move and its words (ADR 0539 §14). +type RecordNeed struct { + Node string + Digest string + Judgement Judgement + Words string } -// recorded says whether the module is of policy record: a person's choice at the assigner over the manifest's word (ADR -// 0565 §7, catalogue.Upgrade); for a module with none, as the controller holds it beside the composition's digest until the -// lease write (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it. -func (m Module) recorded(d ComposedDigest) bool { - if m.Chosen != nil { - policy, _, _, _ := catalogue.Upgrade(m.Manifest, m.Chosen) - return policy == catalogue.PolicyRecord - } - if d.Policy != "" { - return d.Policy == catalogue.PolicyRecord - } - policy, _, _ := catalogue.DerivedUpgrade(m.Manifest) - return policy == catalogue.PolicyRecord -} - -// userListRevision is the want that names the bus module's composition at the fingerprint the node runs when that composition -// differs from the one the node runs only in the bus's user list (ComposedDigest.UserListOnly), and whether there is one. A node -// whose want names another fingerprint than the one it runs, or a module that does not provide the bus, has none. -func (m Module) userListRevision(node string, before mwc.Want, had bool, n Nodes) (mwc.Want, bool) { - if !catalogue.ProvidesBus(m.Manifest) { - return mwc.Want{}, false - } - running, ran := n.Haves[node][m.Name] - if !ran || running.Fp == "" || (had && (before.Removed || before.Fp != running.Fp)) { - return mwc.Want{}, false - } - d, ok := m.composition(node, running.Fp) - if !ok || !d.UserListOnly { - return mwc.Want{}, false - } - w := mwc.Want{Fp: running.Fp, Composed: d.Digest} - if had && before.Composed == d.Digest { - w.Backup, w.Words = before.Backup, before.Words - } - return w, true -} - -// GrantNamesNode says whether a grant of module names its accounts on node (mesh-sdk's Grant.NamesNode, which a -// put reads by too). -func GrantNamesNode(g grantsstate.Grant, node, module string) bool { - return g.NamesNode(node, module) +// KeptBuild is what the assigner knows of a build: the digest of each of its parts by `:` and the build node that +// keeps it, as the builder's `build.ended` says them, and its released version as the catalog's `version-released` says it (design +// 47 G §379, A-96). A want names the version alone (design 47 H §750). +type KeptBuild struct { + Parts map[string]string + KeptOn string + Version string } diff --git a/modules/mesh-delivery-assigner/internal/assign/wants_test.go b/modules/mesh-delivery-assigner/internal/assign/wants_test.go index e108b997..510b8530 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants_test.go @@ -1,24 +1,18 @@ package assign import ( + "strings" + "testing" + "time" + "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate" + "mesh-delivery-assigner/internal/puts" - "reflect" - "slices" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient" - "git.novox.be/novox/mesh-sdk/go/catalogue" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph" ) func fpOf(c byte) string { return meshcatalogclient.SourcePrefix2 + strings.Repeat(string(c), 64) } @@ -28,804 +22,71 @@ var ( fOld = fpOf('a') ) -// ready is a module mail at fNew on ace and g14 for which every rule holds: built, not held, granted, its provider -// store healthy on ace and the canary ace running fNew healthy. -func ready() (Module, Nodes) { - m := Module{Name: "mail", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"g14", "ace"}, - Manifest: catalogue.Manifest{Module: "mail", Requires: []string{"database"}}} - n := Nodes{ - Online: map[string]bool{"ace": true, "g14": true}, - Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{ - "ace": {"mail": {Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}, "store": {Fp: fpOf('s'), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}}, - "g14": {"mail": {Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}}, - }, - Providers: map[string]map[string][]string{"database": {"store": {"ace"}}}, - } - return m, n -} - // dig is the digest of the composition at a fingerprint in these tests: the fingerprint's last character repeated. func dig(fp string) string { return "sha256:" + strings.Repeat(fp[len(fp)-1:], 64) } -// composedEverywhere is a Module.Composed with a composition for every node at every fingerprint, judged neither. -func composedEverywhere(_, fp string) (ComposedDigest, bool) { - return ComposedDigest{Digest: dig(fp), Judgement: JudgedNeither}, true +// released is a module mail on ace and g14 whose newest version released on its branch with a passed pipeline is 1.2.3 at fNew, +// the second tier of the pipeline main.12. +func released() Module { + t := Target{Fp: fNew, Version: "1.2.3", Pipeline: "main.12", Rank: 2} + return Module{Name: "mail", Nodes: []string{"g14", "ace"}, Target: func(string) (Target, bool) { return t, true }} } func wantedOld() map[string]mwc.Want { - return map[string]mwc.Want{"ace": {Fp: fOld, Composed: dig(fOld)}, "g14": {Fp: fOld, Composed: dig(fOld)}} + return map[string]mwc.Want{"ace": {Fp: fOld, Version: "1.2.2", Pipeline: "main.11", Rank: 1}, "g14": {Fp: fOld, Version: "1.2.2", Pipeline: "main.11", Rank: 1}} } -// 1297/B4: with every rule holding, both nodes' wants move to the new fingerprint (the canary ace runs it healthy, so -// g14 follows at once: step 0, ADR 0528 clause 3). -func TestEveryRuleHoldsSoTheWantsMove(t *testing.T) { - m, n := ready() - out := Wants(m, heardOfHaves(n), wantedOld(), nil) - for _, node := range []string{"ace", "g14"} { - if out.Wants[node].Fp != fNew || out.Wants[node].Rule != 0 || out.Wants[node].Composed != dig(fNew) { - t.Errorf("%s's want = %+v, want %s", node, out.Wants[node], fNew) - } - } -} - -// 1297/B4, one test per rule: given every rule but N holds, no want moves to F, and the rule N is named. -func TestEachRuleHoldsTheWantBack(t *testing.T) { - for rule, breakIt := range map[int]func(*Module, *Nodes){ - mwc.RuleBuildDone: func(m *Module, _ *Nodes) { m.Built = false }, - mwc.RuleNotHeld: func(m *Module, _ *Nodes) { m.Held = true }, - mwc.RuleCanary: func(_ *Module, n *Nodes) { - n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - }, - mwc.RuleGrant: func(m *Module, _ *Nodes) { m.Granted = false }, - mwc.RuleComposed: func(m *Module, _ *Nodes) { - m.Composed = func(string, string) (ComposedDigest, bool) { return ComposedDigest{}, false } - }, - // 1954/D1 (ADR 0570 §4): a provision of the module on that node still asked holds its want. - mwc.RuleProvisions: func(m *Module, _ *Nodes) { - m.Provisions = func(node string) map[string]meshprovisionerclient.Provision { - return map[string]meshprovisionerclient.Provision{node + ".mail.database": {Provision: "database", State: meshprovisionerclient.Asked}, node + ".mail.oidc": {Provision: "oidc", State: meshprovisionerclient.Provided}} - } - }, - } { - m, n := ready() - breakIt(&m, &n) - out := Wants(m, heardOfHaves(n), wantedOld(), nil) - if w, ok := out.Wants["g14"]; ok && w.Fp == fNew { - t.Errorf("rule %d broken: g14's want moved to F", rule) - } - if got := out.Rules["g14"].Rule; got != rule { - t.Errorf("rule %d broken: g14 is held back by rule %d (%s)", rule, got, out.Rules["g14"].Words) - } - if w := out.Wants["g14"]; w.Fp != "" && (w.Fp != fOld || w.Newest != fNew || w.Rule != int64(rule)) { - t.Errorf("rule %d broken: g14's want is %+v", rule, w) - } - if rule == mwc.RuleProvisions && (!strings.Contains(out.Rules["g14"].Words, "g14.mail.database") || !strings.Contains(out.Rules["g14"].Words, "asked")) { - t.Errorf("the provision's hold does not name its key and state: %s", out.Rules["g14"].Words) - } - } - // 1954/D1: every provision provided, the want moves; with the seat's state not read, nothing waits on it. - for _, all := range []func(string) map[string]meshprovisionerclient.Provision{ - func(node string) map[string]meshprovisionerclient.Provision { - return map[string]meshprovisionerclient.Provision{node + ".mail.database": {Provision: "database", State: meshprovisionerclient.Provided}} - }, nil} { - m, n := ready() - m.Provisions = all - if out := Wants(m, heardOfHaves(n), wantedOld(), nil); out.Wants["g14"].Fp != fNew { - t.Errorf("provided provisions held g14's want: %+v (%s)", out.Wants["g14"], out.Rules["g14"].Words) - } - } - // ADR 0614 §4: a want is held on a node whose engine does not state a reader capability the manifest needs, under rule 7 - // with the capability as its words, and moves once the engine states it; an engine stating nothing reads the baseline, - // so a manifest of the baseline's words, the node-engine's own new build among them, is not held. - hostNeeds, err := catalogue.ManifestNeeds([]byte(`{"module":"mesh-host","version":"1","slug":"host","build":{"artifacts":[` + - `{"name":"host-arch","kind":"bundle","language":"go","system":"arch","from":"cmd/mesh-host"}]},"consumes":["restic.backed-up"],` + - `"state":[{"name":"health"}],"config":{"x":{"type":"number","minimum":1,"default":50,"runtime":true}},` + - `"resources":[{"id":"k","type":"file","path":"/var/lib/k","content":"${config:x}\n"}]}`)) - if err != nil { - t.Fatal(err) - } +// A6/AC1, A5/AC1 (design 47 H §750): a want names the newest released version whose pipeline passed, with the pipeline and the +// mover's rank, on every node the module is assigned to; nothing else holds it: no provider, canary, grant, composition, reader +// capability, provision, served verb or bus state is read. A node with no target keeps the want written; a node no longer +// assigned is removed; a want that stands is not written again and keeps the answers it carries. +func TestAWantNamesTheNewestPassedVersion(t *testing.T) { + want := mwc.Want{Fp: fNew, Version: "1.2.3", Pipeline: "main.12", Rank: 2} for _, c := range []struct { - why string - name string - needs []string - reads map[string][]string - held bool - // forms are the build's reference forms (ADR 0624 §2), and names what the hold's words name. - forms []string - names string + name string + module func() Module + before map[string]mwc.Want + wrote map[string]mwc.Want }{ - {"a word the baseline lacks, on engines stating nothing", "mail", []string{"word:module", "word:tool-runner"}, nil, true, nil, "word:tool-runner"}, - {"the same, once both engines state it", "mail", []string{"word:module", "word:tool-runner"}, - map[string][]string{"ace": catalogue.ManifestReads(), "g14": catalogue.ManifestReads()}, false, nil, ""}, - {"the baseline's words on engines stating nothing", "mail", []string{"word:claims.serves", "seat:mesh-catalog", "form:config"}, nil, false, nil, ""}, - {"the node-engine's new build, whose contract has more words than its manifest uses", "mesh-host", hostNeeds, nil, false, nil, ""}, - // Issue 1966: g14's engine states its words and forms but no seat (its statement made before its seat list was in - // use); needs of the bootstrapped seats mesh-catalog and node-backup move all the same (ADR 0636 §2), one of a seat - // not bootstrapped is held. - {"mesh-catalog on an engine stating no seat", "mesh-host", []string{"seat:mesh-catalog", "word:module"}, - map[string][]string{"g14": {"word:module"}}, false, nil, ""}, - {"node-backup on an engine stating no seat", "restic", []string{"seat:node-backup", "word:module"}, - map[string][]string{"g14": {"word:module"}}, false, nil, ""}, - {"a seat not bootstrapped on an engine stating no seat: held, naming it", "mail", []string{"seat:mesh-vault", "word:module"}, - map[string][]string{"g14": {"word:module"}}, true, nil, "seat:mesh-vault"}, - // ADR 0624 §2, with form:new a form no engine reads unless it states it and artifact-store the first form. - {"g14 back stating {old}, a build carrying {old,new}: moves", "mail", nil, map[string][]string{"g14": {"word:module"}}, false, - []string{"form:artifact-store", "form:new"}, ""}, - {"an engine stating nothing, a build carrying the baseline's form: moves", "mail", nil, nil, false, []string{"form:artifact-store"}, ""}, - {"a build carrying {new} only: held, naming the form", "mail", nil, map[string][]string{"g14": {"word:module"}}, true, - []string{"form:new"}, "form:new"}, - {"the same, once g14's engine states it", "mail", nil, map[string][]string{"g14": {"form:new"}}, false, []string{"form:new"}, ""}, - {"a manifest needing an unstated word, its build in the first form: held, naming the word", "mail", []string{"word:tool-runner"}, nil, true, - []string{"form:artifact-store"}, "word:tool-runner"}, + {"both nodes move to the passed version", released, wantedOld(), map[string]mwc.Want{"ace": want, "g14": want}}, + {"a first want", released, nil, map[string]mwc.Want{"ace": want, "g14": want}}, + {"a placed node composes its own", func() Module { + m := released() + m.Self = func(node string) bool { return node == "ace" } + return m + }, nil, map[string]mwc.Want{"ace": {Fp: fNew, Version: "1.2.3", Pipeline: "main.12", Rank: 2, Self: true}, "g14": want}}, + {"no passed version yet: the want written stays", func() Module { + m := released() + m.Target = func(string) (Target, bool) { return Target{}, false } + return m + }, wantedOld(), map[string]mwc.Want{}}, + {"a want that stands is not written again", released, map[string]mwc.Want{"ace": want, "g14": want}, map[string]mwc.Want{}}, + {"the backup answer stays while the build does", released, + map[string]mwc.Want{"ace": {Fp: fNew, Version: "1.2.3", Pipeline: "main.12", Rank: 2, Backup: mwc.BackupSkip, Words: BackupAnswerWords("skip", 0)}, "g14": want}, + map[string]mwc.Want{}}, + {"a node no longer assigned is removed", func() Module { + m := released() + m.Nodes = []string{"ace"} + return m + }, map[string]mwc.Want{"ace": want, "g14": want}, map[string]mwc.Want{"g14": {Removed: true, Words: "mail is no longer assigned to g14"}}}, } { - m, n := ready() - m.Name, m.Needs, n.Reads, m.FormNeeds = c.name, c.needs, c.reads, c.forms - n.Haves["ace"][c.name], n.Haves["g14"][c.name] = n.Haves["ace"]["mail"], n.Haves["g14"]["mail"] - out := Wants(m, heardOfHaves(n), wantedOld(), nil) - if moved := out.Wants["g14"].Fp == fNew; moved == c.held { - t.Errorf("%s: g14's want %+v, held by %+v", c.why, out.Wants["g14"], out.Rules["g14"]) - } - if r := out.Rules["g14"]; c.held && (r.Rule != mwc.RuleComposed || !strings.Contains(r.Words, c.names) || - !strings.Contains(r.Words, "node-engine of g14")) { - t.Errorf("%s: held by %+v, want rule 7 naming the capability", c.why, r) - } - if w := out.Wants["g14"]; c.held && (w.Rule != mwc.RuleComposed || w.Newest != fNew) { - t.Errorf("%s: g14's want %+v, want it kept at what it runs and naming rule 7", c.why, w) - } - } - // ADR 0614 §2, §3: a want waits, under the provisions' rule, until each verb it invokes is served by a running build, - // and moves once one is; a verb a running build serves already adds no wait, and an edge the graph leaves out on a cycle - // is not among Invokes, so it holds nothing; nothing read of what runs waits for none. - for _, c := range []struct { - name string - invokes []string - served []string - nilRead bool - holds string - }{ - {"a verb no running build serves", []string{"store.store_put", "seat:mesh-catalog.modules"}, []string{"store.store_put"}, false, "seat:mesh-catalog.modules"}, - {"once a build serving it runs", []string{"seat:mesh-catalog.modules"}, []string{"seat:mesh-catalog.modules"}, false, ""}, - {"an edge left out on a cycle", nil, nil, false, ""}, - {"nothing read of what runs", []string{"seat:mesh-catalog.modules"}, nil, true, ""}, - // ADR 0614 §5: a module asking bus state is held while no build offering bus-state runs, and moves once one does. - {"bus state no running build offers", nil, nil, false, graph.BusState}, - {"bus state a running build offers", nil, []string{graph.BusState}, false, ""}, - {"bus state its own module offers", nil, nil, false, ""}, - } { - m, n := ready() - m.Invokes = c.invokes - if !c.nilRead { - m.Served = func(v string) bool { return slices.Contains(c.served, v) } - } - if strings.HasPrefix(c.name, "bus state") { - m.Manifest.BusState = []catalogue.BusStateEntry{{Name: "cache", Storage: "key-value"}} - m.Offering = func(p string) bool { return slices.Contains(c.served, p) } - if c.name == "bus state its own module offers" { - m.Manifest.Provides = []catalogue.Offer{{Name: graph.BusState, Scope: catalogue.ScopeMesh}} - } - } - words := graph.VerbWaitWords(c.holds, "mail", "g14") - if c.holds == graph.BusState { - words = graph.BusStateWaitWords("mail", "g14") - } - out := Wants(m, heardOfHaves(n), wantedOld(), nil) - if c.holds == "" { - if out.Wants["g14"].Fp != fNew { - t.Errorf("%s: held %+v", c.name, out.Rules["g14"]) - } + out := Wants(c.module(), c.before) + if len(out.Wants) != len(c.wrote) { + t.Errorf("%s: wrote %+v, want %+v", c.name, out.Wants, c.wrote) continue } - if out.Wants["g14"].Fp == fNew || out.Rules["g14"].Rule != mwc.RuleProvisions || out.Rules["g14"].Words != words { - t.Errorf("%s: want %+v, held by %+v", c.name, out.Wants["g14"], out.Rules["g14"]) - } - } - // ADR 0603 §9: the newest source revision governs. The module now at fNew no longer requires the database an older revision - // asked for and requires oidc instead; the seat's keys follow the newest manifest, so the older edge is no wait: a provided - // oidc moves the want, an asked one holds it and names oidc and never database. - for _, c := range []struct { - name string - state, older string - moves bool - }{{"the replaced edge provided, the older edge retiring", meshprovisionerclient.Provided, meshprovisionerclient.Retiring, true}, - {"the replaced edge provided, the older edge asked", meshprovisionerclient.Provided, meshprovisionerclient.Asked, true}, - {"the replaced edge asked, the older edge retiring", meshprovisionerclient.Asked, meshprovisionerclient.Retiring, false}} { - m, n := ready() - m.Manifest.Requires = []string{"oidc"} - m.Provisions = func(node string) map[string]meshprovisionerclient.Provision { - return map[string]meshprovisionerclient.Provision{node + ".mail.oidc": {Provision: "oidc", State: c.state}, - node + ".mail.database": {Provision: "database", State: c.older, Why: "the operator's answer"}} - } - out := Wants(m, heardOfHaves(n), wantedOld(), nil) - if got := out.Wants["g14"].Fp == fNew; got != c.moves || (!c.moves && (!strings.Contains(out.Rules["g14"].Words, "oidc") || strings.Contains(out.Rules["g14"].Words, "database"))) { - t.Errorf("%s: want %+v, held by %q", c.name, out.Wants["g14"], out.Rules["g14"].Words) - } - } - // 1954/D1 (the lead, 2026-10-29): a required provision with no key yet holds the want, named; and only one the - // composition keys (catalogue.Awaited, issue 774), replaying 2026-10-30: grafana (not handed) waited on ace for - // oidc-client, which the controller still makes, and mesh-build-go (handed) for go-toolchain on its own machine. - { - handed := &catalogue.Handed{Source: "s", Digest: "d", Templates: map[string]string{"c": "c"}} - keycloak := catalogue.Manifest{Module: "keycloak", Provides: []catalogue.Offer{{Name: "oidc-client", Scope: catalogue.ScopeMesh}}} - proxy := catalogue.Manifest{Module: "route-proxy", Provides: []catalogue.Offer{{Name: "route", Scope: catalogue.ScopeMesh, - AnsweredBy: catalogue.AnsweredByHelper}}} - store := catalogue.Manifest{Module: "store", Provides: []catalogue.Offer{{Name: "database", Scope: catalogue.ScopeMesh}}} - goTools := catalogue.Manifest{Module: "go", Provides: []catalogue.Offer{{Name: "go-toolchain", Scope: catalogue.ScopeNode}}} - gitea := catalogue.Manifest{Module: "gitea", Provides: []catalogue.Offer{{Name: "forge-account", Scope: catalogue.ScopeMesh}}} - answering := func(p string) []catalogue.Manifest { - for _, m := range []catalogue.Manifest{keycloak, proxy, store, goTools, gitea} { - if m.Provides[0].Name == p { - return []catalogue.Manifest{m} - } - } - return nil - } - for _, c := range []struct { - name string - manifest catalogue.Manifest - missing string // the provision named, "" for none: the want moves - }{ - {"a handed module's database with no key", catalogue.Manifest{Module: "mail", Moved: true, Handed: handed, - Requires: []string{"database"}}, "database"}, - {"a module not handed waits for no login the controller makes (grafana on ace, oidc-client)", - catalogue.Manifest{Module: "mail", Requires: []string{"oidc-client"}}, ""}, - {"a module not handed waits for its route, which the helper answers (grafana's route)", - catalogue.Manifest{Module: "mail", Requires: []string{"oidc-client", "route"}}, "route"}, - {"a handed builder waits for no toolchain on its own machine (mesh-build-go, go-toolchain)", - catalogue.Manifest{Module: "mail", Moved: true, Handed: handed, Requires: []string{"go-toolchain"}}, ""}, - {"and does wait for its forge account", catalogue.Manifest{Module: "mail", Moved: true, Handed: handed, - Requires: []string{"go-toolchain", "forge-account"}}, "forge-account"}, - } { - m, n := ready() - m.Manifest, m.Answering = c.manifest, answering - m.Provisions = func(string) map[string]meshprovisionerclient.Provision { - return map[string]meshprovisionerclient.Provision{} - } - out := Wants(m, heardOfHaves(n), wantedOld(), nil) - if c.missing == "" { - if out.Wants["g14"].Fp != fNew || len(m.MissingProvisions("g14")) != 0 { - t.Errorf("%s: held %+v", c.name, out.Rules["g14"]) - } - continue - } - if out.Rules["g14"].Rule != mwc.RuleProvisions || !strings.Contains(out.Rules["g14"].Words, - "provision "+c.missing+" has no key yet") || len(m.MissingProvisions("g14")) != 1 { - t.Errorf("%s: %+v", c.name, out.Rules["g14"]) + for node, w := range c.wrote { + if got, ok := out.Wants[node]; !ok || !mwc.SameWant(normal(got), w) { + t.Errorf("%s: %s's want = %+v (written %v), want %+v", c.name, node, got, ok, w) } } } - // ADR 0603 §5: a requirement nobody offers is an unknown provider and waits, loudly; one something offers (a module's own - // name, a module the mesh provides itself) is left to catalogue.Awaited as before and does not. - for _, c := range []struct { - name, requires string - offered bool - }{{"a module's own name (node-tools)", "node-tools", true}, {"a module the mesh provides itself (mesh-addressing)", "mesh-addressing", true}, - {"a provision nobody offers", "database", false}} { - m, n := ready() - m.Manifest.Requires = []string{c.requires} - m.Answering = func(string) []catalogue.Manifest { return nil } - m.Offered = func(p string) bool { return c.offered && p == c.requires } - m.Provisions = func(string) map[string]meshprovisionerclient.Provision { - return map[string]meshprovisionerclient.Provision{} - } - out := Wants(m, heardOfHaves(n), wantedOld(), nil) - if got := out.Wants["g14"].Fp == fNew; got != c.offered || (!c.offered && (out.Rules["g14"].Rule != mwc.RuleProvisions || - !strings.Contains(out.Rules["g14"].Words, "no module offers database yet") || !strings.Contains(out.Rules["g14"].Words, "past its bound"))) { - t.Errorf("%s: want %+v, held by %+v", c.name, out.Wants["g14"], out.Rules["g14"]) - } - } - // 1960: rule 5 holds no want, and no module is left without a first want for a provider down or a node offline; the - // provider down is said (Unreached). novox on 2026-10-29: route-proxy down held de-spiegel and link2pay, minio not - // healthy held photos, and novox offline left a module with no canary. - for _, c := range []struct { - name, module, provision, provider string - have *nodeengine.HaveByNodeModuleValue // what novox runs of the module; nil: a new assignment - before map[string]mwc.Want - online bool - want string - }{ - {"de-spiegel, route-proxy down, old form", "de-spiegel", "route", "route-proxy", - &nodeengine.HaveByNodeModuleValue{Fp: "build-1", Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown}, nil, true, fNew}, - {"link2pay, route-proxy down, new assignment", "link2pay", "route", "route-proxy", nil, nil, true, fNew}, - {"photos, minio not healthy, moving its want", "photos", "s3-bucket", "minio", - &nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}, - map[string]mwc.Want{"novox": {Fp: fOld, Composed: dig(fOld)}}, true, fNew}, - {"novox offline, no canary, new assignment", "link2pay", "route", "route-proxy", nil, nil, false, fNew}, - {"novox offline, no canary, moving its want", "photos", "s3-bucket", "minio", - &nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}, - map[string]mwc.Want{"novox": {Fp: fOld, Composed: dig(fOld)}}, false, fOld}, - } { - m := Module{Name: c.module, FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"novox"}, - Manifest: catalogue.Manifest{Module: c.module, Requires: []string{c.provision}}} - n := Nodes{Online: map[string]bool{"novox": c.online}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"novox": { - c.provider: {Fp: fpOf('p'), Phase: nodeengineclient.PhaseFailed, Health: nodeengineclient.HealthFailed}}}, - Providers: map[string]map[string][]string{c.provision: {c.provider: {"novox"}}}} - if c.have != nil { - n.Haves["novox"][c.module] = *c.have - } - out := Wants(m, heardOfHaves(n), c.before, nil) - w, written := out.Wants["novox"] - if !written { - w = c.before["novox"] - } - if w.Fp != c.want || w.Removed { - t.Errorf("%s: novox's want = %+v, want %s", c.name, w, short(c.want)) - } - if _, on := out.Neither["novox"]; on { - t.Errorf("%s: on neither list: %+v", c.name, out.Neither) - } - if c.online && !strings.Contains(out.Unreached["novox"], c.provision) { - t.Errorf("%s: the provider down is not said: %q", c.name, out.Unreached["novox"]) - } - } - - // 1126/AC-SW1, AC-SW2, AC-SW3: a want written Self while g14 was placed, kept by a rule after the assignments is gone, names - // the controller's composition of the fingerprint the rule keeps (on 2026-10-27 g14's systemd-resolved, held, and forticlient, - // rule 5, kept their Self wants, which g14 could no longer compose); with no such composition, or while a record ask waits, it - // stays and is said; a placed node's want is Self with no composition. - selfAt := func(fp string) map[string]mwc.Want { - return map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fp, Self: true, Yes: dig(fp)}} - } - atF := func(_ *Module, n *Nodes) { - n.Haves["g14"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: dig(fNew), Phase: nodeengineclient.PhaseFailed, Health: nodeengineclient.HealthOK} - } - for _, c := range []struct { - name string - set func(*Module, *Nodes) - before string - want mwc.Want // the want of g14 written; zero: none written - rule int // the rule g14's holding names - asked bool - }{ - {name: "rule 5 at F", before: fNew, set: func(m *Module, n *Nodes) { - atF(m, n) - n.Haves["ace"]["store"] = nodeengine.HaveByNodeModuleValue{Fp: fpOf('s'), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthWaits} - }, want: mwc.Want{Fp: fNew, Composed: dig(fNew)}}, - {name: "rule 3 at F", before: fNew, set: func(m *Module, n *Nodes) { - atF(m, n) - n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - }, want: mwc.Want{Fp: fNew, Composed: dig(fNew)}, rule: mwc.RuleCanary}, - {name: "rule 3 held back at an older fingerprint", before: fOld, set: func(_ *Module, n *Nodes) { - n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - }, want: mwc.Want{Fp: fOld, Composed: dig(fOld), Newest: fNew, Rule: mwc.RuleCanary, - Words: "the canary ace does not run bbbbbbbbbbbb healthy yet"}, rule: mwc.RuleCanary}, - {name: "held at F with nothing to set back to", before: fNew, set: func(m *Module, n *Nodes) { atF(m, n); m.Held = true }, - want: mwc.Want{Fp: fNew, Composed: dig(fNew), Words: "mail is held at bbbbbbbbbbbb and g14 ran no other fingerprint healthy to set it back to"}, - rule: mwc.RuleNotHeld}, - {name: "held data module", before: fNew, set: func(m *Module, n *Nodes) { - atF(m, n) - m.Held = true - m.Manifest.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Class: catalogue.ClassIrreplaceable}}} - }, want: mwc.Want{Fp: fNew, Composed: dig(fNew), - Words: "mail holds data, so it is not set back from bbbbbbbbbbbb: the operator is asked whether to restore it"}, rule: mwc.RuleNotHeld}, - {name: "a record ask waits", before: fNew, set: func(m *Module, n *Nodes) { - atF(m, n) - recording(m) - m.Held = true - m.Composed = func(_, fp string) (ComposedDigest, bool) { - return ComposedDigest{Digest: dig(fp), Judgement: JudgedRecreates, Against: dig(fOld)}, true - } - }, rule: mwc.RuleNotHeld, asked: true}, - {name: "no composition of F", before: fNew, set: func(m *Module, n *Nodes) { - atF(m, n) - m.Composed = func(node, fp string) (ComposedDigest, bool) { - if node == "g14" { - return ComposedDigest{}, false - } - return composedEverywhere(node, fp) - } - }, rule: mwc.RuleComposed}, - {name: "placed", before: fNew, set: func(m *Module, n *Nodes) { - atF(m, n) - m.Self = func(node string) bool { return node == "g14" } - m.Composed = func(node, fp string) (ComposedDigest, bool) { - if node == "g14" { - return ComposedDigest{Judgement: JudgedNeither}, true - } - return composedEverywhere(node, fp) - } - }, want: mwc.Want{Fp: fNew, Self: true}}, - // A-430a: a placed node's want held at F by the rules names F's release, as a moved want does (novox's want of - // mesh-delivery-pipeline stood as {fp, self}, the hold having rebuilt it without its version). - {name: "placed, held at F, names its release", before: fNew, set: func(m *Module, n *Nodes) { - atF(m, n) - m.Held = true - m.Self = func(node string) bool { return node == "g14" } - m.KeptBuilds = func(fp string) (KeptBuild, bool) { - return KeptBuild{Parts: map[string]string{"mail:image": "sha256:1"}, KeptOn: "g14", Version: "1.2.3"}, fp == fNew - } - }, rule: mwc.RuleNotHeld, want: mwc.Want{Fp: fNew, Self: true, Version: "1.2.3", Yes: dig(fNew), - Words: "mail is held at " + short(fNew) + " and g14 ran no other fingerprint healthy to set it back to"}}, - } { - m, n := ready() - c.set(&m, &n) - before := selfAt(c.before) - if c.name == "placed" { - before["g14"] = mwc.Want{Fp: fNew, Composed: dig(fNew)} - } - out := Wants(m, heardOfHaves(n), before, nil) - if got := out.Wants["g14"]; !reflect.DeepEqual(got, c.want) { - t.Errorf("%s: g14's want = %+v, want %+v", c.name, got, c.want) - } - if got := out.Rules["g14"].Rule; got != c.rule { - t.Errorf("%s: g14 is held by rule %d (%s), want %d", c.name, got, out.Rules["g14"].Words, c.rule) - } - if asked := len(out.Record) == 1 && out.Record[0].Node == "g14"; asked != c.asked { - t.Errorf("%s: record asks %+v", c.name, out.Record) - } - if w, ok := out.Wants["g14"]; ok { - if _, err := mwc.EncodeWant(w); err != nil { - t.Errorf("%s: g14's want does not encode: %v", c.name, err) - } - } - } - - // Rule 6 per node: a grant written for the module's other nodes does not grant a node whose account it does not name - // (issue 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus). - m, n := ready() - grant := grantsstate.Grant{Accounts: []string{"ace.mail", "ace.mail.send"}} - m.GrantedOn = func(node string) bool { return GrantNamesNode(grant, node, "mail") } - out := Wants(m, heardOfHaves(n), wantedOld(), nil) - if w := out.Wants["ace"]; w.Fp != fNew { - t.Errorf("ace, whose account the grant names, wants %+v", w) - } - if got := out.Rules["g14"]; got.Rule != mwc.RuleGrant || !strings.Contains(got.Words, "on g14") { - t.Errorf("g14, whose account the grant does not name, is held by %+v", got) - } - for _, c := range []struct { - accounts []string - node, module string - named bool - }{ - {nil, "g14", "mail", true}, - {[]string{"g14.mail.send"}, "g14", "mail", true}, - {[]string{"g14.mailer"}, "g14", "mail", false}, - {[]string{"ace.mail"}, "g14", "mail", false}, - } { - if got := GrantNamesNode(grantsstate.Grant{Accounts: c.accounts}, c.node, c.module); got != c.named { - t.Errorf("%v names %s of %s: %v, want %v", c.accounts, c.module, c.node, got, c.named) - } - } } -// 1297/B4, ADR 0435 §3: a canary that is offline is passed over: the next node by name is the canary, and moves first. -func TestAnOfflineCanaryIsPassedOver(t *testing.T) { - m, n := ready() - n.Online["ace"] = false - n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - n.Providers["database"]["store"] = []string{"ace", "g14"} - n.Haves["g14"]["store"] = nodeengine.HaveByNodeModuleValue{Fp: fpOf('s'), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - if c := Canary(m.Nodes, n.Online); c != "g14" { - t.Fatalf("canary = %q, want g14", c) - } - out := Wants(m, heardOfHaves(n), wantedOld(), nil) - if out.Wants["g14"].Fp != fNew { - t.Errorf("the canary g14's want = %+v", out.Wants["g14"]) - } - if out.Rules["ace"].Rule != mwc.RuleCanary { - t.Errorf("ace, behind the canary g14, is held back by %+v", out.Rules["ace"]) - } -} - -// 1297/B4: a failed health check of F holds F and sets the nodes that want it back to their last healthy fingerprint. -func TestAFailedHealthHoldsAndSetsBack(t *testing.T) { - m, n := ready() - n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"} - m.Now = time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC) - out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fOld, Composed: dig(fOld)}}, map[string]string{"ace": fOld}) - // 1906/H3: the held record keeps its cause in words: which machine (its node), which health, when, and the node's words. - if out.Held == nil || out.Held.Node != "ace" || out.Held.Words != "health failed, phase running, seen at 2026-10-25T01:00:00Z: connection refused" { - t.Fatalf("held = %+v", out.Held) - } - if w := out.Wants["ace"]; w.Fp != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule != mwc.RuleNotHeld { - t.Errorf("ace's want = %+v, want set back to the last healthy fingerprint", w) - } - if w, ok := out.Wants["g14"]; ok && w.Fp == fNew { - t.Errorf("g14 moved to a held fingerprint: %+v", w) - } - - // 1496, ADR 0557 §1, §2: the same failure while F runs healthy on another node is that node's: F is not held, nothing is set - // back, the failing node keeps its want, and the outcome names it (on ace a newly assigned module's first check held a version healthy for an hour on three nodes). - n.Haves["g14"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - before := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fNew, Composed: dig(fNew)}} - out = Wants(m, heardOfHaves(n), before, map[string]string{"ace": fOld, "g14": fOld}) - if out.Held != nil || len(out.Wants) != 0 { - t.Fatalf("held %+v, wants %+v: want no hold and no want changed", out.Held, out.Wants) - } - if f := out.NodeFailed["ace"]; !slices.Equal(f.Others, []string{"g14"}) || f.Words != "health failed, phase running: connection refused" || - len(out.NodeFailed) != 1 { - t.Errorf("node failed = %+v", out.NodeFailed) - } - - // ADR 0557 §2 (version 9): while one node fails F alone, novox, whose want names fOld, gets no new want naming F; the wants - // at F stay. The canary ace runs F healthy here and g14 fails it, so the canary rule (3) would let novox move: only §2 holds it. - three := m - three.Nodes = []string{"ace", "g14", "novox"} - failedHave := n.Haves["ace"]["mail"] - n.Haves["ace"]["mail"], n.Haves["g14"]["mail"] = n.Haves["g14"]["mail"], failedHave - n.Online["novox"] = true - n.Haves["novox"] = map[string]nodeengine.HaveByNodeModuleValue{"mail": {Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}} - before["novox"] = mwc.Want{Fp: fOld, Composed: dig(fOld)} - out = Wants(three, heardOfHaves(n), before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld}) - if _, alone := out.NodeFailed["g14"]; !alone { - t.Errorf("g14 does not fail alone: %+v", out.NodeFailed) - } - if w, written := out.Wants["novox"]; out.Held != nil || len(out.NodeFailed) != 1 || (written && w.Fp == fNew) || out.Rules["novox"].Rule != mwc.RuleNotHeld { - t.Errorf("held %+v, node failed %+v, novox's want %+v (%+v): want novox kept off F", out.Held, out.NodeFailed, out.Wants["novox"], out.Rules["novox"]) - } - if _, written := out.Wants["g14"]; written { - t.Errorf("g14's want at F was rewritten: %+v", out.Wants["g14"]) - } - // A node that runs F healthy with no want, while g14 fails it alone, gets its want at F and is on no neither list (issue 1945). - n.Haves["novox"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - delete(before, "novox") - out = Wants(three, heardOfHaves(n), before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld}) - if w := out.Wants["novox"]; w.Fp != fNew || len(out.Neither) != 0 { - t.Errorf("novox running F with no want: want %+v, neither %+v; want its want at F and no neither", w, out.Neither) - } - n.Haves["ace"]["mail"], n.Haves["g14"]["mail"] = failedHave, n.Haves["ace"]["mail"] - - // ADR 0557 §2: a witness that is offline keeps the have it last wrote and witnesses nothing (D1); F failing on a second node is - // not one node's failure (D3). Either way F is held and ace is set back. - for _, c := range []struct { - name string - online map[string]bool - novox string - }{ - {"the witness sleeps", map[string]bool{"ace": true, "g14": false, "novox": true}, nodeengineclient.HealthOK}, - {"F fails on a second node", map[string]bool{"ace": true, "g14": true, "novox": true}, nodeengineclient.HealthFailed}, - } { - n.Online = c.online - n.Haves["novox"] = map[string]nodeengine.HaveByNodeModuleValue{"mail": {Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: c.novox}} - if c.novox == nodeengineclient.HealthOK { - n.Haves["novox"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - } - before["novox"] = mwc.Want{Fp: fNew, Composed: dig(fNew)} - out = Wants(three, heardOfHaves(n), before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld}) - if out.Held == nil || out.Held.Node != "ace" || len(out.NodeFailed) != 0 { - t.Errorf("%s: held %+v, node failed %+v: want F held at ace", c.name, out.Held, out.NodeFailed) - } - if w := out.Wants["ace"]; w.Fp != fOld { - t.Errorf("%s: ace's want = %+v, want set back", c.name, w) - } - } -} - -// 1297/B4: a module holding data is not set back; its want stays and says why. -func TestADataModuleIsNotSetBack(t *testing.T) { - m, n := ready() - m.Manifest.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Class: catalogue.ClassIrreplaceable}}} - n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed} - out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fNew}}, map[string]string{"ace": fOld}) - if out.Held == nil { - t.Fatal("F was not held") - } - if w := out.Wants["ace"]; w.Fp != fNew || !strings.Contains(w.Words, "holds data") { - t.Errorf("ace's want = %+v, want it left at F, saying it holds data", w) - } - // 1906/D3: and the node is to be asked whether to restore it - if !slices.Equal(out.Restore, []string{"ace"}) { - t.Errorf("restore asks = %v, want ace", out.Restore) - } -} - -// 1906/D3: a module without data is set back and asked about nothing. -func TestAModuleWithoutDataIsNotAskedToRestore(t *testing.T) { - m, n := ready() - n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed} - out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fNew}}, map[string]string{"ace": fOld}) - if len(out.Restore) != 0 { - t.Errorf("restore asks = %v", out.Restore) - } -} - -// 1297/B4, ADR 0408 §4: a node that leaves the module's nodes gets its want written removed, never deleted; a want that -// is already as the rules give it is not written again. -func TestALeavingNodeIsRemovedAndNothingIsWrittenTwice(t *testing.T) { - m, n := ready() - m.Nodes = []string{"ace"} - out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {V: 1, Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fOld, Composed: dig(fOld)}}, nil) - if w := out.Wants["g14"]; !w.Removed { - t.Errorf("g14's want = %+v, want removed", w) - } - if _, ok := out.Wants["ace"]; ok { - t.Errorf("ace's want, unchanged, was written again: %+v", out.Wants["ace"]) - } -} - -// 1906/D2 (ADR 0539 §6): the seventh rule. A want waits for the composition of the module for that node at F, and names its -// digest once it exists; a node with no want before still gets none until then. -func TestAWantWaitsForItsCompositionAndNamesItsDigest(t *testing.T) { - m, n := ready() - have := map[string]bool{} - m.Composed = func(node, fp string) (ComposedDigest, bool) { - if !have[node+"|"+fp] { - return ComposedDigest{}, false - } - return composedEverywhere(node, fp) - } - out := Wants(m, heardOfHaves(n), wantedOld(), nil) - for node, w := range out.Wants { - if w.Fp == fNew { - t.Errorf("%s's want moved to F before its composition exists: %+v", node, w) - } - } - if r := out.Rules["ace"]; r.Rule != mwc.RuleComposed || !strings.Contains(r.Words, "ace") { - t.Errorf("ace is held back by %+v, want rule 7 naming the node", r) - } - // and a node that never had a want gets none - if out := Wants(m, heardOfHaves(n), nil, nil); len(out.Wants) != 0 { - t.Errorf("wants without a composition: %+v", out.Wants) - } - have["ace|"+fNew] = true - out = Wants(m, heardOfHaves(n), wantedOld(), nil) - if w := out.Wants["ace"]; w.Fp != fNew || w.Composed != dig(fNew) { - t.Errorf("ace's want = %+v, want F naming the composition's digest", w) - } - if w, ok := out.Wants["g14"]; ok && w.Fp == fNew { - t.Errorf("g14, whose composition does not exist, moved: %+v", w) - } -} - -// 1906/D2 (ADR 0539 §5): a new revision of the composition at the same fingerprint reaches the node only through a new want, -// by the same rules: written when they hold, and not while one does not. -func TestARevisionAtTheSameFingerprintIsAWantByTheSameRules(t *testing.T) { - m, n := ready() - revised := "sha256:" + strings.Repeat("9", 64) - m.Composed = func(node, fp string) (ComposedDigest, bool) { - return ComposedDigest{Digest: revised, Judgement: JudgedNeither}, true - } - running := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fNew, Composed: dig(fNew)}} - out := Wants(m, heardOfHaves(n), running, nil) - for _, node := range []string{"ace", "g14"} { - if w := out.Wants[node]; w.Fp != fNew || w.Composed != revised { - t.Errorf("%s's want = %+v, want F naming the revised digest", node, w) - } - } - m.Granted = false - out = Wants(m, heardOfHaves(n), running, nil) - if w, ok := out.Wants["ace"]; ok && w.Composed == revised { - t.Errorf("a revision was written while a rule holds: %+v", w) - } - - // A want held back at an earlier fingerprint follows the new revision of that fingerprint's kept composition, which the - // controller composed again with today's config (novox/hq ADR 0545: on g14 a capacity of 0 after a rollback), while - // the rules still hold F; for a record module whose revision recreates a container, only after the ask. - out = Wants(m, heardOfHaves(n), wantedOld(), nil) - if w := out.Wants["g14"]; w.Fp != fOld || w.Composed != revised || w.Newest != fNew { - t.Errorf("g14, held back at its earlier fingerprint, wants %+v; want it at that fingerprint naming the revised digest", w) - } - m.Composed = func(node, fp string) (ComposedDigest, bool) { - return ComposedDigest{Digest: revised, Judgement: JudgedRecreates, Policy: catalogue.PolicyRecord, - Against: dig(fOld), Words: "recreates"}, true - } - out = Wants(m, heardOfHaves(n), wantedOld(), nil) - if w := out.Wants["g14"]; w.Composed == revised { - t.Errorf("a recorded module's revision that recreates a container was written with no ask: %+v", w) - } - if !slices.ContainsFunc(out.Record, func(r RecordNeed) bool { return r.Node == "g14" && r.Digest == revised }) { - t.Errorf("the revision was not asked: %+v", out.Record) - } -} - -// 1906/D2: a set-back names the composition of the fingerprint it sets back to, and when that is gone the want stays and says so. -func TestASetBackNamesTheOlderCompositionOrStays(t *testing.T) { - m, n := ready() - n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "x"} - running := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}} - m.Composed = func(node, fp string) (ComposedDigest, bool) { - if fp == fOld { - return ComposedDigest{}, false - } - return composedEverywhere(node, fp) - } - out := Wants(m, heardOfHaves(n), running, map[string]string{"ace": fOld}) - if w := out.Wants["ace"]; w.Fp == fOld { - t.Errorf("set back to a fingerprint whose composition is gone: %+v", w) - } - m.Composed = composedEverywhere - out = Wants(m, heardOfHaves(n), running, map[string]string{"ace": fOld}) - if w := out.Wants["ace"]; w.Fp != fOld || w.Composed != dig(fOld) { - t.Errorf("set-back want = %+v", w) - } -} - -// movingTo is a module ready to move, whose composition at F is judged j against the one it runs, and whose digest is dig(F). -func movingTo(j Judgement) (Module, Nodes, map[string]mwc.Want) { - m, n := ready() - m.Composed = func(node, fp string) (ComposedDigest, bool) { - d := ComposedDigest{Digest: dig(fp), Judgement: j} - if j != JudgedNeither { - d.Against, d.Words = dig(fOld), "judged" - } - return d, true - } - return m, n, wantedOld() -} - -func recording(m *Module) { - m.Manifest.Upgrade = &catalogue.UpgradePolicy{Policy: catalogue.PolicyRecord, Why: "a person takes each build"} -} - -// 1906/D4 (ADR 0539 §14): a module of policy record whose move is judged to recreate a container, to move data, or cannot be -// judged, is not written until its ask is answered yes; the same at a new revision of the same fingerprint. -func TestARecordModuleWaitsForItsAskWhenTheMoveIsNotJudgedNeither(t *testing.T) { - for _, j := range []Judgement{JudgedRecreates, JudgedMovesData, JudgedCannot} { - m, n, wanted := movingTo(j) - recording(&m) - out := Wants(m, heardOfHaves(n), wanted, nil) - if w, ok := out.Wants["ace"]; ok { - t.Errorf("%s: ace's want was written without the ask's yes: %+v", j, w) - } - var ace *RecordNeed - for i := range out.Record { - if out.Record[i].Node == "ace" { - ace = &out.Record[i] - } - } - if ace == nil || ace.Digest != dig(fNew) || ace.Judgement != j { - t.Errorf("%s: the asks = %+v, want one for ace naming the digest and the judgement", j, out.Record) - } - // a new revision at the same fingerprint is asked too - same := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fOld)}} - if out := Wants(m, heardOfHaves(n), same, nil); len(out.Record) == 0 || len(out.Wants) != 0 && out.Wants["ace"].Composed == dig(fNew) { - t.Errorf("%s: a new revision at the same fingerprint: %+v", j, out) - } - m.Allowed = func(node, digest string) bool { return node == "ace" && digest == dig(fNew) } - out = Wants(m, heardOfHaves(n), wanted, nil) - if w := out.Wants["ace"]; w.Fp != fNew || w.Composed != dig(fNew) { - t.Errorf("%s: after the yes ace's want = %+v", j, w) - } - } -} - -// 1906/D4: a move judged neither is written at once, whatever the policy; so is a module the node never ran; and a module -// that rolls (not record) is never asked. -func TestAMoveJudgedNeitherOrNeverRunOrNotRecordedIsWrittenAtOnce(t *testing.T) { - m, n, wanted := movingTo(JudgedNeither) - recording(&m) - if out := Wants(m, heardOfHaves(n), wanted, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 { - t.Errorf("neither: %+v", out) - } - m, n, _ = movingTo(JudgedRecreates) - recording(&m) - m.Nodes = []string{"ace"} // g14 runs it with no want, so its first want is asked (1945/AC19): this case is ace's alone - delete(n.Haves["ace"], "mail") - if out := Wants(m, heardOfHaves(n), nil, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 { - t.Errorf("a node that never ran it: %+v", out) - } - // but one that runs an older fingerprint with no want yet is asked like any other (one running F is not: N1 below) - m, n, _ = movingTo(JudgedCannot) - recording(&m) - n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - if out := Wants(m, heardOfHaves(n), nil, nil); out.Wants["ace"].Fp == fNew || len(out.Record) == 0 { - t.Errorf("a node that runs it with no want: %+v", out) - } - m, n, wanted = movingTo(JudgedRecreates) - if out := Wants(m, heardOfHaves(n), wanted, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 { - t.Errorf("a rolled module: %+v", out) - } -} - -// 1906/D4 (ADR 0539 §7, §15): the bus module's new fingerprint is never written by the assigner, but a revision of its -// composition at the same fingerprint that only changes the user list (judged neither) is written at once. -func TestTheBusMovesByTheRulesAndItsUserListRevisionIsWrittenAtOnce(t *testing.T) { - m, n, _ := movingTo(JudgedNeither) - m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}} - // ADR 0601 §1, §2: the bus module's want is written by the rules, as any module's, and no snapshot is asked for it: its node - // backs its streams up before it applies. - if out := Wants(m, heardOfHaves(n), wantedOld(), nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 || len(out.Neither) != 0 { - t.Errorf("the bus did not move by the rules: %+v", out) - } - running := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fOld)}, "g14": {Fp: fNew, Composed: dig(fOld)}} - out := Wants(m, heardOfHaves(n), running, nil) - if w := out.Wants["ace"]; w.Fp != fNew || w.Composed != dig(fNew) { - t.Errorf("a user-list revision of the bus: ace's want = %+v", w) - } - if len(out.Record) != 0 { - t.Errorf("a user-list revision was asked: %+v", out.Record) - } - // ADR 0546: a put owns the node's want from its asking until it ends. While it waits the rules write nothing there; once - // wanted its want stands, marked put and held, and a held bus still takes its user-list revision (ADR 0539 §7). +// ADR 0546: a put owns the node's want from its asking until it ends; while it waits the rule writes nothing there, once wanted its +// want stands, marked put and held, and once released the rule writes the passed version again. +func TestAPutOwnsItsNodesWant(t *testing.T) { + m := released() put := puts.Put{FP: fNew, Hold: true, Why: "the bus's user list", Composed: dig(fNew)} for _, c := range []struct { name string @@ -834,11 +95,11 @@ func TestTheBusMovesByTheRulesAndItsUserListRevisionIsWrittenAtOnce(t *testing.T want *mwc.Want // nil: nothing written on ace }{ {"waiting for the operator", puts.Person, wantedOld(), nil}, - {"wanted", puts.Wanted, wantedOld(), &mwc.Want{Fp: fNew, Composed: dig(fNew), Put: true, Hold: true, Words: "put: the bus's user list"}}, + {"wanted", puts.Wanted, wantedOld(), &mwc.Want{Fp: fNew, Put: true, Hold: true, Words: "put: the bus's user list"}}, {"holding, written already", puts.Holding, map[string]mwc.Want{"ace": PutWant(put)}, nil}, - {"released: the rules again", puts.Released, map[string]mwc.Want{"ace": PutWant(put)}, &mwc.Want{Fp: fNew, Composed: dig(fNew)}}, - {"not heard: a held want at an older fingerprint stands", "", map[string]mwc.Want{"ace": {Fp: fOld, Composed: dig(fOld), Put: true, Hold: true, Words: "put: back"}}, nil}, - {"not heard: a held want the rules would rewrite stands", "", map[string]mwc.Want{"ace": PutWant(put)}, nil}, + {"released: the rule again", puts.Released, map[string]mwc.Want{"ace": PutWant(put)}, &mwc.Want{Fp: fNew, Version: "1.2.3", Pipeline: "main.12", Rank: 2}}, + {"not heard: a held want at an older fingerprint stands", "", map[string]mwc.Want{"ace": {Fp: fOld, Put: true, Hold: true, Words: "put: back"}}, nil}, + {"not heard: a held want the rule would rewrite stands", "", map[string]mwc.Want{"ace": PutWant(put)}, nil}, } { p := put p.Step = c.step @@ -846,30 +107,28 @@ func TestTheBusMovesByTheRulesAndItsUserListRevisionIsWrittenAtOnce(t *testing.T if c.step == "" { m.Puts = nil } - out := Wants(m, heardOfHaves(n), c.wanted, nil) - got, wrote := out.Wants["ace"] + got, wrote := Wants(m, c.wanted).Wants["ace"] switch { case c.want == nil && wrote: - t.Errorf("put %s: the rules wrote ace's want %+v", c.name, got) + t.Errorf("put %s: the rule wrote ace's want %+v", c.name, got) case c.want != nil && (!wrote || !mwc.SameWant(normal(got), *c.want)): t.Errorf("put %s: ace's want = %+v (written %v), want %+v", c.name, got, wrote, *c.want) } } - userList := "sha256:" + strings.Repeat("7", 64) - m.Composed = func(node, fp string) (ComposedDigest, bool) { - return ComposedDigest{Digest: userList, Judgement: JudgedNeither, UserListOnly: true}, true - } - n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: dig(fNew), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - held := put - held.Step = puts.Holding - m.Puts = map[string]puts.Put{"ace": held} - if w := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": PutWant(held)}, nil).Wants["ace"]; w.Composed != userList || !w.Put || !w.Hold { - t.Errorf("a held bus did not take its user-list revision as a put: %+v", w) - } - m.Puts = nil stepsOfAPut(t) } +// §520: health is read from the node-engine seat's `running` alone, and a build of which nothing was heard is not healthy. +func TestHealthIsReadFromRunningAlone(t *testing.T) { + yes, no := true, false + n := Nodes{Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"ace": {"mail": {Fp: fNew, Phase: nodeengineclient.PhaseFailed, Health: nodeengineclient.HealthFailed}}}, + Running: map[string]map[string]nodeengine.Running{"ace": {"mail": {Fingerprint: fNew, Healthy: &yes}}, "g14": {"mail": {Fingerprint: fNew, Healthy: &no}}}} + if !n.RunsHealthy("ace", "mail", fNew) || n.RunsHealthy("g14", "mail", fNew) || n.RunsHealthy("ace", "mail", fOld) || n.RunsHealthy("novox", "mail", fNew) { + t.Errorf("health read from running: ace %v, g14 %v, ace at fOld %v, novox %v", n.RunsHealthy("ace", "mail", fNew), + n.RunsHealthy("g14", "mail", fNew), n.RunsHealthy("ace", "mail", fOld), n.RunsHealthy("novox", "mail", fNew)) + } +} + // stepsOfAPut takes a put through its steps (ADR 0546 clauses 3 to 8): the build, the controller's composition, the // operator's yes, the bus's snapshot, the want, and the have. func stepsOfAPut(t *testing.T) { @@ -1004,706 +263,3 @@ func stepsOfAPut(t *testing.T) { } } } - -// 1906/D4 (ADR 0539 §16): the answer to a failed backup stays in the want while its target stays. -func TestTheBackupAnswerStaysInTheWantWhileItsTargetStays(t *testing.T) { - m, n := ready() - running := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew), Backup: mwc.BackupSkip, Words: BackupAnswerWords("skip", 0)}} - if w, ok := Wants(m, heardOfHaves(n), running, nil).Wants["ace"]; ok { - t.Errorf("the answer was erased: %+v", w) - } - // A placed node composes its own: its want names the build alone, and the answer stays in it all the same (the digest its - // have names is not in the want, so a want kept for its digest would erase the answer at every pass). - m.Self = func(string) bool { return true } - placed := map[string]mwc.Want{"ace": {Fp: fNew, Self: true, Backup: mwc.BackupSkip, Words: BackupAnswerWords("skip", 0)}} - if w, ok := Wants(m, heardOfHaves(n), placed, nil).Wants["ace"]; ok { - t.Errorf("the answer in a placed node's want was erased: %+v", w) - } -} - -// 1906/D4 (ADR 0539 §19): a hold released by the operator is not held again by a health check that failed before, and the -// wants move by the rules again. -func TestAReleasedHoldIsNotHeldAgain(t *testing.T) { - m, n := ready() - n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "x"} - m.Released = true - out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fOld, Composed: dig(fOld), Newest: fNew, Rule: mwc.RuleNotHeld}}, map[string]string{"ace": fOld}) - if out.Held != nil { - t.Errorf("a released fingerprint was held again: %+v", out.Held) - } - if w := out.Wants["ace"]; w.Fp != fNew || w.Composed != dig(fNew) || w.Rule != 0 { - t.Errorf("ace's want = %+v, want the held fingerprint written again", w) - } -} - -// 1906/D4: a data module whose restore ask the operator answered "restore" is set back like any other, losing the newer writes; -// answered "leave stopped" (or not answered) it stays. -func TestARestoreAnswerSetsADataModuleBack(t *testing.T) { - m, n := ready() - m.Manifest.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Class: catalogue.ClassIrreplaceable}}} - n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed} - running := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}} - m.Restored = func(node string) bool { return false } - if w, ok := Wants(m, heardOfHaves(n), running, map[string]string{"ace": fOld}).Wants["ace"]; ok && w.Fp == fOld { - t.Errorf("set back without the operator's restore: %+v", w) - } - m.Restored = func(node string) bool { return node == "ace" } - if w := Wants(m, heardOfHaves(n), running, map[string]string{"ace": fOld}).Wants["ace"]; w.Fp != fOld || w.Composed != dig(fOld) { - t.Errorf("after restore ace's want = %+v", w) - } -} - -// 1906/D4 (ADR 0539 §14): the policy the controller holds beside the composition decides, the operator's own choice included; -// a composition with none (made by hand) falls back to what the manifest derives. -func TestTheRecordPolicyIsTheOneTheControllerHolds(t *testing.T) { - m, n, wanted := movingTo(JudgedMovesData) - with := func(policy, from string) { - inner := m.Composed - m.Composed = func(node, fp string) (ComposedDigest, bool) { - d, ok := inner(node, fp) - d.Policy, d.PolicyFrom = policy, from - return d, ok - } - } - with(catalogue.PolicyRecord, "person") // a roll module the operator set to record - if out := Wants(m, heardOfHaves(n), wanted, nil); len(out.Record) == 0 || out.Wants["ace"].Fp == fNew { - t.Errorf("the operator's record policy was not honoured: %+v", out) - } - recording(&m) // the manifest says record, the operator chose roll - with("roll", "person") - if out := Wants(m, heardOfHaves(n), wanted, nil); len(out.Record) != 0 || out.Wants["ace"].Fp != fNew { - t.Errorf("the operator's roll policy was not honoured: %+v", out) - } - with("", "") // made by hand: the manifest decides - if out := Wants(m, heardOfHaves(n), wanted, nil); len(out.Record) == 0 { - t.Errorf("a hand-made composition fell through the manifest's record policy: %+v", out) - } - // 1953/D4 (ADR 0565 §7): a person's choice at the assigner is over the controller's beside the composition. - with(catalogue.PolicyRecord, "person") - m.Chosen = &catalogue.PersonPolicy{V: catalogue.PersonPolicyVersion, Module: m.Name, Policy: catalogue.PolicyRoll, Why: "w", By: "op"} - if out := Wants(m, heardOfHaves(n), wanted, nil); len(out.Record) != 0 || out.Wants["ace"].Fp != fNew { - t.Errorf("a person's roll at the assigner was not honoured over the controller's record: %+v", out) - } -} - -// A node that runs the bus with no readable want gets its want by the rules, at the fingerprint it runs or at a new one alike: the -// bus has no bar of its own (ADR 0601 §1). -func TestTheBusWithNoWantIsComparedWithWhatItRuns(t *testing.T) { - m, n, _ := movingTo(JudgedNeither) - m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}} - n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - if w, ok := Wants(m, heardOfHaves(n), nil, nil).Wants["ace"]; !ok || w.Fp != fNew { - t.Errorf("the bus runs F with no want and was not given one: %+v %v", w, ok) - } - n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - if w, ok := Wants(m, heardOfHaves(n), nil, nil).Wants["ace"]; !ok || w.Fp != fNew { - t.Errorf("the bus running an older fingerprint with no want was not given its want by the rules: %+v %v", w, ok) - } -} - -// Review round 1, item 1 (ADR 0539 §7): a revision of the bus module's composition that differs only in its user list is written -// at once, at the fingerprint the node runs, before the record hold, whatever the controller judges of the file's mount (it -// says recreates) and whatever policy the bus has (record, from the bus). The shape is the controller's: judged recreates against -// the composition the node runs, policy record, UserListOnly set. -func TestAUserListOnlyRevisionOfTheBusIsWrittenAtOnceBeforeTheRecordHold(t *testing.T) { - m, n, _ := movingTo(JudgedRecreates) - m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}} - userList := "sha256:" + strings.Repeat("7", 64) - m.Composed = func(node, fp string) (ComposedDigest, bool) { - d := ComposedDigest{Digest: dig(fp), Judgement: JudgedRecreates, Against: dig(fOld), Words: "the user list file is mounted", - Policy: catalogue.PolicyRecord, PolicyFrom: "the bus"} - if fp == fOld { - d.Digest, d.UserListOnly = userList, true // the composition at the fingerprint the node runs, with the new user list - } - return d, true - } - n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - for name, wanted := range map[string]map[string]mwc.Want{ - "a want at the fingerprint it runs": {"ace": {Fp: fOld, Composed: dig(fOld)}}, - "no readable want": nil, - } { - out := Wants(m, heardOfHaves(n), wanted, nil) - if w := out.Wants["ace"]; w.Fp != fOld || w.Composed != userList { - t.Errorf("%s: ace's want = %+v, want the running fingerprint naming the user-list revision", name, w) - } - if len(out.Record) != 0 { - t.Errorf("%s: the user-list revision was asked: %+v", name, out.Record) - } - } - // the same flag on a module that is not the bus changes nothing - m.Manifest.Provides = nil - if out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fOld, Composed: dig(fOld)}}, nil); out.Wants["ace"].Composed == userList { - t.Errorf("a module that is not the bus was written at once: %+v", out.Wants["ace"]) - } -} - -// Review round 2, N1 (ADR 0539 §9, §11): right after the switch no node has a readable want and every have comes from the old node-engine -// with a fingerprint and no composition, so every move is judged cannot-judge. A node running F with such a have gets its want at F -// with no record hold, for a record module and for the bus alike (the node agent applies nothing when the composition equals the -// declaration it kept); a new fingerprint still waits for its ask. -func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) { - for name, bus := range map[string]bool{"a record module": false, "the bus": true} { - // only a composition the controller judged unchanged (it equals the declaration the node last applied, ADR 0540 §9) skips the hold - unchanged := func(node, fp string) (ComposedDigest, bool) { - return ComposedDigest{Digest: dig(fp), Judgement: JudgedUnchanged}, true - } - m, n, _ := movingTo(JudgedCannot) - recording(&m) - m.Nodes = []string{"ace"} - if bus { - m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}} - } - m.Composed = unchanged - n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} // no composition named - out := Wants(m, heardOfHaves(n), nil, nil) - if w := out.Wants["ace"]; w.Fp != fNew || w.Composed != dig(fNew) { - t.Errorf("%s: ace's want = %+v, want F naming its composition", name, w) - } - if len(out.Record) != 0 { - t.Errorf("%s: asked at the switch: %+v", name, out.Record) - } - // a composition that differs from the declaration the node kept (a changed setting or secret), and one the controller did not judge, - // still wait for the ask - for what, j := range map[string]Judgement{"differing": JudgedRecreates, "unjudged": JudgedCannot, "judged neither": JudgedNeither} { - m2, n2, _ := movingTo(j) - recording(&m2) - m2.Nodes = []string{"ace"} - if bus { - m2.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}} - } - n2.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - out := Wants(m2, heardOfHaves(n2), nil, nil) - if j == JudgedNeither { - if out.Wants["ace"].Fp != fNew || len(out.Record) != 0 { - t.Errorf("%s, %s: %+v", name, what, out) - } - continue - } - if w, ok := out.Wants["ace"]; ok || len(out.Record) == 0 { - t.Errorf("%s, %s: written at the switch with no yes: %+v %+v", name, what, w, out.Record) - } - } - // a have that names a composition is judged as before - m.Composed = func(node, fp string) (ComposedDigest, bool) { - return ComposedDigest{Digest: dig(fp), Judgement: JudgedRecreates, Against: dig(fOld), Words: "judged"}, true - } - n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: dig(fOld), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - if out := Wants(m, heardOfHaves(n), nil, nil); out.Wants["ace"].Fp == fNew && len(out.Record) == 0 { - t.Errorf("%s: a have naming a composition skipped the hold: %+v", name, out) - } - // a new fingerprint still asks (the bus is barred instead) - n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - out = Wants(m, heardOfHaves(n), nil, nil) - if w, ok := out.Wants["ace"]; ok { - t.Errorf("%s: a new fingerprint was written with no yes: %+v", name, w) - } - if !bus && len(out.Record) == 0 { - t.Errorf("%s: a new fingerprint was not asked", name) - } - } -} - -// 1906/H3: a failed check whose node gave no words is held saying so, never with empty words. -func TestAHoldOfAFailureWithoutWordsSaysSo(t *testing.T) { - m, n := ready() - n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed} - out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}}, map[string]string{"ace": fOld}) - if out.Held == nil || out.Held.Words != "health failed, phase running: the node gave no words" { - t.Fatalf("held = %+v", out.Held) - } -} - -// 1906/H2: given a held fingerprint that every node running the module now runs healthy, when the assigner passes, then the -// hold is lifted by itself and the wants move to it by the rules again. A node set back to another fingerprint, a node whose -// check fails, or a hold the operator answered "keep" leaves it held; a node that does not run the module at all does not count. -func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) { - everywhere := func() (Module, Nodes) { - m, n := ready() - m.Held = true - n.Haves["g14"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - return m, n - } - m, n := everywhere() - out := Wants(m, heardOfHaves(n), wantedOld(), map[string]string{"ace": fOld, "g14": fOld}) - if !out.Lifted || out.Held != nil { - t.Fatalf("lifted = %v, held = %+v: want the hold lifted", out.Lifted, out.Held) - } - for _, node := range []string{"ace", "g14"} { - if w := out.Wants[node]; w.Fp != fNew || w.Rule != 0 { - t.Errorf("%s's want = %+v, want it moved to the lifted fingerprint", node, w) - } - } - - m, n = everywhere() - delete(n.Haves["g14"], "mail") - if out := Wants(m, heardOfHaves(n), wantedOld(), nil); !out.Lifted { - t.Error("a node that does not run the module kept the hold") - } - - for name, change := range map[string]func(*Module, *Nodes){ - "set back": func(_ *Module, n *Nodes) { - n.Haves["g14"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - }, - "failing": func(_ *Module, n *Nodes) { - n.Haves["g14"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed} - }, - "kept": func(m *Module, _ *Nodes) { m.Kept = true }, - "no haves": func(_ *Module, n *Nodes) { delete(n.Haves["g14"], "mail"); delete(n.Haves["ace"], "mail") }, - "not held": func(m *Module, _ *Nodes) { m.Held = false }, - } { - m, n := everywhere() - change(&m, &n) - if out := Wants(m, heardOfHaves(n), wantedOld(), map[string]string{"ace": fOld, "g14": fOld}); out.Lifted { - t.Errorf("%s: the hold was lifted", name) - } - } -} - -// x11 is screen-lock as on g14 and shanks on 2026-10-25 (issue 1945): built, granted and composed at F and at the older fOld -// on both, requiring x11-display, whose provider xorg runs on each node with its health unknown; each node runs screen-lock at fOld -// (health unknown) and holds no want for it. The canary g14 does not run F. -func x11() (Module, Nodes) { - m := Module{Name: "screen-lock", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"g14", "shanks"}, - GrantedAt: func(fp string) bool { return fp == fNew || fp == fOld }, - Manifest: catalogue.Manifest{Module: "screen-lock", Requires: []string{"x11-display"}}} - n := Nodes{ - Online: map[string]bool{"g14": true, "shanks": true}, - Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{ - "g14": {"screen-lock": {Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown}, - "xorg": {Fp: fpOf('x'), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown}}, - "shanks": {"screen-lock": {Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown}, - "xorg": {Fp: fpOf('x'), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown}}, - }, - Providers: map[string]map[string][]string{"x11-display": {"xorg": {"g14", "shanks"}}}, - } - return m, n -} - -// 1945/AC19: given a node that runs an older fingerprint of the new form, composed and granted, with no want, held back from F by -// the providers or the canary rule, when the assigner passes, then its first want names the fingerprint it runs with that -// composition, never the newest, and says the newest it is held back from; the canary is never passed over. -func TestANodeRunningAModuleWithNoWantGetsItsFirstWantAtWhatItRuns(t *testing.T) { - m, n := x11() - out := Wants(m, heardOfHaves(n), nil, nil) - // 1960: the providers rule holds no want: the canary g14 gets F, its provider's health unknown said - if w := out.Wants["g14"]; w.Fp != fNew || out.Unreached["g14"] == "" { - t.Errorf("g14's want = %+v (unreached %q), want F, the provider said", w, out.Unreached["g14"]) - } - if w := out.Wants["shanks"]; w.Fp != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule != mwc.RuleCanary { - t.Errorf("shanks's want = %+v, want its first want at the fingerprint it runs, held back from F by the canary", w) - } - if len(out.Neither) != 0 { - t.Errorf("left on neither list: %+v", out.Neither) - } - // with the provider healthy, only the canary holds shanks back: still at what it runs - m, n = x11() - n.Haves["g14"]["xorg"] = nodeengine.HaveByNodeModuleValue{Fp: fpOf('x'), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - n.Haves["shanks"]["xorg"] = n.Haves["g14"]["xorg"] - if w := Wants(m, heardOfHaves(n), nil, nil).Wants["shanks"]; w.Fp != fOld || w.Rule != mwc.RuleCanary { - t.Errorf("shanks held back by the canary with no want: %+v, want at fOld by rule 3", w) - } - // a node already running F gets its want at F; a removed want is no want either - m, n = x11() - n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown} - out = Wants(m, heardOfHaves(n), map[string]mwc.Want{"g14": {Removed: true}}, nil) - if w := out.Wants["g14"]; w.Fp != fNew || w.Composed != dig(fNew) || w.Removed { - t.Errorf("g14 running F with a removed want = %+v, want F", w) - } -} - -// 1945/AC19: a node running a fingerprint that is not of the new form, or one whose composition or grant does not exist, gets no -// want past the rules, and is named on neither list; the ordering rules still order every move from a want. -func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) { - for what, change := range map[string]func(*Module, *Nodes){ - "an old-form build": func(_ *Module, n *Nodes) { - n.Haves["shanks"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: "build-2", Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown} - }, - "no grant": func(m *Module, _ *Nodes) { m.GrantedAt = func(fp string) bool { return fp == fNew } }, - "no composition": func(m *Module, _ *Nodes) { - m.Composed = func(_, fp string) (ComposedDigest, bool) { - return ComposedDigest{Digest: dig(fp), Judgement: JudgedNeither}, fp == fNew - } - }, - } { - m, n := x11() - change(&m, &n) - out := Wants(m, heardOfHaves(n), nil, nil) - if what == "an old-form build" { - // the canary g14 does not run F healthy: no want, on neither list by rule 3 (issue 1945) - if w, ok := out.Wants["shanks"]; ok { - t.Errorf("%s: shanks got a want: %+v", what, w) - } - if h, ok := out.Neither["shanks"]; !ok || h.Rule != mwc.RuleCanary { - t.Errorf("%s: shanks on neither list = %+v (named %v), want rule 3", what, h, ok) - } - continue - } - // 1960: no want can name what it runs, so its first want names F rather than none: never on neither list - if w := out.Wants["shanks"]; w.Fp != fNew { - t.Errorf("%s: shanks's want = %+v, want F", what, w) - } - if h, ok := out.Neither["shanks"]; ok { - t.Errorf("%s: shanks on neither list = %+v", what, h) - } - } - m, n := x11() - before := map[string]mwc.Want{"g14": {Fp: fOld, Composed: dig(fOld)}, "shanks": {Fp: fOld, Composed: dig(fOld)}} - out := Wants(m, heardOfHaves(n), before, nil) - if len(out.Neither) != 0 { - t.Errorf("nodes holding a want named on neither list: %+v", out.Neither) - } - // the canary g14 moves to F (its provider's health no rule); shanks waits for it - if w := out.Wants["shanks"]; w.Fp != "" && w.Fp != fOld { - t.Errorf("shanks's want moved from its want past the canary: %+v", w) - } - if w := out.Wants["g14"]; w.Fp != fNew { - t.Errorf("the canary g14's want = %+v, want F", w) - } -} - -// 1945/AC20: a new assignment (a node with no have of the module) waiting for the canary, the build or the grant gets no want and is -// never named on neither list: only a node that runs the module, or whose have names it, is; a module with no fingerprint of the new -// form goes with the node's own composition and is never named. -func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) { - m, n := x11() - delete(n.Haves["shanks"], "screen-lock") - out := Wants(m, heardOfHaves(n), nil, nil) - // 1960: the canary orders moves, never a first want: the new assignment is wanted at F - if w := out.Wants["shanks"]; w.Fp != fNew { - t.Errorf("a new assignment while the canary does not run F: %+v, want F", w) - } - if _, ok := out.Neither["shanks"]; ok { - t.Errorf("a new assignment is named on neither list: %+v", out.Neither) - } - m.Built = false - if out := Wants(m, heardOfHaves(n), nil, nil); len(out.Neither) != 1 || out.Neither["g14"].Rule != mwc.RuleBuildDone { - t.Errorf("with no build: neither = %+v, want g14 alone (it runs the module), by rule 1", out.Neither) - } - // 1945 (the assigner holds a first want back silently): with no build, a node whose want was removed and that does not run - // the module has the reason written into its removed want, once; a node with no key gets no want written, and a node that - // runs the module is on neither list, its removed want left as it is. - gone := mwc.Want{Removed: true, Words: "screen-lock is no longer assigned to shanks"} - for _, c := range []struct { - name string - before map[string]mwc.Want - want map[string]mwc.Want - }{ - {"no key: nothing written", nil, map[string]mwc.Want{}}, - {"a removed want: it says why", map[string]mwc.Want{"shanks": gone}, - map[string]mwc.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is being built"}}}, - {"a removed want that says why already: nothing written", map[string]mwc.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is being built"}}, - map[string]mwc.Want{}}, - {"a node that runs it: on neither list, its removed want left", map[string]mwc.Want{"g14": gone}, map[string]mwc.Want{}}, - } { - out := Wants(m, heardOfHaves(n), c.before, nil) - if len(out.Wants) != len(c.want) { - t.Errorf("%s: wants = %+v, want %+v", c.name, out.Wants, c.want) - continue - } - for node, w := range c.want { - if !mwc.SameWant(normal(out.Wants[node]), w) { - t.Errorf("%s: want of %s = %+v, want %+v", c.name, node, out.Wants[node], w) - } - } - } - m.FP = "" - if out := Wants(m, heardOfHaves(n), nil, nil); len(out.Neither) != 0 { - t.Errorf("a module of the node's own composition named on neither list: %+v", out.Neither) - } -} - -// 1945/AC21: a node whose have names an old-form build (no fingerprint of the new form, so no composition of it) can never be -// named in a want at what it runs; its first want names F, under the canary rule as usual and with everything F needs and the -// providers rule. Given the canary runs F healthy, shanks's first want names F; given it does not yet, shanks gets no want and -// stays on neither list; the providers hold no want (issue 1960). The old-form node that is itself the canary gets F once its providers run healthy. -func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) { - oldForm := func(n Nodes, node string) { - n.Haves[node]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: "build-1792260876956849831", Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown} - } - providersHealthy := func(n Nodes) { - for _, node := range []string{"g14", "shanks"} { - n.Haves[node]["xorg"] = nodeengine.HaveByNodeModuleValue{Fp: fpOf('x'), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - } - } - // the canary g14 runs F healthy: shanks's first want names F - m, n := x11() - providersHealthy(n) - oldForm(n, "shanks") - n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - out := Wants(m, heardOfHaves(n), nil, nil) - if w := out.Wants["shanks"]; w.Fp != fNew || w.Composed != dig(fNew) || w.Newest != "" || w.Rule != 0 { - t.Errorf("canary healthy on F: shanks's want = %+v, want F", w) - } - if _, ok := out.Neither["shanks"]; ok { - t.Errorf("canary healthy on F: shanks still on neither list: %+v", out.Neither) - } - // the canary does not run F healthy yet: no want, and shanks stays on neither list by rule 3 - m, n = x11() - providersHealthy(n) - oldForm(n, "shanks") - n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown} - out = Wants(m, heardOfHaves(n), nil, nil) - if w, ok := out.Wants["shanks"]; ok { - t.Errorf("canary not healthy on F: shanks got a want: %+v", w) - } - if h, ok := out.Neither["shanks"]; !ok || h.Rule != mwc.RuleCanary { - t.Errorf("canary not healthy on F: shanks on neither list = %+v (named %v), want rule 3", h, ok) - } - // 1960: the canary healthy on F but the providers not healthy on shanks: F all the same, the provider down said - m, n = x11() - oldForm(n, "shanks") - n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - out = Wants(m, heardOfHaves(n), nil, nil) - if w := out.Wants["shanks"]; w.Fp != fNew || out.Unreached["shanks"] == "" { - t.Errorf("providers not healthy: shanks's want = %+v (unreached %q), want F, the provider said", w, out.Unreached["shanks"]) - } - if _, on := out.Neither["shanks"]; on { - t.Errorf("providers not healthy: shanks on neither list: %+v", out.Neither) - } - // F not granted: nothing, even with the canary healthy - m, n = x11() - providersHealthy(n) - oldForm(n, "shanks") - n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - m.Granted = false - if w, ok := Wants(m, heardOfHaves(n), nil, nil).Wants["shanks"]; ok { - t.Errorf("F not granted: shanks got a want: %+v", w) - } - // the old-form node is the canary itself: F once its providers run healthy - m, n = x11() - providersHealthy(n) - oldForm(n, "g14") - if w := Wants(m, heardOfHaves(n), nil, nil).Wants["g14"]; w.Fp != fNew || w.Composed != dig(fNew) { - t.Errorf("old-form canary: g14's want = %+v, want F", w) - } -} - -// A failed health counts only once the node is done moving: phase running or failed, never backing-up, applying or checking. -// The live row on novox at 15:27:36 on 2026-10-25 ("health failed, phase applying" at the controller's new fingerprint) held the -// canary two seconds before the new build was healthy. -func TestAFailedHealthWhileTheNodeMovesHoldsNothing(t *testing.T) { - for _, phase := range []string{nodeengineclient.PhaseBackingUp, nodeengineclient.PhaseApplying, nodeengineclient.PhaseChecking} { - m := Module{Name: "mesh-controller", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"novox"}, - Manifest: catalogue.Manifest{Module: "mesh-controller"}} - n := Nodes{Online: map[string]bool{"novox": true}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{ - "novox": {"mesh-controller": {Fp: fNew, Phase: phase, Health: nodeengineclient.HealthFailed}}}} - before := map[string]mwc.Want{"novox": {Fp: fNew, Composed: dig(fNew)}} - if out := Wants(m, heardOfHaves(n), before, map[string]string{"novox": fOld}); out.Held != nil { - t.Errorf("phase %s, health failed: held %+v, want no hold while the node moves", phase, out.Held) - } - } - for _, phase := range []string{nodeengineclient.PhaseRunning, nodeengineclient.PhaseFailed} { - m := Module{Name: "mesh-controller", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"novox"}, - Manifest: catalogue.Manifest{Module: "mesh-controller"}} - n := Nodes{Online: map[string]bool{"novox": true}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{ - "novox": {"mesh-controller": {Fp: fNew, Phase: phase, Health: nodeengineclient.HealthFailed}}}} - before := map[string]mwc.Want{"novox": {Fp: fNew, Composed: dig(fNew)}} - if out := Wants(m, heardOfHaves(n), before, map[string]string{"novox": fOld}); out.Held == nil || out.Held.Node != "novox" { - t.Errorf("phase %s, health failed: held %+v, want a hold on novox", phase, out.Held) - } - } -} - -// A hold is lifted by itself once every node that runs the module runs F healthy: a have of a node that is not running the module -// (removed, held for its backup, or still moving) is not counted, so it does not keep the hold up for ever; a node running another -// fingerprint, or failed, still does. -func TestAHoldIsLiftedOverHavesThatAreNotRunning(t *testing.T) { - setup := func(phase string, fp string, health string) (Module, Nodes) { - m := Module{Name: "web", FP: fNew, Built: true, Granted: true, Held: true, Composed: composedEverywhere, Nodes: []string{"ace", "g14"}} - n := Nodes{Online: map[string]bool{"ace": true, "g14": true}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{ - "ace": {"web": {Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}}, - "g14": {"web": {Fp: fp, Phase: phase, Health: health}}}} - return m, n - } - for _, phase := range []string{nodeengineclient.PhaseRemoved, nodeengineclient.PhaseHeld, nodeengineclient.PhaseApplying, nodeengineclient.PhaseBackingUp, nodeengineclient.PhaseChecking} { - m, n := setup(phase, fOld, nodeengineclient.HealthUnknown) - if out := Wants(m, heardOfHaves(n), nil, nil); !out.Lifted { - t.Errorf("g14's have in phase %s kept the hold up", phase) - } - } - for what, h := range map[string]nodeengine.HaveByNodeModuleValue{ - "running another fingerprint": {Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}, - "failed": {Fp: fNew, Phase: nodeengineclient.PhaseFailed, Health: nodeengineclient.HealthFailed}, - } { - m, n := setup(h.Phase, h.Fp, h.Health) - if out := Wants(m, heardOfHaves(n), nil, nil); out.Lifted { - t.Errorf("g14 %s: the hold was lifted", what) - } - } - // a failed health on a removed have holds nothing - m, n := setup(nodeengineclient.PhaseRemoved, fNew, nodeengineclient.HealthFailed) - m.Held = false - if out := Wants(m, heardOfHaves(n), nil, nil); out.Held != nil { - t.Errorf("a removed have with a failed health held F: %+v", out.Held) - } -} - -// A module of policy record whose want was removed but which the node still runs gets its first want through the record ask: a -// removed want is no want, so a move the controller judges to recreate or move data waits for the operator's yes. -func TestAFirstWantAfterARemovedWantGoesThroughTheRecordAsk(t *testing.T) { - m, n := x11() - recording(&m) - m.Composed = func(_, fp string) (ComposedDigest, bool) { - return ComposedDigest{Digest: dig(fp), Judgement: JudgedRecreates, Policy: catalogue.PolicyRecord}, true - } - out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"shanks": {Removed: true}}, nil) - if w, ok := out.Wants["shanks"]; ok { - t.Errorf("shanks's first want after a removed want was written with no record ask: %+v", w) - } - asked := false - for _, r := range out.Record { - asked = asked || (r.Node == "shanks" && r.Digest == dig(fOld) && r.Judgement == JudgedRecreates) - } - if !asked { - t.Errorf("the record asks = %+v, want one for shanks at the composition it runs", out.Record) - } -} - -// 1945/AC23: given a module held at F under rule 2 (the operator holds systemd-resolved on shanks, 2026-10-25), or held by the -// assigner in this pass after a failed health check of F, when a node that runs it holds no want for it, then the node is not on -// neither list: a held module is on the held list, whose hold is said and answered on its own. -func TestAHeldModuleWithNoWantIsNotOnNeitherList(t *testing.T) { - for what, change := range map[string]func(*Module, *Nodes){ - "held at F": func(m *Module, _ *Nodes) { m.Held, m.Kept = true, true }, - "held at F, running an old-form build": func(m *Module, n *Nodes) { - m.Held, m.Kept = true, true - n.Haves["shanks"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: "build-2", Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown} - }, - "held by this pass": func(_ *Module, n *Nodes) { - n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed} - n.Haves["shanks"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: "build-2", Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown} - }, - } { - m, n := x11() - change(&m, &n) - out := Wants(m, heardOfHaves(n), nil, nil) - if len(out.Neither) != 0 { - t.Errorf("%s: a held module is on neither list: %+v", what, out.Neither) - } - if out.Rules["shanks"].Rule != mwc.RuleNotHeld { - t.Errorf("%s: shanks is held back by %+v, want rule 2", what, out.Rules["shanks"]) - } - } -} - -// Main file *A provider before its consumers* (design 47 G §309): from G a provider runs healthy while the build it runs is -// healthy, as the node-engine seat's `running` says it, whatever a newer want of it waits for (its have's phase waiting-yes). -func TestAProviderRunsHealthyByItsRunning(t *testing.T) { - waiting := nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseWaitingYes, Health: nodeengineclient.HealthOK} - for _, c := range []struct { - name string - running map[string]map[string]nodeengine.Running - ok bool - }{ - {"its new want waits for a yes, its old build runs", map[string]map[string]nodeengine.Running{"novox": {"nats": {Node: "novox", Module: "nats", Fingerprint: fOld}}}, true}, - {"its running build is stopped", map[string]map[string]nodeengine.Running{"novox": {"nats": {Node: "novox", Module: "nats", Stopped: true}}}, false}, - {"it said no running", map[string]map[string]nodeengine.Running{"novox": {}}, false}, - {"its running names no build", map[string]map[string]nodeengine.Running{"novox": {"nats": {Node: "novox", Module: "nats"}}}, false}, - {"the seat's events are not heard: the have says it, and waiting-yes is not running", nil, false}, - } { - n := Nodes{Online: map[string]bool{"novox": true}, Running: c.running, - Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"novox": {"nats": waiting}}, - Providers: map[string]map[string][]string{"bus-state": {"nats": {"novox"}}}} - if ok, words := providedHealthy("bus-state", "novox", n); ok != c.ok { - t.Errorf("%s: healthy = %v (%s), want %v", c.name, ok, words, c.ok) - } - } -} - -// A4/AC1 (design 47 H §750, §764: A-436's digests end): a want names a released build by its version, never its parts or a keeping -// node, whatever the node's node-engine states (the former capability want-parts included); a build with no version is named by -// its fingerprint alone. -func TestNoWantNamesParts(t *testing.T) { - released := KeptBuild{Parts: map[string]string{"mail:image": "sha256:1"}, KeptOn: "g14", Version: "1.2.3"} - unversioned := KeptBuild{Parts: map[string]string{"mail:image": "sha256:1"}, KeptOn: "g14"} - for name, row := range map[string]struct { - reads []string - kept func(string) (KeptBuild, bool) - version string - }{ - "released, stating want-parts": {[]string{"want-parts"}, func(fp string) (KeptBuild, bool) { return released, fp == fNew }, "1.2.3"}, - "released, stating none": {[]string{"form:oci"}, func(fp string) (KeptBuild, bool) { return released, fp == fNew }, "1.2.3"}, - "unversioned, stating want-parts": {[]string{"want-parts"}, func(fp string) (KeptBuild, bool) { return unversioned, fp == fNew }, ""}, - "build not kept": {[]string{"want-parts"}, func(string) (KeptBuild, bool) { return KeptBuild{}, false }, ""}, - "no kept builds": {[]string{"want-parts"}, nil, ""}, - "another fingerprint's release": {[]string{"want-parts"}, func(fp string) (KeptBuild, bool) { return released, fp == fOld }, ""}, - } { - m, n := ready() - n.Reads = map[string][]string{"ace": row.reads, "g14": row.reads} - m.KeptBuilds = row.kept - w := Wants(m, heardOfHaves(n), wantedOld(), nil).Wants["ace"] - if w.Fp != fNew || w.Version != row.version || len(w.Parts) != 0 || w.KeptOn != "" { - t.Errorf("%s: ace's want = %+v", name, w) - } - } -} - -// heardOfHaves gives each have that runs the `running` the node-engine seat says of it (design 47 G §520: health is read from -// `running` alone), for a fixture written as haves; a fixture that names its own `running` is left as it is. -func heardOfHaves(n Nodes) Nodes { - if n.Running != nil { - return n - } - n.Running = map[string]map[string]nodeengine.Running{} - for node, byModule := range n.Haves { - n.Running[node] = map[string]nodeengine.Running{} - for module, h := range byModule { - if h.Phase == nodeengineclient.PhaseRunning && h.Fp != "" { - ok := h.Health == nodeengineclient.HealthOK - n.Running[node][module] = nodeengine.Running{Node: node, Module: module, Fingerprint: h.Fp, Healthy: &ok} - } - } - } - return n -} - -// §520: health is read from the node-engine seat's `running` alone, whatever the have says, and a build of which nothing was heard -// is not healthy. -func TestHealthIsReadFromRunningAlone(t *testing.T) { - yes, no := true, false - say := func(node, module, fp string, healthy *bool) nodeengine.Running { - return nodeengine.Running{Node: node, Module: module, Fingerprint: fp, Healthy: healthy} - } - for _, c := range []struct { - name string - running map[string]map[string]nodeengine.Running - rule int - }{ - {"all healthy by running", map[string]map[string]nodeengine.Running{"ace": {"mail": say("ace", "mail", fNew, &yes), "store": say("ace", "store", fpOf('s'), &yes)}}, 0}, - {"a running with no healthy reads healthy", map[string]map[string]nodeengine.Running{"ace": {"mail": say("ace", "mail", fNew, nil), "store": say("ace", "store", fpOf('s'), nil)}}, 0}, - {"the canary says unhealthy, its have says healthy", map[string]map[string]nodeengine.Running{"ace": {"mail": say("ace", "mail", fNew, &no), "store": say("ace", "store", fpOf('s'), &yes)}}, mwc.RuleCanary}, - {"nothing heard of the canary", map[string]map[string]nodeengine.Running{"ace": {"store": say("ace", "store", fpOf('s'), &yes)}}, mwc.RuleCanary}, - } { - m, n := ready() - n.Running = c.running - if got := Wants(m, n, wantedOld(), nil).Rules["g14"].Rule; got != c.rule { - t.Errorf("%s: g14 held by rule %d, want %d", c.name, got, c.rule) - } - } - // rule 5: a provider that says unhealthy, or that nothing was heard of, holds no want but leaves the consumer unreached - m, n := ready() - n.Running = map[string]map[string]nodeengine.Running{"ace": {"mail": say("ace", "mail", fNew, &yes), "store": say("ace", "store", fpOf('s'), &no)}} - if out := Wants(m, n, wantedOld(), nil); len(out.Unreached) == 0 { - t.Errorf("a provider saying unhealthy: no module unreached, %+v", out) - } -} - -// §575: a hold under rule 1 says how the build stands: failed with the error whole, or being built. -func TestRuleOneSaysHowTheBuildStands(t *testing.T) { - for _, c := range []struct { - name string - failed bool - err string - want string - }{ - {"a build that runs", false, "", "the build of mail at " + short(fNew) + " is being built"}, - {"a failed build", true, "transport fault: no responders at mesh-build-go", "the build of mail at " + short(fNew) + " failed: transport fault: no responders at mesh-build-go"}, - } { - m, n := ready() - m.Built, m.BuildFailed, m.BuildError = false, c.failed, c.err - held, rule, words := Holds(m, "g14", n) - if held || rule != mwc.RuleBuildDone || words != c.want { - t.Errorf("%s: held %v rule %d words %q, want %q", c.name, held, rule, words, c.want) - } - } -} diff --git a/modules/mesh-delivery-assigner/internal/assigngraph/runner.go b/modules/mesh-delivery-assigner/internal/assigngraph/runner.go deleted file mode 100644 index 77089d23..00000000 --- a/modules/mesh-delivery-assigner/internal/assigngraph/runner.go +++ /dev/null @@ -1,19 +0,0 @@ -package assigngraph - -import ( - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" -) - -// Runner is the module that holds the seat mesh-pipeline-runner, read from the acting edges' seat -// (A-334); "" when none is read. -func Runner(entries []meshwants.AssignmentEntry) string { - for _, e := range entries { - for _, edge := range e.Edges { - if edge.Kind == EdgeActing && edge.Seat == meshpipelinerunner.Seat { - return edge.Module - } - } - } - return "" -} diff --git a/modules/mesh-delivery-assigner/internal/buckets/buckets.go b/modules/mesh-delivery-assigner/internal/buckets/buckets.go index d84ebb78..a894baaf 100644 --- a/modules/mesh-delivery-assigner/internal/buckets/buckets.go +++ b/modules/mesh-delivery-assigner/internal/buckets/buckets.go @@ -63,10 +63,6 @@ const ( Graph = IndexerModule + "_graph" ModulesRead = IndexerModule + "." + catalog.StateModules ManifestsRead = IndexerModule + "." + catalog.StateManifests - // ProvisionerModule is the holder of the seat mesh-provisioner, named from this module's manifest (reads its provisions); Provisions - // is its state provisions. - ProvisionerModule = "mesh-provisioner" - Provisions = ProvisionerModule + "_provisions" // TellerModule is the module whose hand the operator's answers pass: the one that holds each ask's answer, and the // name the hand-act log records a warrant under. Held equal to the manifest's reads by its test. TellerModule = "mesh-delivery-teller" diff --git a/modules/mesh-delivery-assigner/internal/buckets/buckets_test.go b/modules/mesh-delivery-assigner/internal/buckets/buckets_test.go index 2b9dd207..fa5bcb64 100644 --- a/modules/mesh-delivery-assigner/internal/buckets/buckets_test.go +++ b/modules/mesh-delivery-assigner/internal/buckets/buckets_test.go @@ -12,9 +12,9 @@ func TestBucketsAreTheEarlierNames(t *testing.T) { for name, c := range map[string][2]string{ "Have": {Have, "node-engine_have"}, "ComposedRead": {ComposedRead, "mesh-host.composed"}, "Modules": {Modules, "mesh-delivery-indexer_modules"}, "Manifests": {Manifests, "mesh-delivery-indexer_manifests"}, - "Answers": {Answers, "mesh-delivery-teller_answers"}, - "Grants": {Grants, "mesh-grants_grants"}, - "Provisions": {Provisions, "mesh-provisioner_provisions"}, "TellerModule": {TellerModule, "mesh-delivery-teller"}, + "Answers": {Answers, "mesh-delivery-teller_answers"}, + "Grants": {Grants, "mesh-grants_grants"}, + "TellerModule": {TellerModule, "mesh-delivery-teller"}, } { if c[0] != c[1] { t.Errorf("%s = %q, want %q", name, c[0], c[1]) diff --git a/modules/mesh-delivery-assigner/internal/outside/outside.go b/modules/mesh-delivery-assigner/internal/outside/outside.go index 64d81ebc..47235075 100644 --- a/modules/mesh-delivery-assigner/internal/outside/outside.go +++ b/modules/mesh-delivery-assigner/internal/outside/outside.go @@ -87,21 +87,6 @@ func Causes(conds []Cond, m string, moved func(owner string) bool) []Cause { return out } -// OwnNetworkKey is the key of a machine's own open network condition, which holds the machine as a whole (§706 *M's own -// network*). -func OwnNetworkKey(m string) string { return "machine." + m + ".network" } - -// OwnNetwork are the machines whose own network condition is open among conds. -func OwnNetwork(conds []Cond) map[string]bool { - out := map[string]bool{} - for _, c := range conds { - if p := strings.Split(c.Key, "."); len(p) == 3 && p[0] == "machine" && p[2] == "network" && c.Key == OwnNetworkKey(p[1]) { - out[p[1]] = true - } - } - return out -} - // Wait is one outside wait as the assigner keeps it: the move of Module at FP on Node, or its settle (Settling). Decided is // what ended it for the rest of that fingerprint on that node: `go-on` (counted healthy by the operator's word) or `failed` // (the delivery's failure), so a check that goes on failing is not waited on again. diff --git a/modules/mesh-delivery-assigner/internal/outside/outside_test.go b/modules/mesh-delivery-assigner/internal/outside/outside_test.go index eb39c173..541d4dd2 100644 --- a/modules/mesh-delivery-assigner/internal/outside/outside_test.go +++ b/modules/mesh-delivery-assigner/internal/outside/outside_test.go @@ -31,9 +31,6 @@ func TestCauses(t *testing.T) { if !slices.Equal(got, want) { t.Errorf("causes = %v, want %v (a rewritten naming a moved module, an unreachable not listing ace and ace's own network are none)", got, want) } - if own := OwnNetwork(conds); !own["ace"] || !own["hub2"] || own["hub"] { - t.Errorf("own network = %v", own) - } } func facts(now time.Time, conds ...Cond) Facts { diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index 4b89c73f..e0967a0c 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -50,14 +50,14 @@ func TestManifest(t *testing.T) { // §431, §507: the gate's source is read through its verb `source` and its event `source-moved`, and no `merges` is read. // A-175, A-178: the teller's answers are followed through its first view and its views family, never its bucket. // §389: the watchman's machine-away and machine-back are consumed, never its presence. - want := []string{"mesh-grants.grants", "mesh-host.composed", - "mesh-provisioner.provisions", "node-engine.have"} + // A5/AC1 (design 47 H §750): no want waits for a provision, so the provisioner's provisions are no read; the grants stay + // for a put's own judgement (ADR 0546 clause 4). + want := []string{"mesh-grants.grants", "mesh-host.composed", "node-engine.have"} if !reflect.DeepEqual(reads, want) { t.Errorf("reads = %v, want exactly %v", reads, want) } // the buckets the program names are the manifest's reads, `.` as `_` - named := []string{buckets.Grants, - buckets.ComposedRead, "mesh-provisioner_provisions", buckets.Have} + named := []string{buckets.Grants, buckets.ComposedRead, buckets.Have} slices.Sort(named) for i, r := range want { if got := strings.Replace(named[i], "_", ".", 1); got != r { @@ -70,7 +70,7 @@ func TestManifest(t *testing.T) { } if want := map[string]int{"asks": 5, "differences": 64, "puts": 20, "wants": 20, "assignments": 64, "placement": 64, "copies": 1, "proposals": 5, "candidates": 5, "pulses": 0, "findings": 5, "policies": 20, "seat-holders": 5, views.MarksState: 1, views.StatusState: 1, - "engine-reads": 1, "written": 5, "waits": 5, "put-waits": 1, "inputs": 1, "composition-runs": 1, "pins": 5}; !reflect.DeepEqual(states, want) { + "engine-reads": 1, "written": 5, "waits": 5, "put-waits": 1, "inputs": 1, "composition-runs": 1}; !reflect.DeepEqual(states, want) { t.Errorf("state = %v, want %v", states, want) } if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 { @@ -105,21 +105,18 @@ func TestManifest(t *testing.T) { // which lists and asks again what each node's engine reads. if !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts", "assigner_waits", "assigner_inputs", "assigner_inputs_drop", "assigner_compositions", "assigner_compositions_drop", "assignments", "seat-holders", "wants", "asks", "assigner_read_assignments", "assigner_read_seat_holders", "assigner_read_wants", "assigner_read_findings", "assigner_read_differences", "assigner_read_copies", - "assigner_first_views", "assigner_first_view_again", "assigner_engine_reads", "assigner_seat_list", "assigner_seat_list_again", "assigner_ask_again", "assigner_written", "assigner_why", "assigner_pins", "assigner_unpin"}) { + "assigner_first_views", "assigner_first_view_again", "assigner_engine_reads", "assigner_seat_list", "assigner_seat_list_again", "assigner_ask_again", "assigner_written", "assigner_why"}) { t.Errorf("the assigner serves tools %v, want its three put tools, three first views, six read tools, assigner_engine_reads, the seat list's two (ADR 0636 §7) assigner_ask_again (ADR 0668 §5) and assigner_why (design 47 G A-439)", m.Tools) } if m.RunsAs != m.Module || m.SecretsOwner != m.RunsAs { t.Errorf("runs-as = %q: the bundle runs on the assigner's own account (ADR 0621 §1)", m.RunsAs) } - // ADR 0621 §2: each first-view tool answers exactly its readers, never the assigner. - // Design 47 G §717: the teller and the status view take their first views of the seat mesh-wants' asks and wants, and of - // the assignments, from the tools named for the states. - callers := map[string]string{"assignments": "mesh-grants,node-tools,mesh-delivery-indexer,mesh-delivery-builder,mesh-delivery-checker,mesh-delivery-view", - "seat-holders": "node-tools,mesh-delivery-builder,mesh-delivery-view,mesh-delivery-teller,mesh-delivery-checker,mesh-delivery-indexer", "wants": "mesh-grants,mesh-delivery-view", "asks": "mesh-delivery-teller,mesh-delivery-view"} - env := m.Build.Artifacts[0].Env - for tool, list := range callers { - if env[views.CallersEnv(tool)] != list || views.ParseCallers(env[views.CallersEnv(tool)]).Allows("n1."+m.Module) { - t.Errorf("%s callers = %q, want %q", tool, env[views.CallersEnv(tool)], list) + // design 47 H §752, H-A-66: every first-view tool answers every caller, so the manifest names no callers list for any. + for _, a := range m.Build.Artifacts { + for k := range a.Env { + if strings.HasPrefix(k, "MESH_VIEW_CALLERS_") { + t.Errorf("the artifact %s names the callers list %s: no seat or tool keeps one (design 47 H §752)", a.Name, k) + } } } // ADR 0635 §5: its views family is its ordinary events, `views.` among its emits, on no stream of its own. @@ -138,7 +135,7 @@ func TestManifest(t *testing.T) { // issue 1966 note 151: and the teller's ask-ended, its subject the one the assigner hears; §389: the watchman's machine-away // and machine-back, read whole at its start. A1/AC1 (design 47 H §750, §764): no version-delivered, delivery-waits, // delivered-again or assigner_delivered_again: a want waits on no dependency's delivery. - if !slices.Equal(m.Consumes, []string{"mesh-conditions.findings-asked", "mesh-conditions.condition-raised", "mesh-conditions.condition-changed", "mesh-conditions.condition-cleared", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-delivery.source-moved", "mesh-catalog.version-released", "mesh-watchdogs.machine-away", "mesh-watchdogs.machine-back", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", builds.BuilderModule + "." + builds.BuildsStreamEvents, buckets.TellerModule + "." + tellerevents.AskEnded, buckets.TellerModule + ".views." + buckets.AnswersState, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-build-go.step-ended", "node-build-go.ask-taken", "node-build-go.step-progress", "node-engine.manifest-reads-changed", "node-engine.running", "mesh-delivery.pinned", "mesh-delivery.freed", "mesh-delivery.heads-replaced", "mesh-delivery.pins-stated"}) { + if !slices.Equal(m.Consumes, []string{"mesh-conditions.findings-asked", "mesh-conditions.condition-raised", "mesh-conditions.condition-changed", "mesh-conditions.condition-cleared", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-delivery.source-moved", "mesh-catalog.version-released", "mesh-watchdogs.machine-away", "mesh-watchdogs.machine-back", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", builds.BuilderModule + "." + builds.BuildsStreamEvents, buckets.TellerModule + "." + tellerevents.AskEnded, buckets.TellerModule + ".views." + buckets.AnswersState, "mesh-pipeline-runner.pipeline-ended", "node-build-go.step-ended", "node-build-go.ask-taken", "node-build-go.step-progress", "node-engine.manifest-reads-changed", "node-engine.running"}) { t.Errorf("consumes = %v, want the catalog half's revisions (design 47 G §195), the teller's ask-ended, the engines' reads changed and what they run", m.Consumes) } if want := []string{"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index c320bb89..57cc0156 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -34,7 +34,6 @@ "reads": [ "mesh-grants.grants", "mesh-host.composed", - "mesh-provisioner.provisions", "node-engine.have" ], "uses": [ @@ -57,16 +56,11 @@ "mesh-delivery-teller.ask-ended", "mesh-delivery-teller.views.answers", "mesh-pipeline-runner.pipeline-ended", - "mesh-pipeline-runner.group-ended", "node-build-go.step-ended", "node-build-go.ask-taken", "node-build-go.step-progress", "node-engine.manifest-reads-changed", - "node-engine.running", - "mesh-delivery.pinned", - "mesh-delivery.freed", - "mesh-delivery.heads-replaced", - "mesh-delivery.pins-stated" + "node-engine.running" ], "invokes": [ "mesh-delivery-indexer.manifests", @@ -135,9 +129,7 @@ "assigner_seat_list_again", "assigner_ask_again", "assigner_written", - "assigner_why", - "assigner_pins", - "assigner_unpin" + "assigner_why" ], "state": [ { "name": "assignments", "history": 64 }, @@ -160,8 +152,7 @@ { "name": "waits", "history": 5 }, { "name": "put-waits", "history": 1 }, { "name": "inputs", "history": 1, "max-mib": 64 }, - { "name": "composition-runs", "history": 1, "key-ttl": true, "max-mib": 64 }, - { "name": "pins", "history": 5, "max-mib": 4 } + { "name": "composition-runs", "history": 1, "key-ttl": true, "max-mib": 64 } ], "config": { "outside-wait-minutes": { @@ -344,13 +335,7 @@ "binary": "mesh-delivery-assigner", "loads": [ "mesh-delivery-assigner" - ], - "env": { - "MESH_VIEW_CALLERS_ASKS": "mesh-delivery-teller,mesh-delivery-view", - "MESH_VIEW_CALLERS_ASSIGNMENTS": "mesh-grants,node-tools,mesh-delivery-indexer,mesh-delivery-builder,mesh-delivery-checker,mesh-delivery-view", - "MESH_VIEW_CALLERS_SEAT_HOLDERS": "node-tools,mesh-delivery-builder,mesh-delivery-view,mesh-delivery-teller,mesh-delivery-checker,mesh-delivery-indexer", - "MESH_VIEW_CALLERS_WANTS": "mesh-grants,mesh-delivery-view" - } + ] } ] }