diff --git a/.gitignore b/.gitignore index 48b8bf9..420d8ac 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ vendor/ +# A compiled program never enters a commit (issue 1577): `go build .` in a command directory leaves one there. +modules/*/cmd/*/mesh-* +!modules/*/cmd/*/mesh-*.go diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go index eac7f57..d6d3157 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go @@ -172,6 +172,9 @@ func TestTheAssignerStartsAndSaysWhatItKeeps(t *testing.T) { // 1297/B2b: a bucket it reads that the bus does not hold stops the start, naming the bucket. func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) { for _, missing := range readBuckets { + if missing == delivery.PlacementBucket { + continue // it starts without it, below + } b := startBed(t, missing) select { case err := <-b.start(t): @@ -218,6 +221,34 @@ func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) { t.Fatal("the put was not taken further once its bucket was made") } } + + // Review of L2 (D1): without the indexer's placement state it starts all the same, and once the bucket is made (the + // indexer and the assigner come in one delivery) it is read with no restart: a module only the placement names is + // said as placed by one source only. + pb := startBed(t, delivery.PlacementBucket) + pdone := pb.start(t) + select { + case err := <-pdone: + t.Fatalf("without %s the assigner stopped: %v", delivery.PlacementBucket, err) + case <-time.After(2 * time.Second): + } + placement, err := pb.js.CreateKeyValue(pb.ctx, jetstream.KeyValueConfig{Bucket: delivery.PlacementBucket, History: 5}) + if err != nil { + t.Fatal(err) + } + pl, _ := delivery.EncodePlacement(delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "web"}}}) + plkey, _ := delivery.PlacementKey("n1") + if _, err := placement.Put(pb.ctx, plkey, pl); err != nil { + t.Fatal(err) + } + for deadline := time.Now().Add(15 * time.Second); ; time.Sleep(200 * time.Millisecond) { + if c, ok := pb.conditionsSaid(t)[delivery.AssignerModule+".placement.n1.web"]; ok && c.State == asks.StateRaised { + break + } + if time.Now().After(deadline) { + t.Fatal("the placement state made after the start was not read") + } + } } // putJSON writes a value into one of the bed's buckets. @@ -406,6 +437,32 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) { if !cleared { t.Errorf("the condition of web on neither list on g14 was not cleared once g14 got its want") } + + // Issue 1126 (ADR 0547 §2): the catalogue's placement reaches the indexer's state naming web on every node the + // assignments name it on. Each node then composes its own: its want names the build alone (Self, no composition), and + // its own want says it composes its own, which the node agent reads (mesh-host's TestNats...ComposesAndAppliesItsOwn). + for _, node := range []string{"ace", "g14", "shanks"} { + pl, err := delivery.EncodePlacement(delivery.Placement{Node: node, Modules: []delivery.PlacedModule{{Module: "web"}}}) + if err != nil { + t.Fatal(err) + } + pkey, _ := delivery.PlacementKey(node) + b.putJSON(t, delivery.PlacementBucket, pkey, pl) + } + if w := b.wantNow(t, "ace", "web", "ace's want once placed: the build alone, composed by the node", func(w delivery.Want) bool { return w.Self }); w.Composed != "" || w.FP != fNew { + t.Errorf("ace's want once placed = %+v", w) + } + nkey, _ := delivery.WantNodeKey("ace") + var self bool + for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && !self; time.Sleep(100 * time.Millisecond) { + if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, nkey); err == nil { + nw, err := delivery.DecodeNodeWant(e.Value()) + self = err == nil && nw.Self + } + } + if !self { + t.Error("ace's own want does not say it composes its own once placed") + } } // 1297/B9 (the assigner's half, ADR 0528 clause 6): started whole, the assigner holds a fingerprint whose health check diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 3ba0133..a3d582b 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -59,6 +59,7 @@ var readBuckets = []string{ delivery.CandidatesBucket, delivery.ManifestsBucket, delivery.ModulesBucket, + delivery.PlacementBucket, delivery.PresenceBucket, delivery.HaveBucket, delivery.StepsBucket(goBuildModule), @@ -98,6 +99,12 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any logf("the bucket %s cannot be opened (%v): no put is taken further until the assigner starts again", name, err) continue } + if name == delivery.PlacementBucket && err != nil { + // The indexer's placement state comes with the second milestone (ADR 0549 §2): without it the controller's + // assignments alone name where modules run, as before any placement was read (ADR 0549 §5). + logf("the bucket %s cannot be opened (%v): the controller's assignments alone are read until the assigner starts again", name, err) + continue + } if err != nil { return nil, process.EssentialErr("assigner", name, "write a want from the whole mesh", err) } @@ -151,7 +158,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro logger.Printf("what the assigner keeps could not be said on the bus, so its readers are not granted until it is: %v", err) } logger.Printf("started on the account %s, reading %d buckets", account.User, len(kvs)) - return follow(ctx, js, kvs, sayVia(ctx, js), emitting{said}, logger) + return follow(ctx, js, conn, kvs, sayVia(ctx, js), emitting{said}, logger) } // emitting is the process connection as the puts' caller: its calls, and its events with no options. @@ -169,6 +176,7 @@ var watched = []string{ delivery.BuilderBuildsBucket, delivery.CandidatesBucket, delivery.ModulesBucket, + delivery.PlacementBucket, delivery.PresenceBucket, delivery.HaveBucket, delivery.AssignerAsksBucket, @@ -179,7 +187,7 @@ var watched = []string{ // follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after // each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends. -func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(asks.Condition) error, bus caller, logger *log.Logger) error { +func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, kvs buckets, say func(asks.Condition) error, bus caller, logger *log.Logger) error { w := newWorld() dirty := make(chan struct{}, 1) wake := func() { @@ -263,7 +271,12 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a } logger.Printf("the bucket %s is open: puts are taken further", delivery.AssignerPutsBucket) } + started := 0 for _, name := range watched { + if kvs[name] == nil && name == delivery.PlacementBucket { + continue // not on the bus at this start (open said so) + } + started++ var opts []jetstream.WatchOpt if name == delivery.HaveBucket { opts = append(opts, jetstream.IncludeHistory()) // what each node ran healthy before, for a set-back @@ -275,7 +288,20 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a if err := watch(delivery.ManifestsBucket, jetstream.MetaOnly()); err != nil { return err } - for range len(watched) + 1 { + // The indexer's placement state arrives with the same delivery as this assigner (ADR 0549 §2): when it is not on the + // bus at this start it is bound once the bus says it was created, and watched from then on, with no restart (issue + // 1790's kind). Until then the controller's assignments alone name where modules run (ADR 0549 §5). + placementAppeared := make(chan jetstream.KeyValue, 1) + if kvs[delivery.PlacementBucket] == nil && js != nil { + process.AwaitBucket(ctx, js, nc, delivery.PlacementBucket, "the controller's assignments alone are read", + logger.Printf).Ready(func(kv jetstream.KeyValue) { placementAppeared <- kv }) + } + takePlacement := func(kv jetstream.KeyValue) error { + kvs[delivery.PlacementBucket] = kv + logger.Printf("the bucket %s is open: placement is read beside the controller's assignments", delivery.PlacementBucket) + return watch(delivery.PlacementBucket) + } + for range started + 1 { select { case <-ctx.Done(): return nil @@ -297,6 +323,10 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a case err := <-errs: return err case <-settled: + case kv := <-placementAppeared: + if err := takePlacement(kv); err != nil { + return err + } case <-dirty: reopenPuts(p) p.pass(ctx) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go index 75d7411..dc9bc87 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go @@ -94,7 +94,7 @@ func TestOpensWhatItsManifestNames(t *testing.T) { kvs[delivery.ControllerComposedBucket].(*watchKV).entries = []kvEntry{{key: ckey, value: cd}} kvs[delivery.AssignerPutsBucket].(*watchKV).closed = true kvs[delivery.AssignerWantsBucket].(*watchKV).wrote = wrote - go follow(ctx, nil, kvs, func(asks.Condition) error { return nil }, nil, log.New(io.Discard, "", 0)) + go follow(ctx, nil, nil, kvs, func(asks.Condition) error { return nil }, nil, log.New(io.Discard, "", 0)) select { case key := <-wrote: if key != "want.ace.node" { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index 45c3e77..7d25c1b 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -3,6 +3,7 @@ package main import ( "context" "errors" + "maps" "slices" "strings" "time" @@ -39,6 +40,9 @@ type passer struct { // noWant is, by node and module, the reason last said why the node gets no want for the module (issue 1945): a reason is // said in the journal when it appears or changes, and its clearing when the node has a want again; one line per change. noWant map[string]string + // nodeFailed is, by node and module, the words of the condition last raised for a failure that is the node's own (ADR 0557 + // §1, issue 1496): raised once per words, cleared once when it ends. + nodeFailed map[string]string } func (p *passer) clock() time.Time { @@ -60,7 +64,23 @@ func (p *passer) pass(ctx context.Context) { // controller has composed it and again whenever that digest changes (ADR 0539 §6). func (p *passer) nodeWants(ctx context.Context) { written := p.w.read(delivery.AssignerWantsBucket) + placed := p.w.placedNodes() + // A node the catalogue places composes its own composition (ADR 0547 §2): its own want names none. + for node := range placed { + key, err := delivery.WantNodeKey(node) + if err != nil { + continue + } + if before, err := delivery.DecodeNodeWant(written[key]); err == nil && before.Self { + continue + } + want := delivery.NodeWant{Self: true, Words: "the node composes its own composition (ADR 0547 §2)"} + p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeNodeWant(want) }) + } for node, d := range p.w.nodeComposed() { + if placed[node] { + continue + } key, err := delivery.WantNodeKey(node) if err != nil { continue @@ -97,6 +117,8 @@ func (p *passer) wants(ctx context.Context) { } } wanted := p.w.wantsOf() + frozen := p.sayPlacement(p.w.placementDiffers()) + placed := p.w.placedNodes() puts := p.putsByModule() p.staleAsks(ctx, trunk) names := map[string]bool{} @@ -119,7 +141,14 @@ func (p *passer) wants(ctx context.Context) { fp := trunk[name].FP2 name := name m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name], - Composed: func(node, fp string) (delivery.ComposedDigest, bool) { return p.w.composed(node, name, fp) }, + Composed: func(node, fp string) (delivery.ComposedDigest, bool) { + if placed[node] { + // The node composes its own (ADR 0547 §2, §4): no composition of the controller's is waited for, the want + // names the build alone, and the node judges the move itself. + return delivery.ComposedDigest{Judgement: delivery.JudgedNeither, Words: "the node composes its own"}, true + } + return p.w.composed(node, name, fp) + }, GrantedAt: func(fp string) bool { key, err := delivery.ControllerGrantKey(name, fp) return err == nil && p.w.has(delivery.ControllerGrantsBucket, key) @@ -153,9 +182,11 @@ func (p *passer) wants(ctx context.Context) { last[node] = p.w.lastHealthy(node, name, fp) } out := assign.Wants(m, nodes, wanted[name], last) + p.yeses(ctx, name, m, placed, wanted[name], &out) gatherNeither(neither, name, out) leftNeither(left, m, wanted[name], out) p.sayNoWant(name, m.Nodes, wanted[name], out) + p.sayNodeFailed(name, fp, m.Nodes, nodes, out, first) if first { startClears(stale, m, out) if fp != "" && !m.Held && out.Held == nil && !out.Lifted { @@ -183,6 +214,9 @@ func (p *passer) wants(ctx context.Context) { p.recordAsk(ctx, name, fp, need) } for node, want := range out.Wants { + if frozen[node+"."+name] { + continue // one source alone names it (ADR 0549 §5): no want, and the one written before stays + } key, err := delivery.WantKey(node, name) if err != nil { p.logf("no want of %s on %s is written: %v", name, node, err) @@ -230,6 +264,60 @@ func (p *passer) sayNoWant(module string, nodes []string, before map[string]deli } } +// sayNodeFailed raises, once per node and words, the condition of a failed health check of a module's newest fingerprint fp +// that is the node's own, since fp runs healthy elsewhere and is not held (ADR 0557 §1, issue 1496), and says it in the journal; +// it clears the condition once when the node no longer fails alone, or is no longer one of the module's nodes. The first pass +// after a start remembers nothing a previous process raised: it clears, for every module, the condition of every node the +// presence bucket knows that does not fail alone, since a node that recovered while no process watched leaves no failed have to +// find it by (a clear of a condition not open is quiet at the router). A condition the router did not take is said again at the next pass. +func (p *passer) sayNodeFailed(module, fp string, nodes []string, known assign.Nodes, out assign.Outcome, first bool) { + if p.nodeFailed == nil { + p.nodeFailed = map[string]string{} + } + look := map[string]bool{} + for _, node := range nodes { + look[node] = true + } + for key := range p.nodeFailed { + if node, m, _ := strings.Cut(key, "."); m == module { + look[node] = true + } + } + if first { + for node := range known.Online { + look[node] = true + } + } + for _, node := range slices.Sorted(maps.Keys(look)) { + key := node + "." + module + f, failing := out.NodeFailed[node] + last, said := p.nodeFailed[key] + switch { + case failing && f.Words+"|"+strings.Join(f.Others, ",") != last: + p.logf("%s on %s: the health check at %s failed here (%s), while it runs healthy on %s: the failure is %s's, so nothing "+ + "is held or set back", module, node, fp, f.Words, strings.Join(f.Others, ", "), node) + if p.say != nil { + if err := p.say(assign.NodeFailedCondition(node, module, fp, f, true)); err != nil { + p.logf("the condition of %s failing on %s alone could not be said, and is said at the next pass: %v", module, node, err) + continue + } + } + p.nodeFailed[key] = f.Words + "|" + strings.Join(f.Others, ",") + case !failing && (said || first): + if said { + p.logf("%s on %s no longer fails its health check alone", module, node) + } + if p.say != nil { + if err := p.say(assign.NodeFailedCondition(node, module, fp, assign.NodeFailure{}, false)); err != nil && said { + p.logf("the condition of %s failing on %s alone could not be cleared, and is cleared at the next pass: %v", module, node, err) + continue + } + } + delete(p.nodeFailed, key) + } + } +} + // neitherOne is a module assigned to a node that holds no want for it after a pass, and the rule holding its first want back. type neitherOne struct { node, module string @@ -728,3 +816,83 @@ func (p *passer) backupAsk(ctx context.Context, key, node, module string, want d } p.said[c.Key] = true } + +// sayPlacement raises, once, the condition of every pair the controller's assignments and the catalogue's placement do not +// both name, clears it once they agree, and answers those pairs by node and module (ADR 0549 §5): the pass writes no want +// for them and keeps the one written before. A say that fails is made again at the next pass. +func (p *passer) sayPlacement(differ []assign.Pair) map[string]bool { + frozen := map[string]bool{} + now := map[string]bool{} + for _, d := range differ { + frozen[d.Node+"."+d.Module] = true + c := assign.PlacementCondition(d, true) + now[c.Key] = true + if p.said[c.Key+".raised"] { + continue + } + p.logf("%s on %s is named by %s alone: no want is written for it there until both sources name it (ADR 0549 §5)", d.Module, d.Node, d.Only) + if err := p.say(c); err != nil { + p.logf("the placement condition of %s on %s could not be said, and is said at the next pass: %v", d.Module, d.Node, err) + continue + } + p.said[c.Key+".raised"] = true + } + prefix := delivery.AssignerModule + ".placement." + for key := range p.said { + c, raised := strings.CutSuffix(key, ".raised") + if !raised || !strings.HasPrefix(c, prefix) || now[c] { + continue + } + node, module, _ := strings.Cut(strings.TrimPrefix(c, prefix), ".") + if err := p.say(assign.PlacementCondition(assign.Pair{Node: node, Module: module}, false)); err != nil { + p.logf("the placement condition of %s on %s could not be cleared, and is cleared at the next pass: %v", module, node, err) + continue + } + delete(p.said, key) + } + return frozen +} + +// yeses writes a placed node's wants naming the build alone, and sees to the moves a node that composes its own judged a +// person must allow (ADR 0547 §3): for a have of phase +// waiting-yes it raises the record ask once, naming the composition waiting and the node's judgement, and on a yes writes +// the want again naming it; a yes once written is kept while the want stays at its build, so the node is not asked again. +func (p *passer) yeses(ctx context.Context, module string, m assign.Module, placed map[string]bool, before map[string]delivery.Want, out *assign.Outcome) { + for _, node := range m.Nodes { + if !placed[node] { + continue + } + was, had := before[node] + w, written := out.Wants[node] + if !written { + if !had || was.Removed { + continue + } + w = was + } + if w.Removed { + continue + } + yes := "" + if had && was.FP == w.FP { + yes = was.Yes + } + key, err := delivery.HaveKey(node, module) + if err != nil { + continue + } + if h, err := delivery.DecodeHave(p.w.value(delivery.HaveBucket, key)); err == nil && h.Phase == delivery.PhaseWaitingYes && h.Waits != yes { + if m.Allowed != nil && m.Allowed(node, h.Waits) { + yes = h.Waits + } else { + p.recordAsk(ctx, module, w.FP, assign.RecordNeed{Node: node, Digest: h.Waits, Judgement: h.Judgement, Words: h.Detail}) + } + } + // A want carried over from before the node composed its own may still name the controller's composition; a placed + // node's want names the build alone (ADR 0547 §2). + if yes != w.Yes || w.Composed != "" || !w.Self { + w.Yes, w.Composed, w.Self = yes, "", true + out.Wants[node] = w + } + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index 7a89fef..1adc490 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -342,6 +342,9 @@ func putOfTheBus(t *testing.T) { // again after a say that failed, and clears it once when the module has a want again; a module on neither list is also written to // the journal, so it is loud with no operator channel. func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) { + // A pair only one source names is a condition of the same kind (issue 1126): folded here to hold the count. + aPairOneSourceNamesAloneGetsNoWantAndOneCondition(t) + t.Run("gathers", testThePassGathersTheModulesOnNeitherList) p := newPasser(buckets{}, time.Date(2026, 10, 25, 15, 0, 0, 0, time.UTC)) var said []asks.Condition failing := true @@ -391,7 +394,7 @@ func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) { } // 1945/AC21: the pass gathers the modules on neither list of every module's outcome, keyed by node and module. -func TestThePassGathersTheModulesOnNeitherList(t *testing.T) { +func testThePassGathersTheModulesOnNeitherList(t *testing.T) { got := map[string]neitherOne{} gatherNeither(got, "screen-lock", assign.Outcome{Neither: map[string]assign.Holding{"g14": {Rule: delivery.RuleGrant, Words: "w"}}}) gatherNeither(got, "rofi", assign.Outcome{}) @@ -443,12 +446,71 @@ func TestThePassGathersTheModulesOnNeitherList(t *testing.T) { t.Errorf("pass %d said %q, want %q", i, got, c.line) } } + // 1496, ADR 0557 §1: a failure that is the node's own (its fingerprint healthy elsewhere, not held) raises the node's condition + // once, naming the node, the module, F and the have's words, and clears it once when it ends. + var conds []asks.Condition + p.say = func(c asks.Condition) error { conds = append(conds, c); return nil } + failing := map[string]assign.NodeFailure{"shanks": {Others: []string{"ace"}, Words: "health failed, phase running: w"}} + for i, c := range []struct { + failed map[string]assign.NodeFailure + state string // the condition said in this pass, "" for none + }{ + {failing, asks.StateRaised}, + {failing, ""}, + {nil, asks.StateCleared}, + {nil, ""}, + } { + conds = nil + p.sayNodeFailed("rofi", fpTest, []string{"shanks"}, assign.Nodes{}, assign.Outcome{NodeFailed: c.failed}, false) + switch { + case c.state == "" && len(conds) != 0, c.state != "" && (len(conds) != 1 || conds[0].State != c.state): + t.Errorf("node failed, pass %d said %+v, want %q", i, conds, c.state) + case c.state == asks.StateRaised && (conds[0].Key != "mesh-delivery-assigner.failed-alone.shanks.rofi" || + !strings.Contains(conds[0].Explanation, "failed on shanks (health failed, phase running: w), while it runs healthy on ace")): + t.Errorf("node failed, raised %+v", conds[0]) + } + } + // D2: a node that leaves the module's nodes while it failed alone is cleared once; the first pass after a start clears every + // node the presence bucket knows, for every module, failed have or not (a node that recovered unwatched leaves none). + conds = nil + p.sayNodeFailed("rofi", fpTest, []string{"shanks"}, assign.Nodes{}, assign.Outcome{NodeFailed: failing}, false) + p.sayNodeFailed("rofi", fpTest, nil, assign.Nodes{}, assign.Outcome{}, false) + p.sayNodeFailed("rofi", fpTest, nil, assign.Nodes{}, assign.Outcome{}, false) + if len(conds) != 2 || conds[1].State != asks.StateCleared || conds[1].Key != "mesh-delivery-assigner.failed-alone.shanks.rofi" { + t.Errorf("a node leaving the module said %+v, want one raise and one clear", conds) + } + known := assign.Nodes{Online: map[string]bool{"ace": true, "g14": false}, + Haves: map[string]map[string]delivery.Have{"g14": {"rofi": {FP: fpTest, Health: delivery.HealthFailed}}}} + for _, c := range []struct { + name string + known assign.Nodes + keys []string + }{ + {"a failed have", known, []string{"mesh-delivery-assigner.failed-alone.ace.rofi", "mesh-delivery-assigner.failed-alone.g14.rofi"}}, + {"no failed have", assign.Nodes{Online: known.Online}, []string{"mesh-delivery-assigner.failed-alone.ace.rofi", "mesh-delivery-assigner.failed-alone.g14.rofi"}}, + } { + conds = nil + fresh := newPasser(buckets{}, time.Date(2026, 10, 26, 22, 0, 0, 0, time.UTC)) + fresh.logf = func(string, ...any) {} + fresh.say = func(c asks.Condition) error { conds = append(conds, c); return nil } + fresh.sayNodeFailed("rofi", fpTest, nil, c.known, assign.Outcome{}, true) + var keys []string + for _, cond := range conds { + if cond.State == asks.StateCleared { + keys = append(keys, cond.Key) + } + } + if !slices.Equal(keys, c.keys) || len(conds) != len(c.keys) { + t.Errorf("first pass, %s: said %+v, want clears of %q", c.name, conds, c.keys) + } + } } // A neither or held condition a previous process raised is cleared by the first pass after a start: a fresh pass remembers // nothing it said, so it clears, once, the neither condition of every assigned module not on neither list and the held condition of // every module that is not held. Clearing a condition that is not open is quiet at the router. func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) { + t.Run("a held module", testAHeldModuleRaisesNoNeitherAndClearsAStandingOne) wantsKV := &memKV{values: map[string][]byte{}} p := newPasser(buckets{delivery.AssignerWantsBucket: wantsKV, delivery.AssignerAsksBucket: &memKV{values: map[string][]byte{}}}, time.Now()) var said []asks.Condition @@ -486,7 +548,7 @@ func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) { // 1945/AC24: given a module held at F under rule 2 on a node that runs it with no want, when the assigner passes, then it raises no // neither condition for it, a neither condition this process raised for it before is cleared, and so is one a previous process // raised (the first pass after a start), each once. -func TestAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) { +func testAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) { setup := func() (*passer, *[]asks.Condition) { p := newPasser(buckets{delivery.AssignerWantsBucket: &memKV{values: map[string][]byte{}}, delivery.AssignerAsksBucket: &memKV{values: map[string][]byte{}}}, time.Now()) said := &[]asks.Condition{} @@ -702,6 +764,8 @@ func mustKey(key string, err error) string { // (ADR 0545). That record ask is not about the trunk's fingerprint: it survives every pass, so the operator is asked, and on a // yes the want names the revised digest. func TestARecordAskOfAHeldBackNodeSurvivesPassesAndItsYesMovesTheWant(t *testing.T) { + // A placed node's yes comes from its have (issue 1126): folded here to hold the count. + aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t) fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64) revised, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64) asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}} @@ -750,3 +814,133 @@ func TestARecordAskOfAHeldBackNodeSurvivesPassesAndItsYesMovesTheWant(t *testing t.Error("the answered record ask was taken away") } } + +// 1126 (ADR 0549 §5): until the third milestone a pair the controller's assignments and the catalogue's placement do not +// both name gets no want and one condition, raised once and cleared once both name it; before any placement was read the +// assignments are the one source and nothing is raised. +func aPairOneSourceNamesAloneGetsNoWantAndOneCondition(t *testing.T) { + p := newPasser(buckets{}, time.Date(2026, 10, 26, 18, 0, 0, 0, time.UTC)) + var said []asks.Condition + p.say = func(c asks.Condition) error { said = append(said, c); return nil } + assigned := func(node, module string) { + raw, _ := json.Marshal(map[string]string{"node": node, "module": module}) + p.w.put(provider.AssignmentsBucket, node+"."+module, raw) + } + place := func(node string, modules ...string) { + pl := delivery.Placement{Node: node} + for _, m := range modules { + pl.Modules = append(pl.Modules, delivery.PlacedModule{Module: m}) + } + raw, err := delivery.EncodePlacement(pl) + if err != nil { + t.Fatal(err) + } + key, _ := delivery.PlacementKey(node) + p.w.put(delivery.PlacementBucket, key, raw) + } + assigned("g14", "a") + assigned("g14", "b") + if got, _ := p.w.assigned(); !slices.Equal(got["g14"], []string{"a", "b"}) || len(p.sayPlacement(p.w.placementDiffers())) != 0 || len(said) != 0 { + t.Fatalf("before any placement: assigned %v, said %v", got, said) + } + none := func(context.Context, string) ([]byte, error) { return nil, errNotThere } + if m := p.w.mesh(context.Background(), none); m.Input != nil || m.InputUnread != "" { + t.Errorf("before any placement the check composes over %+v (%q)", m.Input, m.InputUnread) + } + place("g14", "a", "c") + got, _ := p.w.assigned() + frozen := p.sayPlacement(p.w.placementDiffers()) + p.sayPlacement(p.w.placementDiffers()) + if !slices.Equal(got["g14"], []string{"a"}) || !frozen["g14.b"] || !frozen["g14.c"] || frozen["g14.a"] || len(said) != 2 || + said[0].State != asks.StateRaised || !strings.Contains(said[0].Explanation, "the controller's assignments names b on g14") || + !strings.Contains(said[1].Explanation, "the catalogue's placement names c on g14") { + t.Fatalf("one source alone: assigned %v, frozen %v, said %+v", got, frozen, said) + } + place("g14", "a", "b", "c") + assigned("g14", "c") + if frozen := p.sayPlacement(p.w.placementDiffers()); len(frozen) != 0 || len(said) != 4 || said[2].State != asks.StateCleared || said[3].State != asks.StateCleared { + t.Errorf("both agree again: frozen %v, said %+v", frozen, said) + } + + // 1126 (ADR 0547 §2): placed, the check composes over the input the nodes compose from (frombus), once each placed node + // wrote its record; until then it says why and composes the assignments alone. + if m := p.w.mesh(context.Background(), none); m.Input != nil || !strings.Contains(m.InputUnread, "g14") { + t.Errorf("a placed node with no record: input %+v, unread %q", m.Input, m.InputUnread) + } + rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "g14", TunnelKey: "k"}}) + p.w.put(delivery.HaveBucket, "g14.node", rec) + m := p.w.mesh(context.Background(), none) + if m.Input == nil || len(m.Input.Nodes) != 1 || !slices.Equal(m.Input.Nodes[0].Assigned, []string{"a", "b", "c"}) || m.Input.Nodes[0].Place.Key != "k" { + t.Errorf("the check's input from the bus: %+v (%q)", m.Input, m.InputUnread) + } +} + +// 1126 (ADR 0547 §2 to §4): a node the catalogue places composes its own: its wants name the build alone and its own want +// names no composition; a have waiting for a yes raises the record ask once, and the yes is written into the want and +// kept there while the want stays at its build. +func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) { + fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64) + waiting, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64) + asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}} + p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now()) + p.logf = t.Logf + put := func(bucket, key string, v []byte) { p.w.put(bucket, key, v) } + mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath}) + put(delivery.ModulesBucket, "web", mod) // the trunk's build is not done: rule 1 holds g14 at fpOld + put(provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-1"}`)) + pl, _ := delivery.EncodePlacement(delivery.Placement{Node: "g14", Modules: []delivery.PlacedModule{{Module: "web"}}}) + put(delivery.PlacementBucket, "placement.g14", pl) + on, _ := delivery.EncodePresence(delivery.Presence{Online: true}) + put(delivery.PresenceBucket, "g14", on) + have, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: was, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}) + put(delivery.HaveBucket, "g14.web", have) + wkey, _ := delivery.WantKey("g14", "web") + want, _ := delivery.EncodeWant(delivery.Want{FP: fpOld, Composed: was}) + wantsKV.values[wkey] = want + put(delivery.AssignerWantsBucket, wkey, want) + sync := func() { + for k, v := range wantsKV.values { + put(delivery.AssignerWantsBucket, k, v) + } + } + + p.wants(context.Background()) + p.nodeWants(context.Background()) + sync() + got, err := delivery.DecodeWant(wantsKV.values[wkey]) + if err != nil || got.FP != fpOld || got.Composed != "" || !got.Self { + t.Fatalf("the placed node's want is %+v (%v), want it at %s, the node composing it", got, err, fpOld) + } + nkey, _ := delivery.WantNodeKey("g14") + if nw, err := delivery.DecodeNodeWant(wantsKV.values[nkey]); err != nil || nw.Composed != "" || !nw.Self { + t.Fatalf("the placed node's own want is %+v (%v), want one naming no composition", nw, err) + } + + waits, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: was, Phase: delivery.PhaseWaitingYes, Health: delivery.HealthOK, + Judgement: delivery.JudgedRecreates, Waits: waiting, Detail: "it recreates web.server"}) + put(delivery.HaveBucket, "g14.web", waits) + akey, _ := delivery.RecordAskKey("g14", "web", waiting) + p.wants(context.Background()) + if _, there := asksKV.values[akey]; !there { + t.Fatal("a have waiting for a yes raised no record ask") + } + a, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey]) + if err != nil { + t.Fatal(err) + } + put(delivery.AssignerAsksBucket, akey, asksKV.values[akey]) + yes, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a.Ask.ID + "@2026-10-26T19:00:00Z", Words: "the operator"}) + put(delivery.AnswersBucket, akey, yes) + p.wants(context.Background()) + sync() + if got, err = delivery.DecodeWant(wantsKV.values[wkey]); err != nil || got.Yes != waiting || got.FP != fpOld { + t.Fatalf("after the yes the want is %+v (%v), want it to name the yes %s", got, err, waiting) + } + // The node applied it: its have runs the composition; the yes stays while the want stays at its build. + ran, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: waiting, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}) + put(delivery.HaveBucket, "g14.web", ran) + p.wants(context.Background()) + if got, _ = delivery.DecodeWant(wantsKV.values[wkey]); got.Yes != waiting { + t.Errorf("a pass after the move dropped the yes: %+v", got) + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go index 7fbb11a..22fb1d4 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go @@ -14,6 +14,8 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose" + "git.novox.be/novox/mesh-sdk/go/compose/frombus" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/provider" @@ -162,8 +164,56 @@ func (w *world) manifest(ctx context.Context, get getter, digest string) parsed return p } -// assigned is each node's modules with an open assignment, read from the controller's assignments, in name order. +// assigned is each node's modules both the controller's assignments and the indexer's placement name, in name order +// (ADR 0549 §5; the assignments alone before any placement was read), with every open assignment by node and module. func (w *world) assigned() (map[string][]string, map[string]assignment) { + byAssignment, all := w.assignedByController() + placed, seeded := w.placed() + both, _ := assign.Placed(byAssignment, placed, seeded) + return both, all +} + +// placementDiffers is every pair the controller's assignments and the indexer's placement do not both name. +func (w *world) placementDiffers() []assign.Pair { + byAssignment, _ := w.assignedByController() + placed, seeded := w.placed() + _, differ := assign.Placed(byAssignment, placed, seeded) + return differ +} + +// placed is each node's modules in the indexer's placement (ADR 0549 §2), and whether any placement key was read: a key +// that does not read places nothing on its node and is said by the indexer's own read tool. +func (w *world) placed() (map[string][]string, bool) { + out := map[string][]string{} + values := w.read(delivery.PlacementBucket) + for key, value := range values { + node, err := delivery.ParsePlacementKey(key) + if err != nil { + continue + } + p, err := delivery.DecodePlacement(value) + if err != nil || p.Node != node { + continue + } + out[node] = p.Assigned() + } + return out, len(values) > 0 +} + +// placedNodes is every node the indexer's placement names (ADR 0549 §2): those compose their own modules (ADR 0547 §2), +// so their wants name the build alone and the seventh rule does not hold them. +func (w *world) placedNodes() map[string]bool { + placed, _ := w.placed() + out := make(map[string]bool, len(placed)) + for node := range placed { + out[node] = true + } + return out +} + +// assignedByController is each node's modules with an open assignment, read from the controller's assignments, in +// name order. +func (w *world) assignedByController() (map[string][]string, map[string]assignment) { out, all := map[string][]string{}, map[string]assignment{} for _, value := range w.read(provider.AssignmentsBucket) { var a assignment @@ -210,9 +260,86 @@ func (w *world) mesh(ctx context.Context, manifests getter) assign.Mesh { } m.Manifests[name] = p.manifest } + // Once the catalogue places the nodes, the check composes over the input they compose from (ADR 0547 §2, ADR 0549 + // §3): one composer's input for the check and the nodes. An input that cannot be read is said, and the nodes and their + // assignments alone are composed, as before any placement. + if _, seeded := w.placed(); seeded { + in, err := frombus.Input(ctx, worldReader{w: w, manifests: manifests}, frombus.Mesh{}, frombus.Scope{Every: true}) + if err != nil { + m.InputUnread = err.Error() + } else { + m.Input = &in + } + } return m } +// worldReader is what the assigner watched as frombus reads the bus: the indexer's placement and modules, each node's +// record in its own have key, the manifests by digest; the check reads no config value, own secret or credential (the +// composition it asks resolves, and stands in for what it does not carry). +type worldReader struct { + w *world + manifests getter +} + +func (r worldReader) Placements(context.Context) (map[string]delivery.Placement, error) { + out := map[string]delivery.Placement{} + for key, value := range r.w.read(delivery.PlacementBucket) { + node, err := delivery.ParsePlacementKey(key) + if err != nil { + continue + } + if p, err := delivery.DecodePlacement(value); err == nil && p.Node == node { + out[node] = p + } + } + return out, nil +} + +func (r worldReader) Records(context.Context) (map[string]delivery.NodeRecord, error) { + out := map[string]delivery.NodeRecord{} + for key, value := range r.w.read(delivery.HaveBucket) { + node, _, isNode, err := delivery.ParseHaveKey(key) + if err != nil || !isNode { + continue + } + if h, err := delivery.DecodeHaveNode(value); err == nil && h.Record != nil { + out[node] = *h.Record + } + } + return out, nil +} + +func (r worldReader) Modules(context.Context) (map[string]delivery.Module, error) { + return r.w.trunk(), nil +} + +func (r worldReader) Manifest(ctx context.Context, digest string) ([]byte, error) { + key, err := delivery.ManifestsKey(digest) + if err != nil { + return nil, err + } + raw, err := r.manifests(ctx, key) + if err != nil { + return nil, err + } + m, err := delivery.DecodeManifest(raw) + if err != nil { + return nil, err + } + return []byte(m.JSON), nil +} + +func (worldReader) Values(context.Context, string, string) (map[string]any, error) { return nil, nil } + +func (worldReader) OwnSecrets(context.Context, string) ([]compose.OwnSecretRow, error) { + return nil, nil +} + +func (worldReader) Credentials(context.Context, string, []compose.CredentialKey, []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error) { + return nil, nil, nil +} + // change is a candidate's change for the composition check, or why the composition waits for it. func (w *world) change(ctx context.Context, manifests getter, c delivery.Candidate, trunk map[string]delivery.Module) (assign.Change, string) { ch := assign.Change{Group: c.Group, Heads: c.HeadsHash, Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{}, diff --git a/modules/mesh-delivery-assigner/go.mod b/modules/mesh-delivery-assigner/go.mod index df8760f..9476f6d 100644 --- a/modules/mesh-delivery-assigner/go.mod +++ b/modules/mesh-delivery-assigner/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-assigner go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-assigner/go.sum b/modules/mesh-delivery-assigner/go.sum index 8492faa..e8660dd 100644 --- a/modules/mesh-delivery-assigner/go.sum +++ b/modules/mesh-delivery-assigner/go.sum @@ -1,13 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025195051-20a6dc4920f8 h1:jA7V7hlfJAigVUvJDpKwfgd9ylQ9bqppg3rK9XgLWoA= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025195051-20a6dc4920f8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8 h1:vhYk/iv43u2WGSTrFyQGDvGBxLaRITgMAiiI9DjuA10= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 h1:gE80bR0tPME7MTMBLcdP62v+F/9soVXmdcBYNMOye44= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117 h1:88CNZOoQgu7YPYKW16qkWT8Gw9Ucg07aTrIImA2nYUk= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-assigner/internal/assign/asks.go b/modules/mesh-delivery-assigner/internal/assign/asks.go index 1c7825a..b8125be 100644 --- a/modules/mesh-delivery-assigner/internal/assign/asks.go +++ b/modules/mesh-delivery-assigner/internal/assign/asks.go @@ -81,6 +81,29 @@ func HeldCondition(module, fp, node, words string, raised, needsYou bool) asks.C return c } +// NodeFailedCondition is the condition the assigner raises for a module whose newest fingerprint fp failed its health check on +// node while other nodes run it healthy (ADR 0557 §1, issue 1496): the failure is the node's, so nothing is held or set back and +// the node keeps its want. It names the node, the module, fp and the have's words, keyed by node and module, and is cleared +// (raised false) once the node no longer fails alone. +func NodeFailedCondition(node, module, fp string, f NodeFailure, raised bool) asks.Condition { + key := delivery.AssignerModule + ".failed-alone." + node + "." + module + subject := asks.Scope{Scope: "module", ID: module, Machine: node} + if !raised { + return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared, + Resolved: fmt.Sprintf("%s on %s no longer fails its health check alone.", module, node)} + } + headline := fmt.Sprintf("%s fails on %s alone", module, node) + if n := len([]rune(headline)); n > asks.HeadlineLength { + headline = string([]rune(headline)[:asks.HeadlineLength]) + } + return asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised, + Summary: headline, Headline: headline, + Explanation: fmt.Sprintf("%s The health check of %s at %s failed on %s (%s), while it runs healthy on %s. The failure is %s's, "+ + "so the assigner holds nothing and sets no node back, and %s keeps its want. This clears once %s runs it healthy or "+ + "another version is the newest.", asks.NothingToDo, module, short(fp), node, f.Words, strings.Join(f.Others, ", "), node, + node, node)} +} + // NeitherCondition is the condition of a module assigned to node that holds no want for it (issue 1945): the node's own composition // gives up every module with a fingerprint of the new form, so nothing applies it there and its files are gone. It is raised (urgent) // naming the module, the node and the rule holding its first want back with the rule's words, under the assigner's own name keyed by diff --git a/modules/mesh-delivery-assigner/internal/assign/compose.go b/modules/mesh-delivery-assigner/internal/assign/compose.go index b1f26e4..cb788b4 100644 --- a/modules/mesh-delivery-assigner/internal/assign/compose.go +++ b/modules/mesh-delivery-assigner/internal/assign/compose.go @@ -35,6 +35,13 @@ type Mesh struct { // Digests is every trunk module's manifest digest in the indexer's manifests bucket, readable or not: what a // composer at another mesh-sdk reads (issue 1622). Digests map[string]string + // Input is the rest of the mesh as the nodes compose from it, filled by mesh-sdk's frombus from the catalogue's + // placement and the nodes' records (novox/hq ADR 0547 §2, ADR 0549 §3, issue 1126), so the check and the nodes share + // one composer's input; nil before any placement is read, when the nodes and their assignments alone are composed. + Input *compose.Input + // InputUnread says why that input could not be read from the bus, when it could not: the check then composes the + // nodes and their assignments alone, and says so in its warnings. + InputUnread string } // Change is a group's change as the indexer's candidate holds it: the manifest of every module it moves or adds, those diff --git a/modules/mesh-delivery-assigner/internal/assign/composer.go b/modules/mesh-delivery-assigner/internal/assign/composer.go index b8ccb02..8c60ce8 100644 --- a/modules/mesh-delivery-assigner/internal/assign/composer.go +++ b/modules/mesh-delivery-assigner/internal/assign/composer.go @@ -42,8 +42,15 @@ func HeadJob(m Mesh, c Change) (delivery.ComposeJob, error) { for _, name := range c.Removed { delete(change, name) } - req := composer.Request{V: composer.Version, Input: plainInput(m.Assigned), Depth: composer.DepthResolve, - HoldingsFromClaims: true, Machines: Touched(m, c), + in, fromClaims := plainInput(m.Assigned), true + if m.Input != nil { + // The input the nodes compose from, without its catalogue: the cases carry the manifests (issue 1126). + in = *m.Input + in.Catalogue = nil + fromClaims = len(in.Holdings) == 0 + } + req := composer.Request{V: composer.Version, Input: in, Depth: composer.DepthResolve, + HoldingsFromClaims: fromClaims, Machines: Touched(m, c), Cases: []composer.Case{{Name: CaseTrunk, Manifests: trunk}, {Name: CaseChange, Manifests: change}}} return job(SDKRepository, c.SDKHead, SDKDir, req, "the mesh-sdk head of "+c.Group) } @@ -195,6 +202,10 @@ func FromComposer(m Mesh, c Change, head Ran, readers []Reader, took time.Durati readerFails, readerWarnings := judgeReaders(c, change, readers) fails = append(fails, readerFails...) v.Warnings = append(v.Warnings, readerWarnings...) + if m.InputUnread != "" { + v.Warnings = append(v.Warnings, "composed over the nodes and their assignments alone: the input the nodes compose from "+ + "cannot be read from the bus ("+m.InputUnread+")") + } return finish(v, m, c, machines, fails) } diff --git a/modules/mesh-delivery-assigner/internal/assign/composer_test.go b/modules/mesh-delivery-assigner/internal/assign/composer_test.go index 4b709c8..2bc3258 100644 --- a/modules/mesh-delivery-assigner/internal/assign/composer_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/composer_test.go @@ -8,6 +8,7 @@ import ( "time" "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose" "git.novox.be/novox/mesh-sdk/go/composer" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -100,6 +101,25 @@ func TestAGroupWithAnSDKMemberAsksTheComposerAtItsHead(t *testing.T) { if !slices.Equal(req.Machines, []string{"g14"}) || req.Depth != composer.DepthResolve || !req.HoldingsFromClaims { t.Errorf("request = machines %v, depth %q, holdings from claims %v", req.Machines, req.Depth, req.HoldingsFromClaims) } + + // 1126 (ADR 0547 §2, ADR 0549 §3): once the catalogue places the nodes, the check composes over the input the nodes + // compose from (frombus), its catalogue left to the cases, and the seats held as placement says. + m := digestedMesh() + m.Input = &compose.Input{Catalogue: map[string]catalogue.Manifest{"web": {Module: "web"}}, + Nodes: []compose.Node{{Name: "g14", Assigned: []string{"web"}, Place: compose.Place{Endpoint: "g14.example:51820"}}}, + Holdings: []catalogue.Held{{Claim: "web-seat", Scope: "mesh", Node: "g14", Module: "web"}}} + placedJob, err := HeadJob(m, c) + if err != nil { + t.Fatal(err) + } + placed, err := composer.Decode(placedJob.Request) + if err != nil { + t.Fatal(err) + } + if placed.Input.Catalogue != nil || len(placed.Input.Nodes) != 1 || placed.Input.Nodes[0].Place.Endpoint != "g14.example:51820" || + len(placed.Input.Holdings) != 1 || placed.HoldingsFromClaims { + t.Errorf("the placed check's request = %+v, holdings from claims %v", placed.Input, placed.HoldingsFromClaims) + } } // 1622/AC2, AC3: a catalogue member using a field only the SDK head reads passes with the head's composer, and the diff --git a/modules/mesh-delivery-assigner/internal/assign/placement.go b/modules/mesh-delivery-assigner/internal/assign/placement.go new file mode 100644 index 0000000..9856b12 --- /dev/null +++ b/modules/mesh-delivery-assigner/internal/assign/placement.go @@ -0,0 +1,99 @@ +package assign + +import ( + "fmt" + "sort" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Until ADR 0547's third milestone two sources name which module runs on which node: the controller's assignments and +// the indexer's placement, read from the catalogue (ADR 0549 §5). The assigner writes wants only for a pair both name: +// for a pair one source names alone it writes no want and touches none it wrote before, so nothing moves and nothing is +// removed on one source's word, and it raises a condition naming the pair until both agree. Before the first placement +// file reaches the indexer (no placement key at all) the controller's assignments are the one source, as before. + +// Pair is a module on a node that one source names and the other does not; Only says which: OnlyAssignments or +// OnlyPlacement. +type Pair struct { + Node, Module, Only string +} + +// The source that names a pair alone. +const ( + OnlyAssignments = "the controller's assignments" + OnlyPlacement = "the catalogue's placement" +) + +// Placed is, by node, the modules both sources name, in name order, and every pair one names alone, by node then +// module. With no placement read yet (seeded false) it is the assignments, and no pair differs. +func Placed(assigned, placed map[string][]string, seeded bool) (map[string][]string, []Pair) { + if !seeded { + return assigned, nil + } + set := func(m map[string][]string) map[string]bool { + out := map[string]bool{} + for node, modules := range m { + for _, module := range modules { + out[node+"\x00"+module] = true + } + } + return out + } + inAssigned, inPlaced := set(assigned), set(placed) + both := map[string][]string{} + var differ []Pair + for node, modules := range assigned { + for _, module := range modules { + if inPlaced[node+"\x00"+module] { + both[node] = append(both[node], module) + } else { + differ = append(differ, Pair{Node: node, Module: module, Only: OnlyAssignments}) + } + } + } + for node, modules := range placed { + for _, module := range modules { + if !inAssigned[node+"\x00"+module] { + differ = append(differ, Pair{Node: node, Module: module, Only: OnlyPlacement}) + } + } + } + for _, modules := range both { + sort.Strings(modules) + } + sort.Slice(differ, func(i, j int) bool { + if differ[i].Node != differ[j].Node { + return differ[i].Node < differ[j].Node + } + return differ[i].Module < differ[j].Module + }) + return both, differ +} + +// PlacementCondition is the condition of a pair the two sources do not both name: raised while they differ, cleared once +// both name it or neither does. +func PlacementCondition(p Pair, raised bool) asks.Condition { + key := delivery.AssignerModule + ".placement." + p.Node + "." + p.Module + subject := asks.Scope{Scope: "module", ID: p.Module, Machine: p.Node} + if !raised { + return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared, + Resolved: fmt.Sprintf("the controller's assignments and the catalogue's placement agree on %s on %s.", p.Module, p.Node)} + } + headline := fmt.Sprintf("%s on %s is placed by one source only", p.Module, p.Node) + if n := len([]rune(headline)); n > asks.HeadlineLength { + headline = string([]rune(headline)[:asks.HeadlineLength]) + } + var needs string + if p.Only == OnlyAssignments { + needs = fmt.Sprintf("add %s to the catalogue's placement file of %s, or unassign it at the controller", p.Module, p.Node) + } else { + needs = fmt.Sprintf("assign %s to %s at the controller, as the merged placement file says", p.Module, p.Node) + } + return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised, + Summary: headline, Headline: headline, Needs: needs, + Explanation: fmt.Sprintf("%s%s. %s names %s on %s and the other source does not. Until ADR 0547's third milestone both must "+ + "name a module on a node (ADR 0549 §5): the assigner writes no want for it there and leaves the one it wrote before, so "+ + "nothing moves or is removed. This clears once both name it or neither does.", asks.NeedsYou, needs, p.Only, p.Module, p.Node)} +} diff --git a/modules/mesh-delivery-assigner/internal/assign/wants.go b/modules/mesh-delivery-assigner/internal/assign/wants.go index 7bc6885..bd736c6 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants.go @@ -2,6 +2,7 @@ package assign import ( "fmt" + "maps" "slices" "sort" "strings" @@ -169,6 +170,9 @@ type Outcome struct { // Lifted is the hold of m.FP lifted by itself: every node that runs the module runs m.FP healthy (issue 1906). The held // record and its ask are taken away, and the wants move by the rules again in this pass. Lifted bool + // NodeFailed is, by node, a failed health check of m.FP on a node while another node runs m.FP healthy: the failure is + // that node's, so m.FP is not held and nothing is set back (ADR 0557 §1, issue 1496). The pass raises its condition once. + NodeFailed map[string]NodeFailure // Record is the moves a person must allow before the want names them: each is asked once (ADR 0539 §14). Record []RecordNeed // Restore is the nodes of a module holding data whose held fingerprint failed its health check there and that are not set @@ -228,11 +232,25 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st } if !held && !out.Lifted && !m.Released && m.FP != "" { for _, node := range slices.Sorted(slices.Values(m.Nodes)) { - if h, ok := n.Haves[node][m.Name]; ok && h.FP == m.FP && failed(h) { - out.Held = &delivery.Held{Node: node, Words: HeldWords(h, m.Now)} - held = true - break + h, ok := n.Haves[node][m.Name] + if !ok || h.FP != m.FP || !failed(h) { + continue } + if others := healthyElsewhere(m, n, node); len(others) > 0 && !failedElsewhere(m, n, node) { + // F runs healthy on another node, so its failure here is this node's, not F's: F is not held, no node is set back, + // and the failing node keeps its want (ADR 0557 §1; issue 1496: on ace a newly assigned mesh-build-node failed its + // first check and the hold moved shanks, g14 and novox, healthy on F for an hour, back). The pass raises the node's + // condition. With no online healthy witness, or with F failed on a second node, the hold and the set-back stand (ADR 0557 + // §2, ADR 0429 §6). + if out.NodeFailed == nil { + out.NodeFailed = map[string]NodeFailure{} + } + out.NodeFailed[node] = NodeFailure{Others: others, Words: HeldWords(h, time.Time{})} // said once: no clock in it + continue + } + out.Held = &delivery.Held{Node: node, Words: HeldWords(h, m.Now)} + held = true + break } } mm := m @@ -298,6 +316,13 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st ok, target = true, have.FP } } + if ok && target == m.FP && len(out.NodeFailed) > 0 && !(had && !before.Removed && before.FP == m.FP) && + n.Haves[node][m.Name].FP != m.FP { + // While a node fails F alone, F reaches no further node: a node that neither wants nor runs F gets no new want + // naming it, and wants already written stay (ADR 0557 §2). A node that runs F with no live want gets its want at F, or + // it would be on neither list and its node-engine would give the module up (issue 1945). + ok, rule, words = false, delivery.RuleNotHeld, FailedAloneWords(m.Name, m.FP, slices.Sorted(maps.Keys(out.NodeFailed))) + } switch { case ok: d, _ := m.composition(node, target) @@ -419,6 +444,43 @@ func failed(h delivery.Have) bool { return h.Health == delivery.HealthFailed && (h.Phase == delivery.PhaseRunning || h.Phase == delivery.PhaseFailed) } +// healthyElsewhere is every online node of m but node that runs m.FP healthy, by name. +func healthyElsewhere(m Module, n Nodes, node string) []string { + var out []string + for _, other := range slices.Sorted(slices.Values(m.Nodes)) { + // A witness is online: a node that sleeps keeps the have it last wrote, and a stale have is no evidence that F runs + // healthy now. + if other != node && n.Online[other] && healthy(n.Haves[other][m.Name], m.FP) { + out = append(out, other) + } + } + return out +} + +// failedElsewhere says whether another node of m has a have naming m.FP failed: then the failure is not one node's alone, and +// the hold and the set-back stand at the second failure (ADR 0557 §2). +func failedElsewhere(m Module, n Nodes, node string) bool { + for _, other := range m.Nodes { + if h, ok := n.Haves[other][m.Name]; ok && other != node && h.FP == m.FP && failed(h) { + return true + } + } + return false +} + +// FailedAloneWords say why a node gets no new want naming fp: fp fails its health check on the nodes named, alone. +func FailedAloneWords(module, fp string, failing []string) string { + return fmt.Sprintf("%s at %s fails its health check on %s, so no further node moves to it until that is over", module, short(fp), + strings.Join(failing, ", ")) +} + +// NodeFailure is a failed health check of a module's newest fingerprint on one node while it runs healthy on Others: the have's +// health, phase and words (HeldWords without a time), and the healthy witnesses by name. +type NodeFailure struct { + Others []string + Words string +} + // runs says whether a have is of a module the node runs: running, whatever its health. func runs(h delivery.Have) bool { return h.Phase == delivery.PhaseRunning diff --git a/modules/mesh-delivery-assigner/internal/assign/wants_test.go b/modules/mesh-delivery-assigner/internal/assign/wants_test.go index ea4ca59..ff629e2 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants_test.go @@ -150,6 +150,72 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) { if w, ok := out.Wants["g14"]; ok && w.FP == fNew { t.Errorf("g14 moved to a held fingerprint: %+v", w) } + + // 1496, ADR 0557 §1, §2: the same failure while F runs healthy on another node is that node's: F is not held, nothing is set + // back, the failing node keeps its want, and the outcome names it (on ace a newly assigned module's first check held a version healthy for an hour on three nodes). + n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK} + before := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fNew, Composed: dig(fNew)}} + out = Wants(m, n, before, map[string]string{"ace": fOld, "g14": fOld}) + if out.Held != nil || len(out.Wants) != 0 { + t.Fatalf("held %+v, wants %+v: want no hold and no want changed", out.Held, out.Wants) + } + if f := out.NodeFailed["ace"]; !slices.Equal(f.Others, []string{"g14"}) || f.Words != "health failed, phase running: connection refused" || + len(out.NodeFailed) != 1 { + t.Errorf("node failed = %+v", out.NodeFailed) + } + + // ADR 0557 §2 (version 9): while one node fails F alone, novox, whose want names fOld, gets no new want naming F; the wants + // at F stay. The canary ace runs F healthy here and g14 fails it, so the canary rule (3) would let novox move: only §2 holds it. + three := m + three.Nodes = []string{"ace", "g14", "novox"} + failedHave := n.Haves["ace"]["mail"] + n.Haves["ace"]["mail"], n.Haves["g14"]["mail"] = n.Haves["g14"]["mail"], failedHave + n.Online["novox"] = true + n.Haves["novox"] = map[string]delivery.Have{"mail": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}} + before["novox"] = delivery.Want{FP: fOld, Composed: dig(fOld)} + out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld}) + if _, alone := out.NodeFailed["g14"]; !alone { + t.Errorf("g14 does not fail alone: %+v", out.NodeFailed) + } + if w, written := out.Wants["novox"]; out.Held != nil || len(out.NodeFailed) != 1 || (written && w.FP == fNew) || out.Rules["novox"].Rule != delivery.RuleNotHeld { + t.Errorf("held %+v, node failed %+v, novox's want %+v (%+v): want novox kept off F", out.Held, out.NodeFailed, out.Wants["novox"], out.Rules["novox"]) + } + if _, written := out.Wants["g14"]; written { + t.Errorf("g14's want at F was rewritten: %+v", out.Wants["g14"]) + } + // A node that runs F healthy with no want, while g14 fails it alone, gets its want at F and is on no neither list (issue 1945). + n.Haves["novox"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK} + delete(before, "novox") + out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld}) + if w := out.Wants["novox"]; w.FP != fNew || len(out.Neither) != 0 { + t.Errorf("novox running F with no want: want %+v, neither %+v; want its want at F and no neither", w, out.Neither) + } + n.Haves["ace"]["mail"], n.Haves["g14"]["mail"] = failedHave, n.Haves["ace"]["mail"] + + // ADR 0557 §2: a witness that is offline keeps the have it last wrote and witnesses nothing (D1); F failing on a second node is + // not one node's failure (D3). Either way F is held and ace is set back. + for _, c := range []struct { + name string + online map[string]bool + novox delivery.Health + }{ + {"the witness sleeps", map[string]bool{"ace": true, "g14": false, "novox": true}, delivery.HealthOK}, + {"F fails on a second node", map[string]bool{"ace": true, "g14": true, "novox": true}, delivery.HealthFailed}, + } { + n.Online = c.online + n.Haves["novox"] = map[string]delivery.Have{"mail": {FP: fNew, Phase: delivery.PhaseRunning, Health: c.novox}} + if c.novox == delivery.HealthOK { + n.Haves["novox"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK} + } + before["novox"] = delivery.Want{FP: fNew, Composed: dig(fNew)} + out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld}) + if out.Held == nil || out.Held.Node != "ace" || len(out.NodeFailed) != 0 { + t.Errorf("%s: held %+v, node failed %+v: want F held at ace", c.name, out.Held, out.NodeFailed) + } + if w := out.Wants["ace"]; w.FP != fOld { + t.Errorf("%s: ace's want = %+v, want set back", c.name, w) + } + } } // 1297/B4: a module holding data is not set back; its want stays and says why. diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index cf387b7..3c6e8a9 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -34,7 +34,7 @@ func TestManifest(t *testing.T) { slices.Sort(reads) want := []string{"mesh-build-go.steps", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants", "mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", - "mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "node-engine.have"} + "mesh-delivery-indexer.modules", "mesh-delivery-indexer.placement", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "node-engine.have"} if !reflect.DeepEqual(reads, want) { t.Errorf("reads = %v, want exactly %v", reads, want) } diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index 3c31dc8..a979017 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -12,6 +12,7 @@ "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", + "mesh-delivery-indexer.placement", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "node-engine.have" diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go index 8e5ec44..a6a6936 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go @@ -105,7 +105,7 @@ func TryTools(tr Trying) []stdio.Tool { {Name: check.TryTool, Description: "Try a pushed head before its pull request is opened: the checker judges it as it judges a pull request's head, merged onto its trunk's head, " + "its unit and integration levels through the build modules, and owes the forge nothing. Answers the try's id at once; " + check.TryReadTool + " reads " + - "where it stands. A try of the same head that waits or runs is the answer to a second ask; at most 4 wait or run at once, mesh-wide. A repository " + + "where it stands. A try of the same head that waits or runs is the answer to a second ask; a new try is refused while 4 tries run on a build node or 8 wait for one, mesh-wide; tries let in while waiting may later all run at once, and each node's build capacity bounds what runs on it. A repository " + "the checker checks no level of (one whose trunk the gate does not name) is refused. A try whose ask no build node took within the step " + "bound ends as error; its ask stays in the build seat's stream and may still run later, judged by nobody (issue 734).", Input: map[string]any{"type": "object", "required": []string{"repository", "head"}, "properties": map[string]any{ @@ -135,7 +135,8 @@ func TryTools(tr Trying) []stdio.Tool { }}, {Name: check.TryReadTool, Description: "Where a try stands, as stored: queued, running, or its end (pass, fail, or error when it could not run), with what was said of it, " + - "the head and the trunk commit it was merged onto, the time it took and what failed first. Read-only.", + "the head and the trunk commit it was merged onto, the time it took and what failed first; while it runs, each ask it put as soon as it is put, " + + "with the node that took it, its state and the phase its step says (fetch and merge, a level's digest, lint, tests) and since when. Read-only.", Input: map[string]any{"type": "object", "required": []string{"id"}, "properties": map[string]any{ "id": map[string]any{"type": "string", "description": "the try's id, try-"}}}, Run: func(args map[string]any) (any, error) { diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go index 291e449..3e01724 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go @@ -427,7 +427,8 @@ func (b *bed) start(t *testing.T, env map[string]string) { if err := os.WriteFile(broker, cred, 0o600); err != nil { t.Fatal(err) } - e := map[string]string{"MESH_BROKER_FILE": broker} + // The settle window of a group's heads (check.DefaultSettle) is cut short: each bed's candidate is written whole. + e := map[string]string{"MESH_BROKER_FILE": broker, "MESH_CHECKER_SETTLE": "10ms"} for k, v := range env { e[k] = v } diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go index 767a70e..c4d3902 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go @@ -189,6 +189,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } + settleWindow, err := durationEnv(getenv, "MESH_CHECKER_SETTLE", check.DefaultSettle) + if err != nil { + return err + } link := strings.TrimSpace(getenv("MESH_CHECKER_LINK")) if link == "" { link = defaultLink @@ -221,6 +225,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Bound: checkBound, Link: link, Logf: logger.Printf, + Settle: settleWindow, } // The builder's builds say how the candidates' fingerprints stand (issue 1684); where its bucket is not there yet the line // says nothing of the builds, until the checker starts again. diff --git a/modules/mesh-delivery-checker/go.mod b/modules/mesh-delivery-checker/go.mod index f84a871..d202053 100644 --- a/modules/mesh-delivery-checker/go.mod +++ b/modules/mesh-delivery-checker/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-checker go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026191742-47a66e3c0dc8 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026233910-a2759ec92ec9 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-checker/go.sum b/modules/mesh-delivery-checker/go.sum index 9c91004..918a889 100644 --- a/modules/mesh-delivery-checker/go.sum +++ b/modules/mesh-delivery-checker/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026191742-47a66e3c0dc8 h1:6OaUWIjcpXOtnu70zeqH0VEaF4oGykrxGpFkaFOzCmY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026191742-47a66e3c0dc8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026233910-a2759ec92ec9 h1:MS7i4NVNQHMAMqRRFMdzbto31BHZY+BuEZY1ql3p4Rs= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026233910-a2759ec92ec9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-checker/internal/check/binaries.go b/modules/mesh-delivery-checker/internal/check/binaries.go new file mode 100644 index 0000000..5329c61 --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/binaries.go @@ -0,0 +1,42 @@ +package check + +import ( + "fmt" + "strings" +) + +// A change that adds or modifies a compiled program fails (issue 1577): it is build output, `go build` in a module's folder +// writes one beside go.mod, and `git add -A` takes it. The build module's fetch step, which holds the merged clone, reads the +// first bytes of each changed file and reports the programs (mesh-sdk buildstep.MergeInto); the checker judges the answer. + +// NoProgramsReported is the warning of a build module whose fetch answered no programs field: an older module, which cannot +// tell, so the check does not refuse a program it did not look for. +const NoProgramsReported = "the build module did not report programs: a compiled program in the change is not refused" + +// checkBinariesOf fails a pull request whose change adds or modifies a compiled program, naming each file. It judges the +// first fetch answer of the pull request, whichever build module gave it; a build module that reports no programs leaves a +// warning on the check and passes it. +func (st *state) checkBinariesOf(t *target) verdictPtr { + key := fmt.Sprintf("%s#%d", t.pull.Repository, t.pull.Number) + if t.fetched.Merge == "" { + return nil // no merge was made: nothing was changed to judge + } + if t.fetched.Programs == nil { + st.c.note(fmt.Sprintf("%s: %s (%s)", key, NoProgramsReported, t.module)) + st.c.warn(key, fmt.Sprintf("a compiled program is not refused: %s reported no programs", t.module)) + st.j.logf("%s: %s (%s)", key, NoProgramsReported, t.module) + return nil + } + var found []string + for _, p := range *t.fetched.Programs { + if p.Commit != "" { + found = append(found, fmt.Sprintf("%s: a compiled program added in %s and removed later: it stays in history; squash the branch (issue 1577)", p.Path, short12(p.Commit))) + continue + } + found = append(found, p.Path+": a compiled program is build output: remove it and add it to .gitignore (issue 1577)") + } + if len(found) > 0 { + return failed(t.node, fmt.Sprintf("%s: %s", key, strings.Join(found, "; "))) + } + return nil +} diff --git a/modules/mesh-delivery-checker/internal/check/binaries_test.go b/modules/mesh-delivery-checker/internal/check/binaries_test.go new file mode 100644 index 0000000..2a3ab5e --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/binaries_test.go @@ -0,0 +1,63 @@ +package check + +import ( + "slices" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +func TestAChangeWhoseFetchReportsProgramsFailsNamingEachFile(t *testing.T) { // 1577 + prog := func(p ...string) *[]ProgramFile { + out := []ProgramFile{} + for _, f := range p { + path, commit, _ := strings.Cut(f, "@") // `path@commit`: added by a commit of the branch and removed later + out = append(out, ProgramFile{Path: path, Kind: "ELF", Commit: commit}) + } + return &out + } + for _, c := range []struct { + name string + programs *[]ProgramFile + fail []string // paths named; none passes + warned bool // the build module reported no programs field + }{ + {"none found", prog(), nil, false}, + {"one program", prog("modules/m/m"), []string{"modules/m/m"}, false}, + {"two programs", prog("a/x", "b/y"), []string{"a/x", "b/y"}, false}, + {"added and removed in the branch", prog("gone/prog@0123456789abcdef0123"), []string{"gone/prog: a compiled program added in 0123456789ab and removed later: it stays in history; squash the branch (issue 1577)"}, false}, + {"an older build module reports nothing", nil, nil, true}, + } { + b := newBench(t) + b.fetch.Merge, b.fetch.Programs = "mmmm", c.programs + cs := caseOf(t, pullOf(repoA, 7, head1)) + v := b.judge(t, cs).Verdict + if got := slices.ContainsFunc(cs.Notes(), func(n string) bool { return strings.Contains(n, NoProgramsReported) }); got != c.warned { + t.Errorf("%s: warned = %v, want %v (%v)", c.name, got, c.warned, cs.Notes()) + } + line := Line(v, cs, cs.Pulls[0], nil, nil, nil) + if got := strings.Contains(line, "a compiled program is not refused: mesh-build-go reported no programs"); got != c.warned { + t.Errorf("%s: the status line holds the warning = %v, want %v: %s", c.name, got, c.warned, line) + } + if len(c.fail) == 0 { + if v.State != delivery.VerdictPass { + t.Errorf("%s: verdict = %+v", c.name, v) + } + continue + } + if v.State != delivery.VerdictFail { + t.Errorf("%s: verdict = %+v, want a failure", c.name, v) + continue + } + for _, p := range c.fail { + want := p + if !strings.Contains(p, " added in ") { + want = p + ": a compiled program is build output: remove it and add it to .gitignore (issue 1577)" + } + if !strings.Contains(v.Failed.Words, want) { + t.Errorf("%s: words %q do not name %s", c.name, v.Failed.Words, p) + } + } + } +} diff --git a/modules/mesh-delivery-checker/internal/check/case.go b/modules/mesh-delivery-checker/internal/check/case.go index 169d55e..7252046 100644 --- a/modules/mesh-delivery-checker/internal/check/case.go +++ b/modules/mesh-delivery-checker/internal/check/case.go @@ -43,9 +43,16 @@ type Case struct { Try *TryOptions notesMu sync.Mutex - notes []string + // warnings are, by pull request, what part of its check did not run (Case.warn). + warnings map[string]string + notes []string // asks are the ids of the asks the check put on the build seats' streams, in order: a try names them (Try.Asks). asks []string + // Watch, when set, hears each ask the check puts, as it is put (step nil), and each value its step key takes while + // the check waits for it: the node that took it, the phase it says, its end. A try writes them into its record, so a + // running try names its asks and where each stands (issue 1890). Called from the check's goroutines; it must not block + // for long. + Watch func(askID string, step *delivery.Step) // Progress says what the check is doing, so a check that ran past its bound names the step it was stuck at. Progress *Progress // Kind is what asked the check (KindNew, KindNodes, KindTrunk): a carry after a trunk or sibling move is asked on the @@ -186,6 +193,23 @@ func (c *Case) note(s string) { } } +// warn says of a pull request (pullKey) that a part of its check did not run, in words for its status line (issue 1577). +func (c *Case) warn(key, words string) { + c.notesMu.Lock() + defer c.notesMu.Unlock() + if c.warnings == nil { + c.warnings = map[string]string{} + } + c.warnings[key] = words +} + +// Warning is what warn said of a pull request, or "". +func (c *Case) Warning(key string) string { + c.notesMu.Lock() + defer c.notesMu.Unlock() + return c.warnings[key] +} + // Notes are the words noted on the case, in the order they were noted. func (c *Case) Notes() []string { c.notesMu.Lock() diff --git a/modules/mesh-delivery-checker/internal/check/checker.go b/modules/mesh-delivery-checker/internal/check/checker.go index e74f944..940f434 100644 --- a/modules/mesh-delivery-checker/internal/check/checker.go +++ b/modules/mesh-delivery-checker/internal/check/checker.go @@ -85,6 +85,10 @@ type Checker struct { Announce Announcer Now func() time.Time Logf func(format string, args ...any) + // Settle is how long a group's heads must stay the same after the candidate last changed them before its check starts + // (issue 1890): the members of a group are announced one after another, and a check of each partial set is thrown + // away at the next. None when 0; DefaultSettle in the module. + Settle time.Duration once sync.Once wg sync.WaitGroup @@ -96,15 +100,77 @@ type Checker struct { epoch uint64 // changes of the nodes seen since this process started nodesFail map[string]uint64 // by verdict key: the epoch a failure of the nodes was written at held map[string]string // by verdict key: the turn that holds its check off, as last said + live map[string]liveCheck // by verdict key: the check running in this process, ended when its heads go + settling map[string]*window // by candidates key: the window its heads settle in +} + +// liveCheck is a check running in this process: the candidates key of its group and the end of its run. +type liveCheck struct { + group string + end context.CancelCauseFunc +} + +// window is a group's settle window: aborted with errUnsettled when its heads change again or the group goes, and with +// errSettled to end it at once. +type window struct{ abort context.CancelCauseFunc } + +var ( + errUnsettled = errors.New("the group's heads changed again") + errSettled = errors.New("the group's heads settled") +) + +// settleNow ends the settle window of every group at once, as if Settle had passed. +func (ch *Checker) settleNow() { + ch.mu.Lock() + defer ch.mu.Unlock() + for _, w := range ch.settling { + w.abort(errSettled) + } +} + +// DefaultSettle is the module's Settle (issue 1890). On 2026-10-26 the three members of a group were announced within 2.8 +// seconds of each other (20:31:52.3, :53.7, :55.1 UTC), a session opening or pushing them one API call after another; +// fifteen seconds holds five times that, and costs a check of 10 to 18 minutes under 2.5 % of its time. A member that comes +// later still is no waste beyond its asks already taken: the check of the heads it replaced is ended at once. +const DefaultSettle = 15 * time.Second + +// EndedWords open the words of a verdict the checker ended without judging it, its heads no candidate's any more (issue +// 1890): never a failure of the change, and judged whole if a candidate holds those heads again. +const EndedWords = "ended: " + +// IsEnded says whether a verdict was ended without being judged (EndedWords). +func IsEnded(v delivery.Verdict) bool { + return v.State == delivery.VerdictFail && v.Failed.Machine == "checker" && strings.HasPrefix(v.Failed.Words, EndedWords) +} + +// superseded is the cause a running check is ended with: the heads that replaced its own, or none when its group went. +type superseded struct{ by string } + +func (s superseded) Error() string { + if s.by == "" { + return EndedWords + "no candidate holds these heads any more: the group was closed or merged" + } + return EndedWords + "superseded by " + s.by } func (ch *Checker) init() { ch.once.Do(func() { ch.known, ch.inflight, ch.again, ch.tried = map[string]delivery.Candidate{}, map[string]bool{}, map[string]bool{}, map[string]bool{} ch.nodesFail, ch.held = map[string]uint64{}, map[string]string{} + ch.live, ch.settling = map[string]liveCheck{}, map[string]*window{} }) } +// endOthers ends every check of the group running in this process whose verdict key is not keep: no candidate holds its +// heads any more (issue 1890). Called with ch.mu held. +func (ch *Checker) endOthers(group, keep, by string) { + for k, l := range ch.live { + if l.group == group && k != keep { + l.end(superseded{by: by}) + } + } +} + func (ch *Checker) logf(format string, args ...any) { if ch.Logf != nil { ch.Logf(format, args...) @@ -137,17 +203,62 @@ func (ch *Checker) Candidate(ctx context.Context, key string, value []byte) { ch.logf("candidates.%s holds what is not a candidate, left: %v", key, err) return } + vkey, _ := delivery.CheckerVerdictKey(c.Group, c.HeadsHash) ch.mu.Lock() + prev, had := ch.known[key] ch.known[key] = c + // A check of heads this candidate no longer holds is ended at once (issue 1890): a member's new head, a member joining + // or leaving. + ch.endOthers(key, vkey, GateHeads(c)) + moved := !had || prev.HeadsHash != c.HeadsHash + if ch.Settle > 0 && (moved || ch.settling[key] != nil) { + // The group's heads changed: its check waits until they have been the same for Settle, so a group whose members + // are announced one after another is checked once, whole. A later change starts the window again. The wait is a + // context's bound, no timer (ADR 0437 §8). + if w := ch.settling[key]; w != nil { + w.abort(errUnsettled) + } + base, abort := context.WithCancelCause(ctx) + w := &window{abort: abort} + ch.settling[key] = w + ch.wg.Add(1) + go func() { + defer ch.wg.Done() + wait, done := context.WithTimeout(base, ch.Settle) + defer done() + <-wait.Done() + settled := errors.Is(wait.Err(), context.DeadlineExceeded) || errors.Is(context.Cause(wait), errSettled) + ch.mu.Lock() + if !settled || ch.settling[key] != w { + ch.mu.Unlock() + return // changed again, forgotten, or the checker stops + } + delete(ch.settling, key) + now, ok := ch.known[key] + ch.mu.Unlock() + abort(nil) + if ok { + ch.consider(ctx, key, now) + } + }() + ch.mu.Unlock() + return + } ch.mu.Unlock() ch.consider(ctx, key, c) } -// Forget takes a candidate out (its key was deleted). +// Forget takes a candidate out (its key was deleted): a check of its heads still running is ended, as no candidate holds +// them any more. func (ch *Checker) Forget(key string) { ch.init() ch.mu.Lock() delete(ch.known, key) + if w := ch.settling[key]; w != nil { + w.abort(errUnsettled) + delete(ch.settling, key) + } + ch.endOthers(key, "", "") ch.mu.Unlock() } @@ -190,32 +301,49 @@ func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidat } cs.Queued = ch.now() ch.mu.Lock() + if ch.settling[key] != nil { + ch.mu.Unlock() + return // its heads are settling: the window's end considers it (Settle) + } if ch.inflight[cs.Key] { ch.again[cs.Key] = true // looked at once more when the check ends, so a change of the trunk is never lost ch.mu.Unlock() return } ch.inflight[cs.Key] = true + run, end := context.WithCancelCause(ctx) + ch.live[cs.Key] = liveCheck{group: key, end: end} ch.mu.Unlock() ch.wg.Add(1) go func() { defer ch.wg.Done() + vkey := cs.Key + defer func() { + ch.mu.Lock() + delete(ch.inflight, vkey) + delete(ch.again, vkey) + delete(ch.live, vkey) + ch.mu.Unlock() + end(nil) + }() for { // Every check is asked at once: the build seats' streams hold the backlog, a new head's check and a check after a // failure of the nodes on check, a re-judge after a trunk move on recheck, pulled after them (ADR 0525 §4, §9; // ADR 0536 §4). The checker holds no place and no capacity of its own. ch.prepare(ctx, cs) - ch.try(ctx, key, cs) + ch.try(ctx, run, key, cs) ch.mu.Lock() - more := ch.again[cs.Key] - delete(ch.again, cs.Key) - if !more || ctx.Err() != nil { - delete(ch.inflight, cs.Key) - ch.mu.Unlock() + more := ch.again[vkey] + delete(ch.again, vkey) + ch.mu.Unlock() + if !more || run.Err() != nil { return } - ch.mu.Unlock() + // Looked at again for the same heads only: a candidate that moved to other heads has its own check (issue 1890). if next := CaseOf(ch.candidate(key, c), ch.heads()); next != nil { + if next.Key != vkey { + return + } cs = next } cs.Queued = ch.now() @@ -252,6 +380,8 @@ func (ch *Checker) prepare(ctx context.Context, cs *Case) { return case IsConflictFailure(v): return // a head git refused: its next check is whole (ADR 0548 §7) + case IsEnded(v): + return // ended unjudged when its heads went (issue 1890): held again, they are checked whole } cs.Kind = KindTrunk if v.State == delivery.VerdictPass && ch.stale(v, cs) { @@ -402,8 +532,8 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64 return &claimed{rev}, nil } return nil, nil - case ch.stale(cur, cs): - retake = true + case IsEnded(cur), ch.stale(cur, cs): + retake = true // heads held again after their check was ended unjudged (issue 1890), or a trunk that moved case cur.State == delivery.VerdictFail && IsNodesFailure(cur): ch.mu.Lock() at, ok := ch.nodesFail[cs.Key] @@ -483,9 +613,13 @@ func IsNodesFailure(v delivery.Verdict) bool { return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, UnheardWords) } -// try claims the check, judges it within its bound and writes the verdict and the owed statuses. A check that ends -// because ctx did is left running: the checker takes it again when it starts. -func (ch *Checker) try(ctx context.Context, key string, cs *Case) { +// try claims the check, judges it within its bound under run and writes the verdict and the owed statuses. A check that ends +// because ctx did is left running: the checker takes it again when it starts. A check whose run was ended because no +// candidate holds its heads any more (issue 1890) is ended: its verdict says so, and no status is written. +func (ch *Checker) try(ctx, run context.Context, key string, cs *Case) { + if run.Err() != nil { + return // ended before it was claimed: nothing to write + } ch.mu.Lock() first := !ch.tried[cs.Key] ch.tried[cs.Key] = true @@ -501,21 +635,26 @@ func (ch *Checker) try(ctx context.Context, key string, cs *Case) { } began := ch.now() ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{}) - out := ch.judge(ctx, cs) + out := ch.judge(run, cs) // An ask the build seat gave up on judged nothing (issue 734): the check is asked again under the same claim, its status // pending, and after LostAskTries it fails as not heard, judged again when the nodes change. - for try := 1; IsLostAsk(out.Verdict) && ctx.Err() == nil; try++ { + for try := 1; IsLostAsk(out.Verdict) && run.Err() == nil; try++ { if try >= LostAskTries { out.Verdict.Failed.Words = strings.TrimSpace(out.Verdict.Failed.Words + "\nasked " + fmt.Sprint(LostAskTries) + " times and given up each time: " + UnheardWords) break } ch.logf("%s: %s; asked again (%d of %d)", cs.Key, out.Verdict.Failed.Words, try+1, LostAskTries) - out = ch.judge(ctx, cs) + out = ch.judge(run, cs) } if ctx.Err() != nil { return } + var why superseded + if errors.As(context.Cause(run), &why) { + ch.end(ctx, cs, cl, why) + return + } v := out.Verdict v.Group, v.Heads, v.Trunk, v.Beside = cs.Group, cs.Heads, cs.Trunk, nil for r, c := range out.Beside { @@ -569,6 +708,26 @@ func (ch *Checker) try(ctx context.Context, key string, cs *Case) { ch.logf("%s: %s", cs.Key, v.State) } +// end writes the verdict of a check whose heads no candidate holds any more (issue 1890), by compare-and-set on its claim, +// and says it once in the log with the asks it had put: they stay in the build seats' streams, since an ask once put cannot +// be taken back (buildrun.Asker has no withdraw), and are no longer waited for. No status is written: the heads that +// replaced these have their own check, whose pending status stands. +func (ch *Checker) end(ctx context.Context, cs *Case, cl *claimed, why superseded) { + v := delivery.Verdict{Group: cs.Group, Heads: cs.Heads, Trunk: cs.Trunk, State: delivery.VerdictFail, + Failed: delivery.FailedFirst{Machine: "checker", Words: why.Error()}} + data, err := delivery.EncodeVerdict(v) + if err == nil { + _, err = ch.Verdicts.Update(ctx, cs.Key, data, cl.rev) + } + if err != nil { + ch.logf("%s: %s, but the verdict could not say so: %v", cs.Key, why.Error(), err) + return + } + stage, _ := cs.Progress.Now() + asks := cs.Asks() + ch.logf("%s: %s; it was at %q with %d asks put %v, which are no longer waited for", cs.Key, why.Error(), stage, len(asks), asks) +} + // judge runs the case within the bound. A judge that fails to judge, and a check that ran past its bound, are verdicts of // failure naming the machine and the step. func (ch *Checker) judge(ctx context.Context, cs *Case) Outcome { @@ -667,7 +826,7 @@ func (ch *Checker) Sweep(ctx context.Context) error { continue } cur.State = delivery.VerdictFail - cur.Failed = delivery.FailedFirst{Machine: "checker", Words: "closed at start: left running, and no candidate holds these heads any more, so nothing will check them"} + cur.Failed = delivery.FailedFirst{Machine: "checker", Words: EndedWords + "closed at start: left running, and no candidate holds these heads any more, so nothing will check them"} data, err := delivery.EncodeVerdict(cur) if err != nil { return err diff --git a/modules/mesh-delivery-checker/internal/check/checker_test.go b/modules/mesh-delivery-checker/internal/check/checker_test.go index e8126c9..bc5820b 100644 --- a/modules/mesh-delivery-checker/internal/check/checker_test.go +++ b/modules/mesh-delivery-checker/internal/check/checker_test.go @@ -3,6 +3,7 @@ package check import ( "context" "encoding/json" + "fmt" "strings" "sync" "testing" @@ -333,7 +334,10 @@ func TestTheStatusIsPendingWhileTheCheckRunsAndEndsSuccessOrFailure(t *testing.T } } -func TestSweepClosesARunningKeyNoCandidateHoldsAndLeavesOneItDoes(t *testing.T) { +// No check of heads no candidate holds is left running (issue 1890): at start the sweep ends a key left running; while the +// checker runs, a check whose group moved to other heads is ended at once, unjudged, and writes no status; heads held again +// are checked whole; and a group's check starts only once its heads settled, so a partial set of members is never checked. +func TestHeadsNoCandidateHoldsAreNeverLeftRunning(t *testing.T) { ctx, j := context.Background(), passJudge() r := newRig(t, j) cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1)) @@ -351,12 +355,91 @@ func TestSweepClosesARunningKeyNoCandidateHoldsAndLeavesOneItDoes(t *testing.T) if err := r.ch.Sweep(ctx); err != nil { t.Fatal(err) } - if v := r.verdict(t, dead); v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "closed at start") { + if v := r.verdict(t, dead); !IsEnded(v) || !strings.Contains(v.Failed.Words, "closed at start") { t.Errorf("the dead key = %+v", v) } if v := r.verdict(t, live); v.State != delivery.VerdictRunning { t.Errorf("the live key = %+v, want it left for the check to take over", v) } + + // A member joins while the check of the lone member's heads runs: that check is ended, its judge's run cancelled. + started, ended := make(chan struct{}), make(chan error, 1) + j = &judgeFunc{f: func(run context.Context, _ *Case, call int) (Outcome, error) { + if call == 1 { + close(started) + <-run.Done() + ended <- context.Cause(run) + return Outcome{}, run.Err() + } + return Outcome{Verdict: passing()}, nil + }} + r = newRig(t, j) + var logged []string + var logMu sync.Mutex + r.ch.Logf = func(f string, a ...any) { logMu.Lock(); logged = append(logged, fmt.Sprintf(f, a...)); logMu.Unlock() } + alone, both := cand, candidateOf(t, "feat/x", pullOf(repoA, 7, head1), pullOf(repoSDK, 3, head2)) + r.ch.Candidate(ctx, "feat%2Fx", alone) + <-started + r.ch.Candidate(ctx, "feat%2Fx", both) + var bothC delivery.Candidate + _ = json.Unmarshal(both, &bothC) + select { + case err := <-ended: + if err == nil || !strings.Contains(err.Error(), "superseded by "+GateHeads(bothC)) { + t.Errorf("the old check's run ended with %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("the check of the heads the candidate left still runs") + } + r.ch.Wait() + if v := r.verdict(t, live); !IsEnded(v) || v.Failed.Words != EndedWords+"superseded by "+GateHeads(bothC) { + t.Errorf("the superseded verdict = %+v", v.Failed) + } + if v := r.verdict(t, keyOf(t, "feat/x", both)); v.State != delivery.VerdictPass { + t.Errorf("the group's verdict = %s", v.State) + } + okey, _ := delivery.CheckerOwedKey(repoA, 7, head1) + if raw, _, _, _ := r.owed.Get(ctx, okey); !strings.Contains(string(raw), `"success"`) { + t.Errorf("the head's status = %s, want the group's success, never the ended check's", raw) + } + logMu.Lock() + said := 0 + for _, l := range logged { + if strings.Contains(l, "ended: superseded") { + said++ + } + } + logMu.Unlock() + if said != 1 { + t.Errorf("the end was said %d times: %v", said, logged) + } + // The member leaves again: the ended heads are held again and checked whole. + r.ch.Candidate(ctx, "feat%2Fx", alone) + r.ch.Wait() + if v := r.verdict(t, live); v.State != delivery.VerdictPass || j.count() != 3 { + t.Errorf("the heads held again = %s after %d judgings, want a pass of a third", v.State, j.count()) + } + + // Members announced one after another: only the settled, whole set is checked. + j = passJudge() + r = newRig(t, j) + r.ch.Settle = time.Hour + sdk := candidateOf(t, "feat/y", pullOf(repoSDK, 4, head2)) + whole := candidateOf(t, "feat/y", pullOf(repoSDK, 4, head2), pullOf(repoA, 8, head1)) + r.ch.Candidate(ctx, "feat%2Fy", sdk) + r.ch.Candidate(ctx, "feat%2Fy", whole) + r.ch.Reconsider(ctx) // a trunk move or a turn while the heads settle starts nothing + if j.count() != 0 { + t.Fatalf("judged %d times before the heads settled", j.count()) + } + r.ch.settleNow() + r.ch.Wait() + if _, _, found, _ := r.verdicts.Get(ctx, keyOf(t, "feat/y", sdk)); found || j.count() != 1 { + t.Errorf("the partial set has a verdict (%v) or the judge ran %d times, want the whole set alone", found, j.count()) + } + if v := r.verdict(t, keyOf(t, "feat/y", whole)); v.State != delivery.VerdictPass { + t.Errorf("the whole set = %s", v.State) + } } func TestTheForgeBridgeIsToldOnceForEachChangeOfAStatus(t *testing.T) { // 1683 (the checker's half of AC5) diff --git a/modules/mesh-delivery-checker/internal/check/judge.go b/modules/mesh-delivery-checker/internal/check/judge.go index f104530..b6a1ed8 100644 --- a/modules/mesh-delivery-checker/internal/check/judge.go +++ b/modules/mesh-delivery-checker/internal/check/judge.go @@ -427,8 +427,14 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l if err := st.j.Asks.Put(ctx, a); err != nil { return fail(t.module, fmt.Sprintf("the check of %s#%d could not be asked: %v", p.Repository, p.Number, err)) } + waiting := ctx + if watch := st.c.Watch; watch != nil { + id := a.ID + watch(id, nil) // put: a try names it at once, not only when it ends (issue 1890) + waiting = buildrun.WithStepSeen(ctx, func(s delivery.Step) { watch(id, &s) }) + } var err error - step, err = st.j.Asks.Wait(ctx, a) + step, err = st.j.Asks.Wait(waiting, a) if err != nil { if ctx.Err() != nil { return fail(t.module, ctx.Err().Error()) @@ -489,6 +495,9 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l if v := st.checkPinsOf(ctx, t); v != nil { return v } + if v := st.checkBinariesOf(t); v != nil { + return v + } } ran := 0.0 for _, la := range answer.Levels { diff --git a/modules/mesh-delivery-checker/internal/check/line.go b/modules/mesh-delivery-checker/internal/check/line.go index f09b839..59d2da8 100644 --- a/modules/mesh-delivery-checker/internal/check/line.go +++ b/modules/mesh-delivery-checker/internal/check/line.go @@ -87,18 +87,20 @@ type pieces struct { judged string // carried says, of a carried verdict, the commits it was carried from (ADR 0548 §8); never dropped. carried string + // warning says that a part of the check did not run, for instance that an older build module could not refuse a compiled program; never dropped. + warning string } // pullKey names a pull request of a case: `#`. func pullKey(p Pull) string { return fmt.Sprintf("%s#%d", p.Repository, p.Number) } func (p pieces) join(drop int) string { - parts := []string{p.head, p.commits, p.carried, p.counts, p.judged} + parts := []string{p.head, p.commits, p.carried, p.warning, p.counts, p.judged} switch { case drop < 3 && p.times != "": parts = append(parts, p.times) case drop == 3 && p.shortTimes != "": - parts = []string{p.head, p.commits, p.carried, p.shortCounts, p.judged, p.shortTimes} + parts = []string{p.head, p.commits, p.carried, p.warning, p.shortCounts, p.judged, p.shortTimes} } if p.over != "" { parts = append(parts, p.over) @@ -210,6 +212,9 @@ func carriedWords(from map[string]string, v delivery.Verdict, cs *Case, p Pull) func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string, judged map[string]string) pieces { var pc pieces pc.judged = judged[pullKey(p)] + if cs != nil { + pc.warning = cs.Warning(pullKey(p)) + } commits := short12(p.Head) + " on " + short12(p.Trunk) switch v.State { case delivery.VerdictRunning: diff --git a/modules/mesh-delivery-checker/internal/check/results.go b/modules/mesh-delivery-checker/internal/check/results.go index 7fc8ee6..768e7c3 100644 --- a/modules/mesh-delivery-checker/internal/check/results.go +++ b/modules/mesh-delivery-checker/internal/check/results.go @@ -45,6 +45,17 @@ type Fetched struct { Conflicts []string `json:"conflicts,omitempty"` Changed []string `json:"changed,omitempty"` Trees map[string]MergeTree `json:"trees,omitempty"` + // Programs are the changed files whose first bytes are an executable binary, read by the build module in its workspace + // (issue 1577); nil for a build module that does not report it, a pointer to an empty list for one that found none. + Programs *[]ProgramFile `json:"programs,omitempty"` +} + +// ProgramFile is a changed file that is a compiled program, and its format (ELF, Mach-O or PE). +type ProgramFile struct { + Path string `json:"path"` + Kind string `json:"kind"` + // Commit is set for a program a commit of the branch added and a later one removed: it stays in history once merged. + Commit string `json:"commit,omitempty"` } // GoPackage is one package of a Go test result. diff --git a/modules/mesh-delivery-checker/internal/check/testdata/captured/README b/modules/mesh-delivery-checker/internal/check/testdata/captured/README index 58a30ed..253863c 100644 --- a/modules/mesh-delivery-checker/internal/check/testdata/captured/README +++ b/modules/mesh-delivery-checker/internal/check/testdata/captured/README @@ -10,7 +10,7 @@ TestFetchWithABaseMergesAndAnswersThePreview and TestFetchWithAConflictEndsPasse into a throwaway test that wrote them out, at mesh-catalog feat/r4c5-checker d5804927 (the git commands are the test's fake, so the shas are its constants; the shape and the field names are the module's own marshalling). node-fetch-clean.json is the FetchResult json.Marshal of mesh-build-node's fetch (fetch.go, preview.go) in its test TestFetchWithABaseMergesAndAnswersThePreview -(cmd/mesh-build-node/preview_test.go) at the same commit. Captured on 2026-10-21 for issue 1679. +(cmd/mesh-build-node/preview_test.go) at the same commit. Captured on 2026-10-21 for issue 1679. Both clean fetches carry `"programs":[]`, which a merging fetch of the build modules answers since issue 1577 (mesh-sdk buildstep.MergeInto's Programs), added by hand to the captured bytes: a fixture without it is an older build module's, and its line warns that no compiled program is refused. forge-get-file-go-mod.json is the answer of the forge's gitea_get_file (mesh-catalog modules/gitea, tools/index.ts) for modules/mesh-delivery-builder/go.mod of novox/mesh-delivery at main, and forge-compare-trunk.json the answer of gitea_compare for novox/mesh-sdk base main, head cb54756c3865 (a commit on the diff --git a/modules/mesh-delivery-checker/internal/check/testdata/captured/go-fetch-clean.json b/modules/mesh-delivery-checker/internal/check/testdata/captured/go-fetch-clean.json index ce2e947..a87c222 100644 --- a/modules/mesh-delivery-checker/internal/check/testdata/captured/go-fetch-clean.json +++ b/modules/mesh-delivery-checker/internal/check/testdata/captured/go-fetch-clean.json @@ -1 +1 @@ -{"workspace":"mvii1iwu5414e4","commit":"0123456789abcdef0123456789abcdef01234567","modules":["."],"base":"abcdef0123456789abcdef0123456789abcdef01","merge":"fedcba9876543210fedcba9876543210fedcba98","changed":["a.go","modules/m/main.go"],"trees":{"modules/m":{"tree":"1111111111111111111111111111111111111111","manifest":"ca3d163bab055381827226140568f3bef7eaac187cebd76878e0b63e9e442356","locks":{"go.sum":"c5fc83c01e92404452b986527d239140ccf9a48b88e0c268fbf38c2e1429e9c9"}}}} +{"workspace":"mvii1iwu5414e4","commit":"0123456789abcdef0123456789abcdef01234567","modules":["."],"base":"abcdef0123456789abcdef0123456789abcdef01","merge":"fedcba9876543210fedcba9876543210fedcba98","changed":["a.go","modules/m/main.go"],"programs":[],"trees":{"modules/m":{"tree":"1111111111111111111111111111111111111111","manifest":"ca3d163bab055381827226140568f3bef7eaac187cebd76878e0b63e9e442356","locks":{"go.sum":"c5fc83c01e92404452b986527d239140ccf9a48b88e0c268fbf38c2e1429e9c9"}}}} diff --git a/modules/mesh-delivery-checker/internal/check/testdata/captured/node-fetch-clean.json b/modules/mesh-delivery-checker/internal/check/testdata/captured/node-fetch-clean.json index 4c8eedb..55ecfec 100644 --- a/modules/mesh-delivery-checker/internal/check/testdata/captured/node-fetch-clean.json +++ b/modules/mesh-delivery-checker/internal/check/testdata/captured/node-fetch-clean.json @@ -1 +1 @@ -{"workspace":"mvii1sxt979f67","commit":"0123456789abcdef0123456789abcdef01234567","modules":["modules/m"],"installed":[],"refused":[],"base":"abcdef0123456789abcdef0123456789abcdef01","merge":"0123456789abcdef0123456789abcdef01234567","changed":["modules/m/package.json"],"trees":{"modules/m":{"tree":"0123456789abcdef0123456789abcdef01234567","manifest":"0581b699d9d05517e69b21f5cedbeb2eee8aba2b205280b561be731594d7feef"}}} +{"workspace":"mvii1sxt979f67","commit":"0123456789abcdef0123456789abcdef01234567","modules":["modules/m"],"installed":[],"refused":[],"base":"abcdef0123456789abcdef0123456789abcdef01","merge":"0123456789abcdef0123456789abcdef01234567","changed":["modules/m/package.json"],"programs":[],"trees":{"modules/m":{"tree":"0123456789abcdef0123456789abcdef01234567","manifest":"0581b699d9d05517e69b21f5cedbeb2eee8aba2b205280b561be731594d7feef"}}} diff --git a/modules/mesh-delivery-checker/internal/check/try.go b/modules/mesh-delivery-checker/internal/check/try.go index af0b409..757ce4e 100644 --- a/modules/mesh-delivery-checker/internal/check/try.go +++ b/modules/mesh-delivery-checker/internal/check/try.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "regexp" + "slices" "sort" "strconv" "strings" @@ -29,8 +30,13 @@ const ( // TriesState is the checker's state of tries: the bucket mesh-delivery-checker_tries, keys try-. TriesState = "tries" TriesBucket = delivery.CheckerModule + "_" + TriesState - // MaxTries is how many tries may wait or run at once, mesh-wide: each holds a build node for minutes. - MaxTries = 4 + // MaxTries and MaxTriesWaiting are checked when a try is asked, mesh-wide: a new try is refused while MaxTries tries run + // on a build node or MaxTriesWaiting wait for one (queued, or their asks not yet taken by a node). It is no bound on + // what runs later: tries let in while waiting may later all run at once, and each node's build capacity bounds what + // runs on it. A try waiting behind the seat's other asks holds no node, so it no longer blocks the next try (issue + // 1890: at 19:53 on 2026-10-26 four tries sat queued behind 44 checks for up to 47 minutes and no fifth could be asked). + MaxTries = 4 + MaxTriesWaiting = 8 // TryScan is how many of the newest tries a new ask reads to find the ones that wait or run (issue 1827). TryScan = 64 // TryKeep is how long an ended try's key is kept before the checker removes it. @@ -79,12 +85,104 @@ type Try struct { Machines []string `json:"machines,omitempty"` // Asks are the ids of the asks the try put on the build seats' streams, each ending on the step key . of its // build module: the delivery view counts those steps as a try's, not a candidate's check (build_usage). - Asks []string `json:"asks,omitempty"` + Asks []string `json:"asks,omitempty"` + // Steps say where each ask stands while the try runs and after: asked until a node takes it, then the node, its + // state, the phase its step says and when (issue 1890: a 31-minute try could not be told queued from running). + Steps []TryStep `json:"steps,omitempty"` Asked time.Time `json:"asked"` Started time.Time `json:"started,omitzero"` Ended time.Time `json:"ended,omitzero"` } +// TryAsked is a TryStep's state before a build node took its ask: it waits in the seat's stream. +const TryAsked = "asked" + +// TryStep is one ask of a try as its step key last said it. +type TryStep struct { + Ask string `json:"ask"` + Node string `json:"node,omitempty"` + State string `json:"state"` + Phase string `json:"phase,omitempty"` + Started time.Time `json:"started,omitzero"` + PhaseAt time.Time `json:"phase-at,omitzero"` + Ended time.Time `json:"ended,omitzero"` +} + +// tryWriter writes a running try's record again as its asks are put and their steps move: one writer per try, holding +// the revision it wrote last, so the try's end is written at that revision. A write refused (the try ended by hand) +// stops the writes; the end's own write then says it. +type tryWriter struct { + mu sync.Mutex + tr *Tryer + ctx context.Context + key string + t Try + rev uint64 + stopped bool +} + +// heard takes an ask put (step nil) or a value of its step key in, and writes the record when it changed. +func (w *tryWriter) heard(askID string, step *delivery.Step) { + w.mu.Lock() + defer w.mu.Unlock() + if w.stopped { + return + } + changed := false + if !slices.Contains(w.t.Asks, askID) { + w.t.Asks, changed = append(w.t.Asks, askID), true + } + i := slices.IndexFunc(w.t.Steps, func(s TryStep) bool { return s.Ask == askID }) + if i < 0 { + w.t.Steps, changed = append(w.t.Steps, TryStep{Ask: askID, State: TryAsked}), true + i = len(w.t.Steps) - 1 + } + if step != nil { + now := TryStep{Ask: askID, Node: step.Node, State: step.State, Phase: step.Phase, Started: step.Started, + PhaseAt: step.PhaseAt, Ended: step.Ended} + changed = changed || now != w.t.Steps[i] + w.t.Steps[i] = now + } + if changed { + w.write() + } +} + +// write writes the record at the revision written last; a refusal stops the writes. +func (w *tryWriter) write() { + data, err := encodeTry(w.t) + if err != nil { + return + } + rev, err := w.tr.Tries.Update(w.ctx, w.key, data, w.rev) + if err != nil { + w.stopped = true + w.tr.logf("%s: its asks' progress is no longer written: %v", w.key, err) + return + } + w.rev = rev +} + +// end stops the writes and answers the record and its revision, for the try's end. +func (w *tryWriter) end() (Try, uint64) { + w.mu.Lock() + defer w.mu.Unlock() + w.stopped = true + return w.t, w.rev +} + +// HoldsANode says whether the try holds a build node now: it runs, and a step of its asks runs on a node, or it names +// no step (a checker that writes its asks only at the end, or one about to put its first). +func (t Try) HoldsANode() bool { + if t.State != TryRunning { + return false + } + if len(t.Steps) == 0 { + return true + } + return slices.ContainsFunc(t.Steps, func(s TryStep) bool { return s.State == delivery.StepRunning }) +} + // Over says whether the try came to its end. func (t Try) Over() bool { return t.State == TryPass || t.State == TryFail || t.State == TryError } @@ -124,8 +222,8 @@ func NotChecked(repository string) error { } // AskTry writes a new try, queued, and answers its id: a try of the same head that waits or runs is the answer to a second -// ask, and at most MaxTries wait or run at once. trunks are the gate's trunk heads of every repository (source.main): a -// repository with none is no repository the checker checks. A trunk asked by name is its trunk head now. +// ask, and a new try is refused while MaxTries tries run on a build node or MaxTriesWaiting wait for one. trunks are the +// gate's trunk heads of every repository (source.main): a repository with none is no repository the checker checks. A trunk asked by name is its trunk head now. func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (string, error) { repo := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(a.Repository), ".git")) head := strings.ToLower(strings.TrimSpace(a.Head)) @@ -188,7 +286,7 @@ func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) ( }); err != nil { return "", err } - var open []string + var holding, waiting []string for i, cur := range tries { if !live[i] { continue @@ -196,10 +294,18 @@ func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) ( if cur.Repository == t.Repository && cur.Head == t.Head && cur.Trunk == t.Trunk && cur.Tests == t.Tests && cur.Level == t.Level { return cur.ID, nil } - open = append(open, fmt.Sprintf("%s (%s@%.12s, %s)", cur.ID, cur.Repository, cur.Head, cur.State)) + said := fmt.Sprintf("%s (%s@%.12s, %s)", cur.ID, cur.Repository, cur.Head, cur.State) + if cur.HoldsANode() { + holding = append(holding, said) + } else { + waiting = append(waiting, said) + } } - if len(open) >= MaxTries { - return "", fmt.Errorf("%d tries wait or run, the most at once: %s; ask again when one ends", len(open), strings.Join(open, ", ")) + // Checked at this ask only: a try queued behind the seat's checks holds no node; tries let in while waiting may later + // all run at once, and each node's build capacity bounds what runs on it. + if len(holding) >= MaxTries || len(waiting) >= MaxTriesWaiting { + return "", fmt.Errorf("a new try is refused while %d tries run on a build node or %d wait for one; now %d run and %d wait: running %s; waiting %s; ask again when one ends", + MaxTries, MaxTriesWaiting, len(holding), len(waiting), orNone(holding), orNone(waiting)) } t.ID = "try-" + strconv.FormatInt(now.UnixNano(), 10) data, err := encodeTry(t) @@ -395,7 +501,8 @@ func (tr *Tryer) run(ctx context.Context, key string) { } return } - cs := &Case{Key: key, Group: key, + progress := &tryWriter{tr: tr, ctx: ctx, key: key, t: t, rev: rev} + cs := &Case{Key: key, Group: key, Watch: progress.heard, Pulls: []Pull{{Repository: t.Repository, Head: t.Head, Trunk: t.Trunk}}, Trunk: map[string]string{t.Repository: t.Trunk}, Queued: t.Asked, @@ -409,6 +516,7 @@ func (tr *Tryer) run(ctx context.Context, key string) { if ctx.Err() != nil { return // left running: the checker takes it over when it starts; or ended by hand, which wrote its end } + t, rev = progress.end() v := out.Verdict t.Ended = tr.now().UTC() t.Took = t.Ended.Sub(t.Started).Round(time.Second).String() @@ -470,3 +578,11 @@ func judgeWithin(ctx context.Context, j Judge, cs *Case, bound time.Duration) (O } return out, true } + +// orNone joins the tries said, or says none. +func orNone(said []string) string { + if len(said) == 0 { + return "none" + } + return strings.Join(said, ", ") +} diff --git a/modules/mesh-delivery-checker/internal/check/try_test.go b/modules/mesh-delivery-checker/internal/check/try_test.go index 27eb986..e617056 100644 --- a/modules/mesh-delivery-checker/internal/check/try_test.go +++ b/modules/mesh-delivery-checker/internal/check/try_test.go @@ -3,6 +3,7 @@ package check import ( "context" "encoding/json" + "fmt" "strings" "testing" "time" @@ -23,6 +24,19 @@ func (m memTries) Create(key string, value []byte) error { return err } +// writtenTries is a memKV that keeps every running try it was given to write, in order. +type writtenTries struct { + *memKV + running []Try +} + +func (w *writtenTries) Update(ctx context.Context, key string, value []byte, rev uint64) (uint64, error) { + if t, err := DecodeTry(value); err == nil && !t.Over() { + w.running = append(w.running, t) + } + return w.memKV.Update(ctx, key, value, rev) +} + var tryTrunks = map[string]string{repoA: trunkA, repoSDK: trunkS, hqRepo: trunkA} func readTry(t testing.TB, kv *memKV, id string) Try { @@ -54,14 +68,38 @@ func TestATryIsAskedQueuedOncePerHeadAndAtMostFourAtOnce(t *testing.T) { // ADR if err != nil || again != id { t.Errorf("a second ask of the same head answered %q, %v, want %q", again, err, id) } - heads := []string{head2, trunkA2, trunkS2} - for i, h := range heads { - if _, err := AskTry(store, tryTrunks, TryAsk{Repository: repoA, Head: h, Trunk: trunkA}, at.Add(time.Duration(i+2)*time.Second)); err != nil { - t.Fatalf("try %d: %v", i+2, err) - } + // Issue 1890: tries on a build node (a step of theirs running, or none named yet) and tries waiting for one (queued, or + // their asks not taken) are bounded apart, MaxTries and MaxTriesWaiting; the refusal names both counts. + onNode := func(i int) Try { + return Try{ID: fmt.Sprintf("try-%d", i), Repository: repoA, Head: fmt.Sprintf("%040x", i), Trunk: trunkA, State: TryRunning, + Steps: []TryStep{{Ask: "a", State: delivery.StepRunning}}} } - if _, err := AskTry(store, tryTrunks, TryAsk{Repository: repoSDK, Head: head1}, at.Add(time.Minute)); err == nil || !strings.Contains(err.Error(), "4 tries wait or run") { - t.Errorf("a fifth try answered %v, want refused naming the four", err) + waitingFor := func(i int) Try { + return Try{ID: fmt.Sprintf("try-%d", i), Repository: repoA, Head: fmt.Sprintf("%040x", i), Trunk: trunkA, State: TryRunning, + Steps: []TryStep{{Ask: "a", State: TryAsked}}} + } + for _, c := range []struct { + what string + holding, wait int + refused string + }{ + {"four queued behind the checks, none on a node: a fifth is taken", 0, 4, ""}, + {"four on a node", MaxTries, 0, "now 4 run and 0 wait"}, + {"three on a node, eight waiting", 3, MaxTriesWaiting, "a new try is refused while 4 tries run on a build node or 8 wait for one; now 3 run and 8 wait"}, + } { + kv := newMemKV() + for i := range c.holding + c.wait { + tr := waitingFor(i + 1) + if i < c.holding { + tr = onNode(i + 1) + } + data, _ := encodeTry(tr) + _, _ = kv.Create(context.Background(), tr.ID, data) + } + _, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoSDK, Head: head1}, at.Add(time.Minute)) + if (c.refused == "") != (err == nil) || (err != nil && !strings.Contains(err.Error(), c.refused)) { + t.Errorf("%s: answered %v, want refused %q", c.what, err, c.refused) + } } } @@ -89,7 +127,8 @@ func TestATryIsJudgedAsAPullRequestsHeadAndOwesTheForgeNothing(t *testing.T) { / t.Fatal(err) } b := newBench(t) - tr := &Tryer{Tries: kv, Judge: b.ex} + written := &writtenTries{memKV: kv} + tr := &Tryer{Tries: written, Judge: b.ex} raw, _, _, _ := kv.Get(ctx, id) tr.Put(ctx, id, raw) tr.Wait() @@ -97,6 +136,15 @@ func TestATryIsJudgedAsAPullRequestsHeadAndOwesTheForgeNothing(t *testing.T) { / if got.State != TryPass || got.Took == "" || got.Head != head1 || got.Trunk != trunkA || got.Said == "" || got.Levels[delivery.LevelUnit].Run != 2 { t.Fatalf("try = %+v", got) } + // Issue 1890: while it runs, the try names its ask as soon as it is put, and then where its step stands; the end + // keeps each ask's node and state. + if len(written.running) < 3 || len(written.running[1].Asks) != 1 || written.running[1].Steps[0].State != TryAsked || + written.running[2].Steps[0].Node != "shanks" { + t.Errorf("the running try's writes = %+v, want its ask named while it ran, then its node", written.running) + } + if len(got.Steps) != 1 || got.Steps[0].Node != "shanks" || got.Steps[0].State != delivery.StepPassed { + t.Errorf("the ended try's steps = %+v", got.Steps) + } // The try names the asks it put, so the delivery view counts their steps as a try's (build_usage). if len(got.Asks) != 1 { t.Errorf("asks = %v, want the one check ask the try put", got.Asks) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go index 9310a73..51fe90a 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go @@ -91,6 +91,28 @@ func treesVia(ctx context.Context, call caller) bridge.Trees { } } +// listedVia asks the forge for the files directly under one directory of a commit through call: gitea_commit_trees with +// listed (issue 1126), answer {listed: {directory: {name: blob id}}}. +func listedVia(ctx context.Context, call caller) bridge.Listed { + return func(owner, repo, sha, dir string) (map[string]string, error) { + raw, err := call(ctx, commitTreesTool, map[string]any{"owner": owner, "repo": repo, "sha": sha, "listed": []string{dir}}) + if err != nil { + return nil, err + } + var a struct { + Listed map[string]map[string]string `json:"listed"` + } + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("the answer of %s is not {listed}: %w", commitTreesTool, err) + } + if a.Listed == nil { + // A forge module from before listed answers none: placement cannot be read, never read as empty. + return nil, fmt.Errorf("the answer of %s lists no directory; the forge module does not list yet", commitTreesTool) + } + return a.Listed[dir], nil + } +} + // getFileTool is the forge's tool for one file at a ref (mesh-catalog modules/gitea tools/index.ts gitea_get_file): // arguments owner, repo, path and ref, answer {file: {path, ref, sha, size, content}}, the content decoded to text // (client.ts getFile). A path the forge does not have fails with `Gitea API : 404 ` (client.ts request). diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go index e069a6c..dbdf411 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go @@ -117,8 +117,8 @@ type publisher interface { // handle turns one event into its message on FORGE and acknowledges it. An event it cannot read is terminated, never // redelivered, and the log names its subject and why. A message it could not write stays unacknowledged: the server // delivers it again (the server's ack wait is the only clock). -func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, logger *log.Logger) { - msg, err := bridge.Convert(m.Subject(), m.Data(), trees, files) +func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, list bridge.Listed, logger *log.Logger) { + msg, err := bridge.ConvertWith(m.Subject(), m.Data(), trees, files, list) switch { case errors.Is(err, bridge.ErrRetry): logger.Printf("%v; heard again in %s", err, retryDelay) @@ -237,7 +237,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // Bounded: the turn is held, and no sync or other event goes on without it. hctx, cancel := context.WithTimeout(ctx, syncDeadline) defer cancel() - handle(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), logger) + handle(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), listedVia(hctx, call), logger) }) }, jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) })) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go index 9a16a64..85df582 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go @@ -81,7 +81,7 @@ func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit }) var logs bytes.Buffer m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{} - handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0)) + handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0)) if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull" { t.Fatalf("published on %v", pub.subjects) } @@ -97,7 +97,7 @@ func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5 var logs bytes.Buffer m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(map[string]any{"head_sha": "short"})}, &fakePub{} - handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0)) + handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0)) if len(pub.subjects) != 0 { t.Errorf("a message was written: %v", pub.subjects) } @@ -112,7 +112,7 @@ func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5 func TestUnwrittenMessageIsNotAcked(t *testing.T) { // 1235 (a failed write is redelivered by the server, not retried here) var logs bytes.Buffer m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{err: errors.New("no stream")} - handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0)) + handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0)) if m.acked != 0 || m.termd != 0 { t.Errorf("acked %d terminated %d, want neither", m.acked, m.termd) } @@ -134,7 +134,7 @@ func TestTreesRefusedLeavesTheEventForTenSecondsAndLogs(t *testing.T) { // 1236/ call := func(context.Context, string, any) (json.RawMessage, error) { return nil, errors.New("the tree listing of novox/mesh-delivery was cut") } - handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), log.New(&logs, "", 0)) + handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), nil, log.New(&logs, "", 0)) if len(pub.subjects) != 0 { t.Errorf("a message was written: %v", pub.subjects) } @@ -163,7 +163,7 @@ func TestMergeCallsTheToolOnceAndWritesTheMergedMessage(t *testing.T) { // 1236/ } return json.RawMessage(`{"root":"r0","trees":{"":"t0","modules/a":"t1"}}`), nil } - handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), log.New(&logs, "", 0)) + handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), nil, log.New(&logs, "", 0)) if len(keys) == 0 || keys[0] != "gitea.gitea_commit_trees" || slices.Index(keys[1:], "gitea.gitea_commit_trees") >= 0 { t.Fatalf("calls %v, want the trees once and first", keys) } diff --git a/modules/mesh-delivery-forge-bridge/go.mod b/modules/mesh-delivery-forge-bridge/go.mod index 6f902ec..6d3738c 100644 --- a/modules/mesh-delivery-forge-bridge/go.mod +++ b/modules/mesh-delivery-forge-bridge/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-forge-bridge/go.sum b/modules/mesh-delivery-forge-bridge/go.sum index bdc8b83..e8660dd 100644 --- a/modules/mesh-delivery-forge-bridge/go.sum +++ b/modules/mesh-delivery-forge-bridge/go.sum @@ -1,7 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117 h1:88CNZOoQgu7YPYKW16qkWT8Gw9Ucg07aTrIImA2nYUk= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go index a6cbc7d..e65f8fd 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "fmt" + "maps" "path" "regexp" "slices" @@ -88,6 +89,17 @@ func unreadable(subject, why string, args ...any) error { // the trunk asks trees once and becomes a merged message; a merge into any other branch is a closed pull message, as the // pull request left its group. A forge that cannot say is ErrRetry; a merge with no module.json is ErrNoModules. func Convert(subject string, data []byte, trees Trees, files Files) (Message, error) { + return ConvertWith(subject, data, trees, files, nil) +} + +// Listed asks the forge for the files directly under one directory of a commit, by name to blob id (gitea_commit_trees +// with listed); a directory the commit lacks lists none. +type Listed func(owner, repo, sha, dir string) (map[string]string, error) + +// ConvertWith is Convert; with list, a merge of the catalogue repository (delivery.CatalogueRepository) also carries every +// placement file of its merge commit (novox/hq ADR 0549 §2, issue 1126), read whole or not at all: a forge that cannot +// say is ErrRetry, as for the manifests, so the indexer never takes a part of placement for all of it. +func ConvertWith(subject string, data []byte, trees Trees, files Files, list Listed) (Message, error) { var action string switch subject { case UpdatedSubject: @@ -114,7 +126,7 @@ func Convert(subject string, data []byte, trees Trees, files Files) (Message, er return Message{}, unreadable(subject, "no branch") } if subject == MergedSubject && p.Base == Trunk { - return merged(subject, p, trees, files) + return merged(subject, p, trees, files, list) } f := delivery.Forge{Kind: delivery.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number, Head: p.HeadSHA, Action: action} @@ -227,7 +239,7 @@ func inputsOf(owner, repo, ref string, dirs []string, files Files) (manifests ma } // merged is the merged message of a merge into the trunk: the trees of the merge commit, asked once. -func merged(subject string, p pull, trees Trees, files Files) (Message, error) { +func merged(subject string, p pull, trees Trees, files Files, list Listed) (Message, error) { if !sha40.MatchString(p.MergeSHA) { return Message{}, unreadable(subject, "the merge commit %q is not 40 hex digits", p.MergeSHA) } @@ -252,8 +264,15 @@ func merged(subject string, p pull, trees Trees, files Files) (Message, error) { if err != nil { return Message{}, unreadable(subject, "%v", err) } - body, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number, - Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs, Manifests: manifests, Locks: locks}) + f := delivery.Forge{Kind: delivery.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number, + Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs, Manifests: manifests, Locks: locks} + if list != nil && delivery.SameRepository(repository, delivery.CatalogueRepository) { + if f.Placements, err = placementsOf(p.Owner, p.Repo, p.MergeSHA, list, files); err != nil { + return Message{}, fmt.Errorf("%w: %s: the placement files of %s at the merge commit %s: %v", ErrRetry, subject, repository, p.MergeSHA, err) + } + f.PlacementsRead = true + } + body, err := delivery.EncodeForge(f) if err != nil { return Message{}, unreadable(subject, "%v", err) } @@ -350,3 +369,30 @@ func OpenMessage(answer []byte) (Message, error) { } return Message{Event: delivery.OpenEvent(), Data: body}, nil } + +// placementsOf is every placement file of the catalogue at a commit, by node, its bytes exact: each `.json` directly +// under delivery.PlacementDir. A file whose name is no node is left out and the rest read; the catalogue's check refuses +// such a file before it merges (ADR 0549 §1). Nil when there is none. +func placementsOf(owner, repo, sha string, list Listed, files Files) (map[string]string, error) { + names, err := list(owner, repo, sha, delivery.PlacementDir) + if err != nil { + return nil, err + } + var out map[string]string + for _, name := range slices.Sorted(maps.Keys(names)) { + file := delivery.PlacementDir + "/" + name + node, ok := delivery.PlacementFileNode(file) + if !ok { + continue + } + raw, err := files(owner, repo, file, sha) + if err != nil { + return nil, fmt.Errorf("%s: %w", file, err) + } + if out == nil { + out = map[string]string{} + } + out[node] = string(raw) + } + return out, nil +} diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/files_test.go b/modules/mesh-delivery-forge-bridge/internal/bridge/files_test.go index a5617ec..d4afe54 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/files_test.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/files_test.go @@ -79,6 +79,47 @@ func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1 if want := map[string]map[string]string{"modules/a": {"go.sum": hexSum(sum)}}; !reflect.DeepEqual(f.Locks, want) { t.Errorf("locks = %v, want %v", f.Locks, want) } + if f.Placements != nil || f.PlacementsRead { + t.Errorf("a merge of another repository than the catalogue carries placement: %v", f.Placements) + } + + // Issue 1126 (ADR 0549 §2): a merge of the catalogue carries every placement file of its merge commit, by node; a file + // whose name is no node is left out; a listing or a file the forge cannot give is a retry, never a part. + catalogue := mergedBody(map[string]any{"repo": "mesh-catalog"}) + forge.have["placement/ace.json"] = []byte(`{"v":1,"node":"ace","modules":[]}`) + forge.have["placement/g14.json"] = []byte(`{"v":1,"node":"g14","modules":[{"module":"zsh"}]}`) + listing := map[string]string{"ace.json": "b1", "g14.json": "b2", "README.md": "b3", "a.b.json": "b4"} + var listedAt []string + list := func(owner, repo, sha, dir string) (map[string]string, error) { + listedAt = append(listedAt, owner+"/"+repo+"@"+sha+":"+dir) + return listing, nil + } + msg, err = ConvertWith(mergedSubject, catalogue, treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get, list) + if err != nil { + t.Fatal(err) + } + if f, err = delivery.DecodeForge(msg.Data); err != nil { + t.Fatal(err) + } + want := map[string]string{"ace": `{"v":1,"node":"ace","modules":[]}`, "g14": `{"v":1,"node":"g14","modules":[{"module":"zsh"}]}`} + if !f.PlacementsRead || !reflect.DeepEqual(f.Placements, want) || !reflect.DeepEqual(listedAt, []string{"novox/mesh-catalog@" + merge + ":placement"}) { + t.Errorf("placements = %v (read %v), listed at %v", f.Placements, f.PlacementsRead, listedAt) + } + listing = map[string]string{} + if msg, err = ConvertWith(mergedSubject, catalogue, treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get, list); err != nil { + t.Fatal(err) + } + if f, _ = delivery.DecodeForge(msg.Data); !f.PlacementsRead || f.Placements != nil { + t.Errorf("a catalogue with no placement file says it read none: %v, %v", f.Placements, f.PlacementsRead) + } + listing = map[string]string{"gone.json": "b5"} + if _, err := ConvertWith(mergedSubject, catalogue, treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get, list); !errors.Is(err, ErrRetry) || !strings.Contains(err.Error(), "placement/gone.json") { + t.Errorf("a placement file the forge cannot give: %v", err) + } + failing := func(_, _, _, _ string) (map[string]string, error) { return nil, errors.New("forge down") } + if _, err := ConvertWith(mergedSubject, catalogue, treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get, failing); !errors.Is(err, ErrRetry) { + t.Errorf("a listing the forge cannot give: %v", err) + } } func TestPullWithKnownDirsCarriesTheSameAtTheHead(t *testing.T) { // 1448/AC2 diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 0b4258d..4c2a403 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -91,6 +91,58 @@ func pinned(want string) *tls.Config { } } +// awaitedStore is a state bound when it appears (process.AwaitBucket) as the indexer's store: until then every call is +// process.ErrNotYet. +type awaitedStore struct{ b *process.AwaitedBucket } + +func (s awaitedStore) store() (kvStore, error) { + kv := s.b.KV() + if kv == nil { + return kvStore{}, process.ErrNotYet + } + return kvStore{kv}, nil +} + +func (s awaitedStore) Get(ctx context.Context, key string) ([]byte, uint64, bool, error) { + st, err := s.store() + if err != nil { + return nil, 0, false, err + } + return st.Get(ctx, key) +} + +func (s awaitedStore) Create(ctx context.Context, key string, value []byte) error { + st, err := s.store() + if err != nil { + return err + } + return st.Create(ctx, key, value) +} + +func (s awaitedStore) Update(ctx context.Context, key string, value []byte, revision uint64) error { + st, err := s.store() + if err != nil { + return err + } + return st.Update(ctx, key, value, revision) +} + +func (s awaitedStore) Delete(ctx context.Context, key string, revision uint64) error { + st, err := s.store() + if err != nil { + return err + } + return st.Delete(ctx, key, revision) +} + +func (s awaitedStore) Keys(ctx context.Context) ([]string, error) { + st, err := s.store() + if err != nil { + return nil, err + } + return st.Keys(ctx) +} + // kvStore is the candidates bucket as the indexer's store: compare-and-set through Create, Update and Delete at a revision. type kvStore struct{ kv jetstream.KeyValue } @@ -140,6 +192,7 @@ type delivered interface { Subject() string Data() []byte Ack() error + DoubleAck(ctx context.Context) error Term() error } @@ -160,7 +213,12 @@ func handle(ctx context.Context, m delivered, ix *index.Indexer, logger *log.Log case err != nil: logger.Printf("%s: %s#%d left for redelivery: %v", m.Subject(), f.Repository, f.Number, err) default: - _ = m.Ack() + // Waits for the server, so the consumer's info no longer counts the message as unacknowledged when the fill that + // follows asks whether the stream is applied (issue 1280); a failed ack is redelivered after the ack wait, which + // kicks the fill again. + if err := m.DoubleAck(ctx); err != nil { + logger.Printf("%s: %s#%d applied, its acknowledgement was not confirmed: %v", m.Subject(), f.Repository, f.Number, err) + } } } @@ -253,6 +311,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } + // Its taken state (ADR 0556 clause 1) is bound when it appears, as every state a delivery adds (issue 1790): until then + // a merge's taken key is said and not written, and the rest of the indexer runs. + taken := process.AwaitBucket(ctx, js, conn, delivery.TakenBucket, "no repository's taken key is written", logger.Printf) + // Its placement state (ADR 0549 §2) is bound when it appears, as every state a delivery adds (issue 1790): until then a + // catalogue merge's placement is said and not written, and the rest of the indexer runs. + placement := process.AwaitBucket(ctx, js, conn, delivery.PlacementBucket, "no catalogue merge's placement is written", logger.Printf) // The controller's build records only name the directory of a module whose key names none (fillPaths): without them // such a module waits for its path, every other is indexed (issue 1945). builds := process.OptionalBucket(ctx, js, delivery.BuildsBucket, "a module whose key names no directory is not given one from the controller's build records until the indexer starts again", logger.Printf) @@ -286,8 +350,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } defer calls.Close() - ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Builds: known, Logf: logger.Printf, - Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call)} + ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known, Logf: logger.Printf, + Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call), + Placement: awaitedStore{placement}, Say: func(event string, body any) error { return calls.Emit(event, body) }} // The gate's source bucket (mesh-delivery-gate_source) comes with the gate (release 3e, novox/hq ADR 0450 §1): its key // main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does // not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate. @@ -409,7 +474,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // Essential: the forge's merges are what the indexer indexes. return fmt.Errorf("the consumer %s_%s on %s cannot be read (%v): the indexer hears no merge without it, so it stops; where the bus refused it or did not answer, this module's consumes (made by the controller with its grant) have not reached the bus yet", cred.Node, cred.Module, forgeStream, err) } - cc, err := cons.Consume(func(m jetstream.Msg) { handle(ctx, m, ix, logger) }, + // A merge applied while the taken state was missing, and the gate's heads once the forge stream is applied whole, get + // their taken key (issue 1280): when the state appears (Ready runs once, now or then), and after each applied message. + ix.Drained = func(ctx context.Context) (bool, error) { + info, err := cons.Info(ctx) + if err != nil { + return false, err + } + return info.NumPending == 0 && info.NumAckPending == 0, nil + } + taken.Ready(func(jetstream.KeyValue) { ix.KickFill(ctx) }) + cc, err := cons.Consume(func(m jetstream.Msg) { handle(ctx, m, ix, logger); ix.KickFill(ctx) }, jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) })) if err != nil { return err diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go index 19b3658..e49c636 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go @@ -16,12 +16,18 @@ type fakeMsg struct { subject string data []byte acked, termd int + doubled int // acked by DoubleAck, which waits for the server } func (f *fakeMsg) Subject() string { return f.subject } func (f *fakeMsg) Data() []byte { return f.data } func (f *fakeMsg) Ack() error { f.acked++; return nil } -func (f *fakeMsg) Term() error { f.termd++; return nil } +func (f *fakeMsg) DoubleAck(context.Context) error { + f.acked++ + f.doubled++ + return nil +} +func (f *fakeMsg) Term() error { f.termd++; return nil } type memStore struct{ vals map[string][]byte } @@ -53,7 +59,7 @@ func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part) m := &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull", data: data} handle(context.Background(), m, ix, log.New(&bytes.Buffer{}, "", 0)) c, err := delivery.DecodeCandidate(st.vals["feat/x"]) - if err != nil || len(c.Pulls) != 1 || c.Pulls[0].Modules[0] != "a" || m.acked != 1 || m.termd != 0 { + if err != nil || len(c.Pulls) != 1 || c.Pulls[0].Modules[0] != "a" || m.acked != 1 || m.doubled != 1 || m.termd != 0 { t.Errorf("candidate %+v err %v acked %d terminated %d", c, err, m.acked, m.termd) } } diff --git a/modules/mesh-delivery-indexer/go.mod b/modules/mesh-delivery-indexer/go.mod index 5a8d354..285279f 100644 --- a/modules/mesh-delivery-indexer/go.mod +++ b/modules/mesh-delivery-indexer/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-indexer go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142434-65ee43f07583 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026211913-68d65e6bbf61 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-indexer/go.sum b/modules/mesh-delivery-indexer/go.sum index e793069..b3dab4f 100644 --- a/modules/mesh-delivery-indexer/go.sum +++ b/modules/mesh-delivery-indexer/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142434-65ee43f07583 h1:ikD/ggf9/5k23sxJOX1PFcGzb4M3+QeC66AT0MyQ9Zs= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142434-65ee43f07583/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026211913-68d65e6bbf61 h1:/c6ncICWjq55nBf8TlDM0ROe/PuH+qZIdnuVbtg8UuM= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026211913-68d65e6bbf61/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 592b309..80d9abf 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -9,6 +9,7 @@ import ( "strings" "sync" "sync/atomic" + "time" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -47,7 +48,11 @@ type Indexer struct { Modules Store // Manifests is the manifests bucket; nil writes no manifests (issue 1449). Manifests Store - Builds *Builds + // Taken is the taken bucket: per repository the last merge taken in (taken.go, ADR 0556); nil writes none. + Taken Store + // Now is the clock of a taken value; nil is time.Now. + Now func() time.Time + Builds *Builds // Named asks the forge for the git object of each named path of the module directories given, at a commit // (gitea.gitea_commit_trees with paths; novox/hq ADR 0499 clauses 2 and 3): the indexer is the one reader of a // manifest's paths (ADR 0495 clause 1, the lead's decision L8 of R4e). Nil asks nothing, and a module naming paths @@ -60,6 +65,13 @@ type Indexer struct { // Contains says whether a commit is the second or an ancestor of it (gitea.gitea_contains). Contains func(ctx context.Context, repository, commit, in string) (bool, error) Logf func(format string, args ...any) + // Placement is the placement bucket (delivery.PlacementBucket, ADR 0549 §2); nil writes no placement. + Placement Store + // Say says one of the indexer's events (placement-changed); nil says none. + Say func(event string, body any) error + // Drained says whether the indexer has applied every message of the forge stream (the consumer has none pending and none + // unacknowledged); FillTaken writes the source's heads only then. Nil fills none from the source (taken.go). + Drained func(ctx context.Context) (bool, error) mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach source sourceNow @@ -71,6 +83,9 @@ type Indexer struct { retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on recompute sync.Mutex // one Recompute at a time + untaken map[string]string // repository to the latest merge applied while the taken state was missing (taken.go) + filling atomic.Bool // a FillTaken runs in the background (KickFill) + fillAgain atomic.Bool // a kick arrived while it ran } func (ix *Indexer) logf(format string, args ...any) { @@ -94,8 +109,14 @@ func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error { if err := ix.applyMerged(ctx, f); err != nil { return err } + if err := ix.writeTaken(ctx, f); err != nil { + return err + } ix.kickRetry(ctx) // a registered module the forge could not read is read again on a merge, in the background (issue 1945) } + if err := ix.applyPlacement(ctx, f); err != nil { + return err + } var digests map[string]string var held map[string]delivery.Module if f.Kind == delivery.ForgePull { diff --git a/modules/mesh-delivery-indexer/internal/index/index_test.go b/modules/mesh-delivery-indexer/internal/index/index_test.go index 74bcc57..3dd794d 100644 --- a/modules/mesh-delivery-indexer/internal/index/index_test.go +++ b/modules/mesh-delivery-indexer/internal/index/index_test.go @@ -17,8 +17,9 @@ const ( // memStore is the candidates bucket's compare-and-set behaviour in memory: a revision per write, a conflict when the // revision read is no longer the key's. Writes counts the writes that went through; beforeWrite lets a test change the -// key under the indexer. +// key under the indexer; unbound, when set, is every read's error (a state not yet on the bus). type memStore struct { + unbound error vals map[string][]byte revs map[string]uint64 next uint64 @@ -29,6 +30,9 @@ type memStore struct { func newMem() *memStore { return &memStore{vals: map[string][]byte{}, revs: map[string]uint64{}} } func (m *memStore) Get(_ context.Context, key string) ([]byte, uint64, bool, error) { + if m.unbound != nil { + return nil, 0, false, m.unbound + } v, ok := m.vals[key] return v, m.revs[key], ok, nil } diff --git a/modules/mesh-delivery-indexer/internal/index/manifests_test.go b/modules/mesh-delivery-indexer/internal/index/manifests_test.go index 08455bb..3cdd5f3 100644 --- a/modules/mesh-delivery-indexer/internal/index/manifests_test.go +++ b/modules/mesh-delivery-indexer/internal/index/manifests_test.go @@ -43,6 +43,8 @@ func withManifests(f delivery.Forge, manifests map[string]string, locks map[stri } func TestAMergedManifestIsKeptByItsDigestAndNamedByTheModule(t *testing.T) { // 1449/AC1 + // A merge's placement files are written as its manifests are (issue 1126): folded here to hold the count. + placementIsWrittenAtAMergeOnlyWhereItChanged(t) avahi := catalogueManifest(t, "avahi") ix, mods, manifests := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs)) f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, nil) diff --git a/modules/mesh-delivery-indexer/internal/index/modules_test.go b/modules/mesh-delivery-indexer/internal/index/modules_test.go index 18987ef..8369a2e 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules_test.go +++ b/modules/mesh-delivery-indexer/internal/index/modules_test.go @@ -8,6 +8,7 @@ import ( "testing" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/process" ) const merge = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" @@ -94,22 +95,92 @@ func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value // writes is the values written by then; commit is each module's commit after. writes int commit map[string]string + // taken is the merge the repository's taken key names after then (issue 1280, ADR 0556 clause 1). + taken string }{ - {name: "the same merge again", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, + {name: "the same merge again", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: merge, then: func(ix *Indexer) error { return ix.Apply(context.Background(), mergedMsg(repo, three)) }}, - {name: "a merge touching one of three modules", writes: 1, commit: map[string]string{"a": next, "b": merge, "c": merge}, + {name: "a merge touching one of three modules", writes: 1, commit: map[string]string{"a": next, "b": merge, "c": merge}, taken: next, then: func(ix *Indexer) error { m := mergedMsg(repo, map[string]string{"modules/a": "treeA2", "modules/b": "treeB", "modules/c": "treeC"}) m.Merge = next return ix.Apply(context.Background(), m) }}, - {name: "a later source revision of another repository's merge", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, + {name: "a merge touching none of them is still taken in", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: next, + then: func(ix *Indexer) error { + m := mergedMsg(repo, three) + m.Merge = next + return ix.Apply(context.Background(), m) + }}, + {name: "a merge while the taken state is not on the bus yet is still taken in", writes: 1, commit: map[string]string{"a": next, "b": merge, "c": merge}, taken: merge, + then: func(ix *Indexer) error { // issue 1790: the state is bound when it appears, so the indexer runs before it does + ix.Taken = &memStore{unbound: process.ErrNotYet} + m := mergedMsg(repo, map[string]string{"modules/a": "treeA2", "modules/b": "treeB", "modules/c": "treeC"}) + m.Merge = next + return ix.Apply(context.Background(), m) + }}, + {name: "a merge heard while the taken state was missing is filled when the state appears", writes: 2, commit: map[string]string{"a": next, "b": merge, "c": merge}, taken: next, + then: func(ix *Indexer) error { // issue 1280: the key was missing, the merge was taken in without it + on := ix.Taken.(*memStore) + key, _ := delivery.RepositoryKey(repo) + delete(on.vals, key) + ix.Taken = &memStore{unbound: process.ErrNotYet} + m := mergedMsg(repo, map[string]string{"modules/a": "treeA2", "modules/b": "treeB", "modules/c": "treeC"}) + m.Merge = next + if err := ix.Apply(context.Background(), m); err != nil { + return err + } + if err := ix.ApplySource(context.Background(), 8, sourceOf(repo, next)); err != nil { + return err + } + ix.Taken = on + return ix.FillTaken(context.Background()) // no Drained: only the merge applied is written, not the source's word + }}, + {name: "a source head whose merge was not applied is not a taken key", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: "", + then: func(ix *Indexer) error { // the forge stream still holds the merge: the consumer is not drained + delete(ix.Taken.(*memStore).vals, takenKey(repo)) + ix.Drained = func(context.Context) (bool, error) { return false, nil } + if err := ix.ApplySource(context.Background(), 8, sourceOf(repo, next)); err != nil { + return err + } + return ix.FillTaken(context.Background()) + }}, + {name: "a source head of a drained forge stream fills a missing key", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: next, + then: func(ix *Indexer) error { + delete(ix.Taken.(*memStore).vals, takenKey(repo)) + ix.Drained = func(context.Context) (bool, error) { return true, nil } + if err := ix.ApplySource(context.Background(), 8, sourceOf(repo, next)); err != nil { + return err + } + return ix.FillTaken(context.Background()) + }}, + {name: "filling from an older source head leaves a later key", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: next, + then: func(ix *Indexer) error { + ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { return commit == merge && in == next, nil } + m := mergedMsg(repo, three) + m.Merge = next + if err := ix.Apply(context.Background(), m); err != nil { + return err + } + ix.Drained = func(context.Context) (bool, error) { return true, nil } + return ix.FillTaken(context.Background()) // the source still names merge + }}, + {name: "an earlier merge heard again after a later one leaves taken", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: merge, + then: func(ix *Indexer) error { + ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { return commit == next && in == merge, nil } + m := mergedMsg(repo, three) + m.Merge = next + return ix.Apply(context.Background(), m) + }}, + {name: "a later source revision of another repository's merge", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: merge, then: func(ix *Indexer) error { return ix.ApplySource(context.Background(), 9, delivery.Source{Group: "g", Heads: map[string]string{repo: merge, "novox/mesh-catalog": otherMerge}}) }}, } { t.Run(tc.name, func(t *testing.T) { ix, mods := modulesIndexer(t) + taken := newMem() + ix.Taken = taken ctx := context.Background() if err := ix.Apply(ctx, mergedMsg(repo, three)); err != nil { t.Fatal(err) @@ -129,6 +200,13 @@ func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value t.Errorf("%s = %+v (%v), want commit %.8s", module, m, ok, want) } } + key, _ := delivery.RepositoryKey(repo) + if _, held := taken.vals[key]; !held && tc.taken == "" { + return + } + if got, err := delivery.DecodeTaken(taken.vals[key]); err != nil || got.Merge != tc.taken || got.Repository != repo { + t.Errorf("taken.%s = %+v (%v), want merge %.8s", key, got, err, tc.taken) + } }) } } @@ -252,3 +330,8 @@ func TestTwoDirectoriesNamingOneModuleWriteOneAndSayTheOther(t *testing.T) { // t.Errorf("module = %+v, want the held directory kept", m) } } + +func takenKey(repo string) string { + key, _ := delivery.RepositoryKey(repo) + return key +} diff --git a/modules/mesh-delivery-indexer/internal/index/placement.go b/modules/mesh-delivery-indexer/internal/index/placement.go new file mode 100644 index 0000000..d611e68 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/placement.go @@ -0,0 +1,123 @@ +package index + +import ( + "context" + "errors" + "fmt" + "maps" + "slices" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/process" +) + +// Placement of modules on nodes (novox/hq ADR 0549 §2, issue 1126): at each merge of the catalogue the forge bridge +// carries every placement file of the merge commit, and the indexer writes each node's key `placement.` in its +// state `placement` only when it differs (the revision alone is no change), deletes the key of a node whose file is +// gone, and says `placement-changed` for each node it wrote or deleted, naming the modules added, removed or changed. +// A merge that changes no placement writes nothing and says nothing. +// +// A file that does not read as a placement, or names another node than its file, is said in the log and its key is left +// as it was: the catalogue's own check refuses such a file before it merges (ADR 0549 §1), so it is a fault of that +// check, and a node keeps composing from the placement it had rather than from none. + +// applyPlacement takes the placement files of a merged message in; a message that read none (another repository, an +// older bridge) changes nothing. +func (ix *Indexer) applyPlacement(ctx context.Context, f delivery.Forge) error { + if ix.Placement == nil || f.Kind != delivery.ForgeMerged || !f.PlacementsRead { + return nil + } + keys, err := ix.Placement.Keys(ctx) + if errors.Is(err, process.ErrNotYet) { + // The state is not on the bus yet: said, and the merge's candidates are still taken out (issue 1790's binding + // writes the next catalogue merge's placement once it appears). + ix.logf("%s at %s: its placement files are not written: %v", f.Repository, f.Merge, err) + return nil + } + if err != nil { + return fmt.Errorf("reading the placement keys: %w", err) + } + gone := map[string]bool{} + for _, key := range keys { + if node, err := delivery.ParsePlacementKey(key); err == nil { + gone[node] = true + } + } + var refused []string + for _, node := range slices.Sorted(maps.Keys(f.Placements)) { + delete(gone, node) + next, err := delivery.DecodePlacement([]byte(f.Placements[node])) + if err == nil && next.Node != node { + err = fmt.Errorf("the file of %s names the node %q", node, next.Node) + } + if err != nil { + refused = append(refused, node) + ix.logf("%s at %s: the placement file of %s is refused and placement.%s is left as it was (the catalogue's check should have refused it): %v", + f.Repository, f.Merge, node, node, err) + continue + } + next.Revision = f.Merge + if err := ix.writePlacement(ctx, node, &next); err != nil { + return err + } + } + for _, node := range slices.Sorted(maps.Keys(gone)) { + if err := ix.writePlacement(ctx, node, nil); err != nil { + return err + } + } + if len(refused) > 0 { + // Said, and the merge's candidates are still taken out: a refused file holds nobody's delivery. + ix.logf("%s at %s: the placement files of %v do not read; their keys are left as they were", f.Repository, f.Merge, refused) + } + return nil +} + +// writePlacement writes one node's key, or deletes it for next nil, only when it differs, and says what changed. +func (ix *Indexer) writePlacement(ctx context.Context, node string, next *delivery.Placement) error { + key, err := delivery.PlacementKey(node) + if err != nil { + return err + } + var change *delivery.PlacementChange + err = write(ctx, ix.Placement, key, func(value []byte, found bool) ([]byte, bool, error) { + change = nil + var before delivery.Placement + if found { + cur, err := delivery.DecodePlacement(value) + if err != nil { + ix.logf("placement.%s holds what is not a placement, written again: %v", node, err) + } else { + before = cur + } + } + if next == nil { + if !found { + return nil, false, nil + } + c := delivery.ChangeOf(before, delivery.Placement{Node: node}) + change = &c + return nil, true, nil + } + if found && delivery.SamePlacement(before, *next) { + return nil, false, nil + } + data, err := delivery.EncodePlacement(*next) + if err != nil { + return nil, false, err + } + c := delivery.ChangeOf(before, *next) + change = &c + return data, false, nil + }) + if err != nil || change == nil { + return err + } + if ix.Say != nil { + if err := ix.Say(delivery.PlacementChangedEvent, *change); err != nil { + // The key is written: a reader that watches it hears the change; the event is said loudly as not said. + ix.logf("placement.%s is written, and %s could not be said: %v", node, delivery.PlacementChangedEvent, err) + } + } + return nil +} diff --git a/modules/mesh-delivery-indexer/internal/index/placement_test.go b/modules/mesh-delivery-indexer/internal/index/placement_test.go new file mode 100644 index 0000000..adb48f1 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/placement_test.go @@ -0,0 +1,97 @@ +package index + +import ( + "context" + "reflect" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// ADR 0549 §2 (issue 1126): a merge changing one node's file writes that key alone and says one event; a merge with no +// placement change writes nothing and says nothing; a node whose file is gone loses its key and says its modules +// removed; a file that does not read leaves its key; a message that read no placement changes none. +func placementIsWrittenAtAMergeOnlyWhereItChanged(t *testing.T) { + ctx := context.Background() + ix, _, logs := newIndexer(t) + place := newMem() + ix.Placement = place + var said []delivery.PlacementChange + ix.Say = func(event string, body any) error { + if event != delivery.PlacementChangedEvent { + t.Errorf("said %s", event) + } + said = append(said, body.(delivery.PlacementChange)) + return nil + } + merged := func(merge string, files map[string]string) delivery.Forge { + return delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: delivery.CatalogueRepository, Branch: "main", + Number: 7, Head: "h", Merge: merge, Root: "r", Trees: map[string]string{"": "t"}, Placements: files, PlacementsRead: true} + } + ace := `{"v":1,"node":"ace","modules":[{"module":"plex"}]}` + g14 := `{"v":1,"node":"g14","modules":[{"module":"zsh"}]}` + written := func(node string) (delivery.Placement, uint64) { + t.Helper() + v, ok := place.vals["placement."+node] + if !ok { + return delivery.Placement{}, 0 + } + p, err := delivery.DecodePlacement(v) + if err != nil { + t.Fatal(err) + } + return p, place.revs["placement."+node] + } + + if err := ix.Apply(ctx, merged("m1", map[string]string{"ace": ace, "g14": g14})); err != nil { + t.Fatal(err) + } + if p, _ := written("ace"); p.Revision != "m1" || !reflect.DeepEqual(p.Assigned(), []string{"plex"}) || len(said) != 2 { + t.Fatalf("first merge: %+v, said %v", p, said) + } + _, aceRev := written("ace") + + said = nil + if err := ix.Apply(ctx, merged("m2", map[string]string{"ace": ace, "g14": g14})); err != nil { + t.Fatal(err) + } + if p, rev := written("ace"); rev != aceRev || p.Revision != "m1" || len(said) != 0 { + t.Errorf("a merge with no placement change wrote %+v (revision %d, was %d) and said %v", p, rev, aceRev, said) + } + + said = nil + if err := ix.Apply(ctx, merged("m3", map[string]string{"ace": ace, "g14": `{"v":1,"node":"g14","modules":[{"module":"zsh"},{"module":"xterm"}]}`})); err != nil { + t.Fatal(err) + } + if _, rev := written("ace"); rev != aceRev || len(said) != 1 || said[0].Node != "g14" || said[0].Revision != "m3" || + !reflect.DeepEqual(said[0].Added, []string{"xterm"}) { + t.Errorf("one file changed: ace's revision %d (was %d), said %+v", rev, aceRev, said) + } + + said = nil + if err := ix.Apply(ctx, merged("m4", map[string]string{"ace": ace, "g14": `{"v":1,"node":"ace","modules":[]}`})); err != nil { + t.Fatal(err) + } + if p, _ := written("g14"); len(p.Modules) != 2 || len(said) != 0 || !strings.Contains(logs.String(), "the placement file of %s is refused") { + t.Errorf("a file naming another node replaced the key: %+v, said %v, logs %s", p, said, logs) + } + + said = nil + if err := ix.Apply(ctx, merged("m5", map[string]string{"ace": ace})); err != nil { + t.Fatal(err) + } + if _, rev := written("g14"); rev != 0 || len(said) != 1 || !reflect.DeepEqual(said[0].Removed, []string{"xterm", "zsh"}) { + t.Errorf("a file removed: g14's key at revision %d, said %+v", rev, said) + } + + said = nil + unread := merged("m6", nil) + unread.PlacementsRead = false + if err := ix.Apply(ctx, unread); err != nil { + t.Fatal(err) + } + if p, _ := written("ace"); p.Revision != "m1" || len(said) != 0 { + t.Errorf("a message that read no placement changed it: %+v, %v", p, said) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/previews_test.go b/modules/mesh-delivery-indexer/internal/index/previews_test.go index 57e7b90..5c575a8 100644 --- a/modules/mesh-delivery-indexer/internal/index/previews_test.go +++ b/modules/mesh-delivery-indexer/internal/index/previews_test.go @@ -164,6 +164,7 @@ func TestAPreviewHeardBeforeItsPullRequestGivesTheFingerprintWhenTheCandidateIsW } func TestAPreviewNamingNoManifestOfTheModulesDirectoryGivesAReason(t *testing.T) { + t.Run("the same preview again", testTheSamePreviewAppliedAgainWritesNothing) ix, _, cands := previewBed(t, true) if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil { t.Fatal(err) @@ -176,7 +177,7 @@ func TestAPreviewNamingNoManifestOfTheModulesDirectoryGivesAReason(t *testing.T) } } -func TestTheSamePreviewAppliedAgainWritesNothing(t *testing.T) { +func testTheSamePreviewAppliedAgainWritesNothing(t *testing.T) { ix, _, cands := previewBed(t, true) if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil { t.Fatal(err) diff --git a/modules/mesh-delivery-indexer/internal/index/taken.go b/modules/mesh-delivery-indexer/internal/index/taken.go new file mode 100644 index 0000000..e8082a0 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/taken.go @@ -0,0 +1,152 @@ +package index + +import ( + "context" + "errors" + "fmt" + "maps" + "slices" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/process" +) + +// What the indexer took in (novox/hq ADR 0556 clause 1, issue 1280): after a merged message is applied whole, the key of +// its repository in the state `taken` names that merge, so a reader of the modules knows they say the repository at that +// merge or later. The indexer is the key's only writer. A merge heard again after a later one (a redelivery) does not put +// the key back: the key moves only to a merge that contains the one it names, as the forge says it; a forge that cannot +// say keeps the key as it is and says so. + +// writeTaken writes the repository's taken key for the merge f carries, unless it names that merge or a later one. +func (ix *Indexer) writeTaken(ctx context.Context, f delivery.Forge) error { + if ix.Taken == nil || f.Merge == "" { + return nil + } + key, err := delivery.RepositoryKey(f.Repository) + if err != nil { + ix.logf("%s at %s: no taken key is written: %v", f.Repository, f.Merge, err) + return nil + } + err = write(ctx, ix.Taken, key, func(value []byte, found bool) ([]byte, bool, error) { + if found { + if cur, err := delivery.DecodeTaken(value); err == nil { + if cur.Merge == f.Merge { + return nil, false, nil + } + if ix.Contains != nil { + later, err := ix.Contains(ctx, f.Repository, f.Merge, cur.Merge) + if err != nil { + ix.logf("%s: whether %s is in %s cannot be read, so taken.%s stays at %s: %v", f.Repository, f.Merge, cur.Merge, key, cur.Merge, err) + return nil, false, nil + } + if later { + return nil, false, nil // a merge heard again after a later one + } + } + } + } + now := time.Now + if ix.Now != nil { + now = ix.Now + } + data, err := delivery.EncodeTaken(delivery.Taken{Repository: delivery.RepositoryName(f.Repository), Merge: f.Merge, At: now().UTC()}) + return data, false, err + }) + if errors.Is(err, process.ErrNotYet) { + // The state is not on the bus yet (a delivery that adds it, bound when it appears as issue 1790 says): said, and + // the merge's modules are still taken in; FillTaken writes its key once the state appears. + ix.logf("%s at %s: its taken key is not written yet, FillTaken writes it when the state appears: %v", f.Repository, f.Merge, err) + ix.mu.Lock() + if ix.untaken == nil { + ix.untaken = map[string]string{} + } + ix.untaken[f.Repository] = f.Merge // the latest heard; FillTaken keeps a later key over an earlier merge + ix.mu.Unlock() + return nil + } + return err +} + +// FillTaken writes the taken keys that were left unwritten, never a merge the indexer has not applied whole (a key names +// a merge only after it was applied: ADR 0556 clause 1, and the view reads it as the upper bound of what is reached). Two +// sources, both through writeTaken as a live merge does (issue 1280): +// - the merges applied while the state was not on the bus (kept by writeTaken), whenever the state is there; +// - the gate's source heads, only when Drained says the indexer has applied every message of the forge stream that was +// there before the heads were read. Otherwise it says so, and the next kick of the fill (after a message is applied) +// tries again. +// +// It is idempotent: a key naming the merge or a later one is left. A state not on the bus yet writes nothing and says +// nothing (the fill runs again when it appears). The first error is returned after every repository was tried. +func (ix *Indexer) FillTaken(ctx context.Context) error { + if ix.Taken == nil { + return nil + } + if _, err := ix.Taken.Keys(ctx); errors.Is(err, process.ErrNotYet) { + return nil + } + ix.mu.Lock() + pending := maps.Clone(ix.untaken) + heads := maps.Clone(ix.source.heads) + ix.mu.Unlock() + var first error + note := func(repository string, err error) { + if err != nil && first == nil { + first = fmt.Errorf("%s: %w", repository, err) + } + } + for _, repository := range slices.Sorted(maps.Keys(pending)) { + err := ix.writeTaken(ctx, delivery.Forge{Repository: repository, Merge: pending[repository]}) + if err == nil { + ix.mu.Lock() + if ix.untaken[repository] == pending[repository] { + delete(ix.untaken, repository) + } + ix.mu.Unlock() + } + note(repository, err) + } + if ix.Drained == nil { + return first + } + if len(heads) == 0 { + ix.logf("the source names no repository head, so no taken key is filled from it") + return first + } + done, err := ix.Drained(ctx) + if err != nil { + ix.logf("whether the forge stream is applied cannot be read, so no taken key is filled from the source: %v", err) + return first + } + if !done { + ix.logf("the forge stream is not applied whole yet, so no taken key is filled from the source; the next applied message tries again") + return first + } + for _, repository := range slices.Sorted(maps.Keys(heads)) { + note(repository, ix.writeTaken(ctx, delivery.Forge{Repository: repository, Merge: heads[repository]})) + } + return first +} + +// KickFill runs FillTaken in the background after a message was applied, one at a time; a kick that arrives while one runs +// makes it run once more. +func (ix *Indexer) KickFill(ctx context.Context) { + if ix.filling.Swap(true) { + ix.fillAgain.Store(true) + return + } + go func() { + for { + ix.fillAgain.Store(false) + if err := ix.FillTaken(ctx); err != nil { + ix.logf("filling the taken keys: %v", err) + } + if !ix.fillAgain.Load() { + ix.filling.Store(false) + if !ix.fillAgain.Load() || ix.filling.Swap(true) { + return + } + } + } + }() +} diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 46b3589..0122ca7 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -38,6 +38,7 @@ type manifest struct { Version string `json:"version"` Slug string `json:"slug"` Tools []string `json:"tools"` + Emits []string `json:"emits"` Consumes []string `json:"consumes"` State []state `json:"state"` Reads []string `json:"reads"` @@ -80,7 +81,11 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build if m.Module != "mesh-delivery-indexer" { t.Errorf("module = %q", m.Module) } - if want := []state{{Name: "candidates", History: 10}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "takeins", History: 1}}; !reflect.DeepEqual(m.State, want) { + // 1126 (ADR 0549 §2): the indexer's placement state, and the event it says when a node's placement changed. + if want := []string{"placement-changed"}; !reflect.DeepEqual(m.Emits, want) { + t.Errorf("emits = %v, want %v", m.Emits, want) + } + if want := []state{{Name: "candidates", History: 10}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "placement", History: 10}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}}; !reflect.DeepEqual(m.State, want) { t.Errorf("state = %+v, want %+v", m.State, want) } if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-controller.assignments", "mesh-controller.builds", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) { @@ -107,8 +112,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build if want := []string{"indexer_take_in"}; !reflect.DeepEqual(m.Tools, want) { t.Errorf("tools = %v, want %v", m.Tools, want) } - // No emits, no streams, no bus, no seat (ADR 0437 §4). - for _, k := range []string{"streams", "logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "seat", "seats", "bus"} { + // No streams, no bus, no seat (ADR 0437 §4); its one event is placement-changed (ADR 0549 §2), checked above. + for _, k := range []string{"streams", "logs", "claims", "provides", "requires", "uses", "defines-seats", "seat", "seats", "bus"} { if _, ok := top[k]; ok { t.Errorf("the manifest declares %q: the indexer has none", k) } diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index a30c24e..04bca8c 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -5,6 +5,9 @@ "tools": [ "indexer_take_in" ], + "emits": [ + "placement-changed" + ], "consumes": [ "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", @@ -15,6 +18,8 @@ { "name": "candidates", "history": 10 }, { "name": "manifests", "history": 1 }, { "name": "modules", "history": 20 }, + { "name": "placement", "history": 10 }, + { "name": "taken", "history": 1 }, { "name": "takeins", "history": 1 } ], "reads": [ diff --git a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller.go b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller.go index 40f1f20..a900626 100644 --- a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller.go +++ b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller.go @@ -286,16 +286,17 @@ func (t *Teller) catchUp(id string) { t.said(h, *w, err) } -// mergeAct is what the approve option of the gate's ask binds: this ask, the answer, and the heads it covers. It is -// spelled as the gate composes it (mesh-catalog modules/mesh-delivery-gate approval.go mergeAct, from +// mergeAct is what the approve option of the gate's ask binds: the ask's verb (delivery.AskVerb: mesh-delivery.merge, or +// mesh-delivery.settle for a person's settle ask, novox/hq ADR 0556 clause 5), this ask, the answer, and the heads it covers. +// It is spelled as the gate composes it (mesh-catalog modules/mesh-delivery-gate approval.go mergeAct, from // modules/mesh-delivery approval.go:63-65) and as mesh-controller verifies it (cmd/mesh-controller/moved.go:98-106). -func mergeAct(ask, option string, heads map[string]string) asks.Act { +func mergeAct(verb, ask, option string, heads map[string]string) asks.Act { var out []string for id, sha := range heads { out = append(out, id+"="+sha) } sort.Strings(out) - return asks.Act{"verb": "mesh-delivery.merge", "ask": ask, "option": option, "heads": strings.Join(out, ",")} + return asks.Act{"verb": verb, "ask": ask, "option": option, "heads": strings.Join(out, ",")} } // headsOf is the heads an ask covers as an act names them: a delivery's id, `owner/repository@, novox/hq ADR 0549): " + + "each node's modules in order with their host ports, bindings and settings placement, its pins and the seats its modules hold, and the catalogue revision. " + + "With node, that node's; with module, every node placing it; with neither, every node's. Keys that do not read are listed as unreadable. Read-only.", + Input: map[string]any{"type": "object", "properties": nodeModule}, + Run: func(args map[string]any) (any, error) { + node, module := nodeModuleOf(args) + return r.Placement(node, module) + }}, {Name: view.CandidateReadTool, Description: "What the indexer stored for one candidate (the open pull requests of a branch) under candidates., as stored: " + "the pull requests and heads, the base, and per pull request the touched modules with their new-form fingerprint (fp2) or why " + @@ -180,7 +201,9 @@ func read(state stdio.KeptState) view.StateRead { func main() { r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds")), CandidatesState: read(stdio.State("mesh-delivery-indexer.candidates")), - WantsState: read(stdio.State("mesh-delivery-assigner.wants")), DifferencesState: read(stdio.State("mesh-delivery-assigner.differences")), HavesState: read(stdio.State("node-engine.have"))} + WantsState: read(stdio.State("mesh-delivery-assigner.wants")), DifferencesState: read(stdio.State("mesh-delivery-assigner.differences")), HavesState: read(stdio.State("node-engine.have")), + ReachState: read(stdio.State("reach")), + PlacementState: read(stdio.State("mesh-delivery-indexer.placement"))} u := view.UsageReader{Haves: r.HavesState, Resolved: read(stdio.State("mesh-config.resolved")), Tries: read(stdio.State("mesh-delivery-checker.tries")), Steps: map[string]view.StateRead{}, Pauses: map[string]view.StateRead{}, Backlog: backlogOf(socket())} for _, lang := range delivery.AskLanguages { diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go index cd98d08..51ca95a 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go @@ -71,6 +71,19 @@ func TestModuleToolAnswersTheStoredValue(t *testing.T) { if _, err := run(t, r, view.ModuleReadTool, map[string]any{"module": ""}); err == nil { t.Errorf("an empty name was answered") } + // 1280 (ADR 0556): view_reach answers the view's reach of one repository as stored, or of every one, and says a + // repository it holds none of. + const reach = `{"v":1,"repository":"novox/web","reached":"c1","reached_src":12}` + r.ReachState = fake{"novox/web": reach}.reader().State + if out, err := run(t, r, view.ReachReadTool, map[string]any{"repository": "novox/Web"}); err != nil || string(out.(json.RawMessage)) != reach { + t.Errorf("view_reach novox/Web = %v, %v", out, err) + } + if out, err := run(t, r, view.ReachReadTool, map[string]any{}); err != nil || !strings.Contains(fmt.Sprint(out), "novox/web") { + t.Errorf("view_reach of every repository = %v, %v", out, err) + } + if _, err := run(t, r, view.ReachReadTool, map[string]any{"repository": "novox/gone"}); err == nil || !strings.Contains(err.Error(), "no reach of novox/gone") { + t.Errorf("a repository with no reach: %v", err) + } } // 1562/AC1: indexer_modules answers every key as stored, and with none2 only those that say why they have no FP2. @@ -92,6 +105,32 @@ func TestModulesToolAnswersEveryKeyAsStored(t *testing.T) { if len(only) != 1 || string(only["b"]) != withNone { t.Errorf("none2 modules = %v", only) } + + // 1126 (ADR 0549 §2): indexer_placement answers a node's key as stored, every node placing a module, or every node's, + // and lists a key that does not read. + const ace, g14 = `{"v":1,"node":"ace","revision":"m1","modules":[{"module":"plex"}]}`, `{"v":1,"node":"g14","revision":"m1","modules":[{"module":"zsh"},{"module":"plex"}]}` + p := view.Reader{PlacementState: fake{"placement.ace": ace, "placement.g14": g14, "placement.bad": `{"v":1}`, "other": "{}"}.reader().State} + got := func(args map[string]any) map[string]json.RawMessage { + t.Helper() + out, err := run(t, p, view.PlacementReadTool, args) + if err != nil { + t.Fatal(err) + } + return out.(map[string]any)["placement"].(map[string]json.RawMessage) + } + if one := got(map[string]any{"node": "ace"}); len(one) != 1 || string(one["ace"]) != ace { + t.Errorf("node ace = %v", one) + } + if zsh := got(map[string]any{"module": "zsh"}); len(zsh) != 1 || string(zsh["g14"]) != g14 { + t.Errorf("module zsh = %v", zsh) + } + out, err = run(t, p, view.PlacementReadTool, map[string]any{}) + if all := out.(map[string]any); err != nil || len(all["placement"].(map[string]json.RawMessage)) != 2 || len(all["unreadable"].([]string)) != 2 { + t.Errorf("every node = %v, %v", all, err) + } + if _, err := run(t, p, view.PlacementReadTool, map[string]any{"node": "novox"}); err == nil || !strings.Contains(err.Error(), "no placement of novox") { + t.Errorf("a node with no key: %v", err) + } } const ( diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go index 4ae4f95..aa41f4f 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go @@ -144,13 +144,31 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } }) model.SetDeliveredEvent(func(e view.DeliveredEvent) { - if err := emitEvent(ctx, js, cred, DeliveredEventType, e, fmt.Sprintf("delivered.%d", e.Revision)); err != nil { + id := fmt.Sprintf("delivered.%d", e.Revision) + if e.Repository != "" { // a repository's reached head (ADR 0556 clause 3) + key, _ := delivery.RepositoryKey(e.Repository) + id = fmt.Sprintf("reached.%s.%d.%d", key, e.Revision, len(e.NotYet)) + if len(e.NotYet) > 0 { + sum := sha256.Sum256([]byte(strings.Join(e.NotYet, ","))) + id += "." + hex.EncodeToString(sum[:6]) + } + } + if err := emitEvent(ctx, js, cred, DeliveredEventType, e, id); err != nil { logger.Printf("the delivery of source revision %d could not be said: %v", e.Revision, err) } }) + model.SetReachWriter(func(w view.ReachWrite) { + if err := putReach(ctx, js, w); err != nil { + logger.Printf("the reach of %s could not be written (written again at its next change): %v", w.Key, err) + } + }) model.SetBacklog(view.BacklogOver(js, 5*time.Second)) - for _, bucket := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket, delivery.CandidatesBucket, delivery.ModulesBucket, delivery.AsksBucket, delivery.AnswersBucket, delivery.BuilderBuildsBucket, delivery.AssignerWantsBucket, delivery.SourceBucket, delivery.ControllerComposedBucket, delivery.AssignerAsksBucket, delivery.ViewAnswersBucket} { - go follow(ctx, js, bucket, model, logger) + for _, bucket := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket, delivery.CandidatesBucket, delivery.ModulesBucket, delivery.AsksBucket, delivery.AnswersBucket, delivery.BuilderBuildsBucket, delivery.AssignerWantsBucket, delivery.SourceBucket, delivery.ControllerComposedBucket, delivery.AssignerAsksBucket, delivery.ViewAnswersBucket, delivery.TakenBucket} { + go follow(ctx, js, bucket, false, model, logger) + } + // The modules' and the source's history give each fingerprint's first commit and each head's source revision (ADR 0556). + for _, bucket := range []string{delivery.ModulesBucket, delivery.SourceBucket} { + go follow(ctx, js, bucket, true, model, logger) } select { case <-ctx.Done(): @@ -181,11 +199,21 @@ func (w kvWriter) Put(key string, value []byte) error { return err } +// putReach writes a reach value under its key in the view's own state reach (ADR 0556 clause 2), its only writer. +func putReach(ctx context.Context, js jetstream.JetStream, w view.ReachWrite) error { + kv, err := js.KeyValue(ctx, delivery.ReachBucket) + if err != nil { + return fmt.Errorf("the bucket %s: %w", delivery.ReachBucket, err) + } + _, err = kv.Put(ctx, w.Key, w.Value) + return err +} + // follow watches a bucket with its last values first and then each change, for as long as ctx lives: a bucket that is // not there yet, or a watch that ends, is tried again. -func follow(ctx context.Context, js jetstream.JetStream, bucket string, model *view.Model, logger *log.Logger) { +func follow(ctx context.Context, js jetstream.JetStream, bucket string, history bool, model *view.Model, logger *log.Logger) { for ctx.Err() == nil { - if err := watchOnce(ctx, js, bucket, model); err != nil && ctx.Err() == nil { + if err := watchOnce(ctx, js, bucket, history, model); err != nil && ctx.Err() == nil { logger.Printf("cannot watch %s, trying again: %v", bucket, err) } select { @@ -195,12 +223,17 @@ func follow(ctx context.Context, js jetstream.JetStream, bucket string, model *v } } -func watchOnce(ctx context.Context, js jetstream.JetStream, bucket string, model *view.Model) error { +func watchOnce(ctx context.Context, js jetstream.JetStream, bucket string, history bool, model *view.Model) error { kv, err := js.KeyValue(ctx, bucket) if err != nil { return err } - w, err := kv.WatchAll(ctx) + var opts []jetstream.WatchOpt + initial := bucket + if history { + opts, initial = append(opts, jetstream.IncludeHistory()), view.ReachHistory(bucket) + } + w, err := kv.WatchAll(ctx, opts...) if err != nil { return err } @@ -214,11 +247,11 @@ func watchOnce(ctx context.Context, js jetstream.JetStream, bucket string, model return errors.New("the watch ended") } if e == nil { // the last values have all been delivered - model.Initial(bucket) + model.Initial(initial) continue } model.Apply(view.Entry{Bucket: bucket, Key: e.Key(), Value: e.Value(), At: e.Created(), - Deleted: e.Operation() != jetstream.KeyValuePut, Revision: e.Revision()}) + Deleted: e.Operation() != jetstream.KeyValuePut, Revision: e.Revision(), History: history}) } } } diff --git a/modules/mesh-delivery-view/go.mod b/modules/mesh-delivery-view/go.mod index ae85fd6..a1dec0c 100644 --- a/modules/mesh-delivery-view/go.mod +++ b/modules/mesh-delivery-view/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-view go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026125731-2aae6f5580f2 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026211913-68d65e6bbf61 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-view/go.sum b/modules/mesh-delivery-view/go.sum index f754077..b3dab4f 100644 --- a/modules/mesh-delivery-view/go.sum +++ b/modules/mesh-delivery-view/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026125731-2aae6f5580f2 h1:s1WcG7Ll1OkEwwvMKEabdWvPvTiFCHghlXiY1LoIJDA= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026125731-2aae6f5580f2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026211913-68d65e6bbf61 h1:/c6ncICWjq55nBf8TlDM0ROe/PuH+qZIdnuVbtg8UuM= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026211913-68d65e6bbf61/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-view/internal/view/reach.go b/modules/mesh-delivery-view/internal/view/reach.go new file mode 100644 index 0000000..d055f0d --- /dev/null +++ b/modules/mesh-delivery-view/internal/view/reach.go @@ -0,0 +1,347 @@ +package view + +import ( + "fmt" + "maps" + "slices" + "strings" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// How far each repository reached (novox/hq ADR 0556 clauses 2 and 3, issue 1280). The view keeps, per repository the gate's +// source names, how far each of its modules reached on the nodes that want it, and derives the repository's reached head: the +// newest head of the repository in the source's history, not past the merge the indexer took in, at which every wanted module's +// floor is at or after its last change at or before that head. It writes that as its state reach and says each new reached +// head as its event delivered, per repository. Nothing here records a merge as delivered (ADR 0429 §1): a merge at or before +// the reached head is on the machines, derived by whoever reads it. + +// ReachHistory names the watch of a bucket with its history, as Initial is told it: the modules' and the source's history +// give each fingerprint's first commit and each head's source revision. +func ReachHistory(bucket string) string { return bucket + "#history" } + +// reachBuckets are the watches the derivation waits for before it writes or says anything, so a start does not write a reach +// derived from half its inputs. +var reachBuckets = []string{delivery.ModulesBucket, delivery.AssignerWantsBucket, delivery.HaveBucket, delivery.SourceBucket, + delivery.TakenBucket, ReachHistory(delivery.ModulesBucket), ReachHistory(delivery.SourceBucket)} + +// ReachWrite is a reach value to write under its key in the view's state reach. +type ReachWrite struct { + Key string + Value []byte +} + +// SetReachWriter sets where the view's reach values go; nothing is written until it is set. +func (m *Model) SetReachWriter(put func(ReachWrite)) { + m.mu.Lock() + defer m.mu.Unlock() + m.reachPut = put +} + +// applyHistory takes one revision of a watch with history: of the indexer's modules (each module's values in order) and of +// the gate's source (each revision's heads). +func (m *Model) applyHistory(e Entry) { + switch e.Bucket { + case delivery.ModulesBucket: + if e.Deleted { + delete(m.modHist, e.Key) + return + } + mod, err := delivery.DecodeModule(e.Value) + if err != nil || mod.FP2 == "" || mod.Commit == "" { + return + } + h := m.modHist[e.Key] + if n := len(h); n > 0 && h[n-1].FP2 == mod.FP2 && h[n-1].Commit == mod.Commit { + if mod.Src != 0 { + h[n-1].Src = mod.Src + } + return + } + h = append(h, mod) + if len(h) > 2*delivery.MostChanges { + h = h[len(h)-delivery.MostChanges:] + } + m.modHist[e.Key] = h + case delivery.SourceBucket: + if e.Key != delivery.SourceMainKey || e.Deleted || e.Revision == 0 { + return + } + src, err := delivery.DecodeSource(e.Value) + if err != nil { + return + } + m.srcHist[e.Revision] = src.Heads + } +} + +// headAt is one head of a repository and the first source revision that named it. +type headAt struct { + commit string + rev uint64 +} + +// headsOf are the heads of a repository in the source's history, oldest first, each with the first revision naming it. +func (m *Model) headsOf(repository string) []headAt { + var out []headAt + for _, rev := range slices.Sorted(maps.Keys(m.srcHist)) { + for r, head := range m.srcHist[rev] { + if !delivery.SameRepository(r, repository) || head == "" { + continue + } + if n := len(out); n == 0 || !sameCommit(out[n-1].commit, head) { + out = append(out, headAt{commit: head, rev: rev}) + } + } + } + return out +} + +// revOf is the first source revision naming a commit as the repository's head, or 0 when the history holds none. +func revOf(heads []headAt, commit string) uint64 { + for _, h := range heads { + if sameCommit(h.commit, commit) { + return h.rev + } + } + return 0 +} + +// sameCommit is whether two spellings of a commit are one (either may be abbreviated to at least seven characters). +func sameCommit(a, b string) bool { + a, b = strings.ToLower(a), strings.ToLower(b) + if len(a) > len(b) { + a, b = b, a + } + return len(a) >= 7 && strings.HasPrefix(b, a) +} + +// noBound is a module's bound when its floor is its newest change: it holds no head back. +const noBound = ^uint64(0) + +// wantKeys are the wants' keys by module, each sorted: read once for every repository a derivation looks at. +func (m *Model) wantKeys() map[string][]string { + out := map[string][]string{} + for _, key := range slices.Sorted(maps.Keys(m.wants)) { + _, module := splitKey(key) + out[module] = append(out[module], key) + } + return out +} + +// deriveReach is the reach of one repository from the model as it stands; wants are the wants' keys by module (wantKeys). +func (m *Model) deriveReach(repository string, wants map[string][]string) delivery.Reach { + r := delivery.Reach{Repository: delivery.RepositoryName(repository), Modules: map[string]delivery.ModuleReach{}} + heads := m.headsOf(repository) + key, _ := delivery.RepositoryKey(repository) + if t, ok := m.taken[key]; ok { + r.Indexed = t.Merge + } + bound := noBound + var holds []string + for _, name := range slices.Sorted(maps.Keys(m.modules)) { + me := m.modules[name] + if me.err != "" || me.value.FP2 == "" || !delivery.SameRepository(me.value.Repository, repository) { + continue + } + mr, b, words := m.moduleReach(name, me.value, heads, wants[name]) + r.Modules[name] = mr + if b < bound { + bound = b + } + if b != noBound { + holds = append(holds, words) + } + } + if len(r.Modules) == 0 { + r.Modules = nil + } + indexed := uint64(0) + if r.Indexed != "" { + indexed = revOf(heads, r.Indexed) + } + switch { + case r.Indexed == "": + holds = append(holds, "the indexer has taken in no merge of "+r.Repository) + case indexed == 0: + holds = append(holds, fmt.Sprintf("the merge the indexer took in, %.12s, is not a head of %s in the source's history", r.Indexed, r.Repository)) + } + for _, h := range heads { + if h.rev <= indexed && h.rev < bound { + r.Reached, r.ReachedSrc = h.commit, h.rev + } + } + if n := len(heads); n > 0 && !sameCommit(heads[n-1].commit, r.Reached) { + if indexed != 0 && indexed < heads[n-1].rev { + holds = append(holds, fmt.Sprintf("the indexer has not yet taken in %.12s", heads[n-1].commit)) + } + slices.Sort(holds) + r.Holds = slices.Compact(holds) + } + return r +} + +// moduleReach is how far one module reached, the source revision it holds the reached head before (noBound when none), and +// what holds it, in words. A node counts for the fingerprint its running have runs, healthy or not (ADR 0556 clause 2). +func (m *Model) moduleReach(name string, mod delivery.Module, heads []headAt, wantKeys []string) (delivery.ModuleReach, uint64, string) { + mr := delivery.ModuleReach{FP: mod.FP2} + hist := m.modHist[name] + if n := len(hist); n == 0 || hist[n-1].FP2 != mod.FP2 || hist[n-1].Commit != mod.Commit { + hist = append(slices.Clone(hist), mod) // the value watched now is the newest, whatever the history watch has given + } + for i := len(hist) - 1; i >= 0 && len(mr.Changes) < delivery.MostChanges; i-- { + c := delivery.Change{Commit: hist[i].Commit, FP: hist[i].FP2, Src: hist[i].Src} + if c.Src == 0 { + c.Src = revOf(heads, c.Commit) + } + if n := len(mr.Changes); n > 0 && mr.Changes[n-1].FP == c.FP { + mr.Changes[n-1] = c // the same fingerprint again, back to back: its first appearance is the older commit + continue + } + mr.Changes = append(mr.Changes, c) + } + var waiting []string + floor, floorNode, unset := -1, "", "" + for _, key := range wantKeys { + node, _ := splitKey(key) + w := m.wants[key] + if w.err != "" || w.value.Removed || w.value.FP == "" { + continue + } + mr.Nodes++ + h, ok := m.haves[key] + switch { + case !ok || h.err != "" || h.have.FP == "": + waiting = append(waiting, fmt.Sprintf("%s on %s runs nothing yet", name, node)) + unset = waiting[len(waiting)-1] + continue + case !runs(h.have.Phase): + waiting = append(waiting, fmt.Sprintf("%s on %s is %s at %s, not running", name, node, h.have.Phase, short(h.have.FP))) + unset = waiting[len(waiting)-1] + continue + case h.have.FP != w.value.FP: + waiting = append(waiting, fmt.Sprintf("%s on %s runs %s, not yet the %s it wants", name, node, short(h.have.FP), short(w.value.FP))) + case h.have.Health == delivery.HealthFailed: + waiting = append(waiting, fmt.Sprintf("%s on %s runs %s and its health check failed", name, node, short(h.have.FP))) + case h.have.Health != delivery.HealthOK: + waiting = append(waiting, fmt.Sprintf("%s on %s runs %s, not yet healthy", name, node, short(h.have.FP))) + } + i := slices.IndexFunc(mr.Changes, func(c delivery.Change) bool { return c.FP == h.have.FP }) + if i < 0 { + unset = fmt.Sprintf("%s on %s runs %s, which is not among its %d known changes", name, node, short(h.have.FP), len(mr.Changes)) + waiting = append(waiting, unset) + continue + } + if i > floor { + floor, floorNode = i, node + } + } + mr.Waiting = strings.Join(waiting, "; ") + switch { + case mr.Nodes == 0: + return mr, noBound, "" + case unset != "": + // A node that wants the module runs nothing countable of it (no have, not running, or a build older than the kept + // history): the module holds no head of the repository back, which would make every apply anywhere, or one module + // stuck failed, freeze the whole repository (review of issue 1280, ADR 0556 clause 2). Its words name it, and a merge + // among its changes is not yet on the machines (notYet). + return mr, noBound, "" + } + mr.Floor = mr.Changes[floor].Commit + if floor == 0 { + return mr, noBound, "" + } + next := mr.Changes[floor-1] + words := fmt.Sprintf("%s on %s runs the build of %.12s, not yet that of %.12s", name, floorNode, mr.Floor, next.Commit) + if next.Src == 0 { + // The next change's order is not known: the head is held at the floor's own revision, or before every head when that is + // not known either. + if own := mr.Changes[floor].Src; own != 0 { + return mr, own + 1, words + } + return mr, 0, words + } + return mr, next.Src, words +} + +// runs is whether a have in this phase runs the fingerprint it names: running, being checked after its apply, or held after a +// failed backup, which keeps running what it has (ADR 0539 clause 16). A have backing up or applying, failed or removed runs +// nothing the derivation can count. +func runs(p delivery.Phase) bool { + return p == delivery.PhaseRunning || p == delivery.PhaseChecking || p == delivery.PhaseHeld +} + +// short is a fingerprint's first twelve characters, after its form. +func short(fp string) string { + fp = strings.TrimPrefix(fp, delivery.SourcePrefix2) + if len(fp) > 12 { + fp = fp[:12] + } + return fp +} + +// reachReady is whether every watch the derivation reads handed over what stood at its start. +func (m *Model) reachReady() bool { + for _, b := range reachBuckets { + if !m.initial[b] { + return false + } + } + return m.source != nil && m.source.err == "" +} + +// reachNow derives the reach of every repository the source names, and answers the values that changed and the reached +// heads not yet said. It is called with the lock held. +func (m *Model) reachNow() ([]ReachWrite, []DeliveredEvent) { + if (m.reachPut == nil && m.deliveredEvent == nil) || !m.reachReady() { + return nil, nil + } + var writes []ReachWrite + var events []DeliveredEvent + wants := m.wantKeys() + for _, repository := range slices.Sorted(maps.Keys(m.source.heads)) { + key, err := delivery.RepositoryKey(repository) + if err != nil { + continue + } + r := m.deriveReach(repository, wants) + if prev, ok := m.reach[key]; !ok || !delivery.SameReach(prev, r) { + if data, err := delivery.EncodeReach(r); err == nil { + m.reach[key] = r + writes = append(writes, ReachWrite{Key: key, Value: data}) + } + } + notYet := notYetOf(r) + said := strings.Join(notYet, ",") + if r.Reached != "" && (r.ReachedSrc > m.reachSaid[key] || said != m.reachNotYet[key]) { + m.reachSaid[key], m.reachNotYet[key] = r.ReachedSrc, said + events = append(events, DeliveredEvent{Revision: r.ReachedSrc, Heads: map[string]string{r.Repository: r.Reached}, + Repository: r.Repository, NotYet: notYet}) + } + } + return writes, events +} + +// notYetOf are the merges at or before a repository's reached head that are not yet on the machines: the changes of each module +// a node wants with no floor (a node runs nothing countable of it), sorted. A consumer takes every other merge at or before +// the reached head as on the machines. +func notYetOf(r delivery.Reach) []string { + var out []string + for _, mr := range r.Modules { + if mr.Nodes == 0 || mr.Floor != "" { + continue + } + for _, c := range mr.Changes { + out = append(out, c.Commit) + } + } + slices.Sort(out) + return slices.Compact(out) +} + +// Reach is the reach of every repository as the view derived it last, by key. +func (m *Model) Reach() map[string]delivery.Reach { + m.mu.Lock() + defer m.mu.Unlock() + return maps.Clone(m.reach) +} diff --git a/modules/mesh-delivery-view/internal/view/reach_test.go b/modules/mesh-delivery-view/internal/view/reach_test.go new file mode 100644 index 0000000..f459efe --- /dev/null +++ b/modules/mesh-delivery-view/internal/view/reach_test.go @@ -0,0 +1,157 @@ +package view + +import ( + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// 1280 (ADR 0556 clauses 2 and 3): the view derives, per repository, how far each module reached and the reached head, writes +// it when it differs, and says each new reached head once as its event delivered. +func TestTheViewDerivesHowFarEachRepositoryReached(t *testing.T) { + const web, sdk = "novox/web", "novox/sdk" + commitA, commitB, commitC := strings.Repeat("a", 40), strings.Repeat("b", 40), strings.Repeat("c", 40) + commitS, commitX := strings.Repeat("5", 40), strings.Repeat("e", 40) + const fpMail = "src2:3333333333333333333333333333333333333333333333333333333333333333" + mod := func(module, fp, commit string, src uint64, history bool) Entry { + v, err := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: strings.Repeat("a", 64), Repository: web, Commit: commit, Src: src, Path: module}) + if err != nil { + t.Fatal(err) + } + return Entry{Bucket: delivery.ModulesBucket, Key: module, Value: v, At: t0, History: history} + } + src := func(rev uint64, webHead string) Entry { + v, err := delivery.EncodeSource(delivery.Source{Group: "g", Heads: map[string]string{web: webHead, sdk: commitS}}) + if err != nil { + t.Fatal(err) + } + return Entry{Bucket: delivery.SourceBucket, Key: delivery.SourceMainKey, Value: v, At: t0, Revision: rev, History: true} + } + taken := func(repository, merge string) Entry { + key, _ := delivery.RepositoryKey(repository) + v, err := delivery.EncodeTaken(delivery.Taken{Repository: repository, Merge: merge, At: t0}) + if err != nil { + t.Fatal(err) + } + return Entry{Bucket: delivery.TakenBucket, Key: key, Value: v, At: t0} + } + haveIn := func(node, module, fp string, phase delivery.Phase, health delivery.Health) Entry { + e := haveFP(t, node, module, fp, health) + v, _ := delivery.EncodeHave(delivery.Have{FP: fp, Phase: phase, Health: health}) + e.Value = v + return e + } + // web changed at A (fpOld) and at C (fpNew); mail first appeared at A. The source named A, B, then C as web's head. + world := func(t *testing.T) []Entry { + cur, _ := delivery.DecodeSource(src(12, commitC).Value) + current := Entry{Bucket: delivery.SourceBucket, Key: delivery.SourceMainKey, Revision: 12, At: t0} + current.Value, _ = delivery.EncodeSource(cur) + return []Entry{src(10, commitA), src(11, commitB), src(12, commitC), current, + mod("web", fpOld, commitA, 10, true), mod("web", fpNew, commitC, 12, true), mod("mail", fpMail, commitA, 0, true), + mod("web", fpNew, commitC, 12, false), mod("mail", fpMail, commitA, 0, false), + taken(web, commitC), taken(sdk, commitS), + wantE(t, "ace", "web", delivery.Want{FP: fpNew}), haveFP(t, "ace", "web", fpNew, delivery.HealthOK), + wantE(t, "g14", "web", delivery.Want{FP: fpNew}), haveFP(t, "g14", "web", fpNew, delivery.HealthOK), + wantE(t, "ace", "mail", delivery.Want{FP: fpMail}), haveFP(t, "ace", "mail", fpMail, delivery.HealthOK)} + } + for _, tc := range []struct { + name string + then []Entry + reached string // web's reached head; "" none + holds string // words one of web's holds carries; "" none + waiting string // words web's module web waits with; "" none + notYet []string // the merges the reached head's event says are not yet on the machines + }{ + {name: "every node runs the newest build", reached: commitC}, + {name: "a node lagging stops the head before the change it lacks", reached: commitB, holds: "web on g14", + then: []Entry{haveFP(t, "g14", "web", fpOld, delivery.HealthOK)}, waiting: "not yet the"}, + {name: "a want held back at an older build holds the head as a lag does", reached: commitB, holds: "web on g14 runs the build of", + then: []Entry{wantE(t, "g14", "web", delivery.Want{FP: fpOld, Newest: fpNew, Rule: 2}), haveFP(t, "g14", "web", fpOld, delivery.HealthOK)}}, + {name: "a failing health check is said and holds nothing", reached: commitC, waiting: "health check failed", + then: []Entry{haveFP(t, "g14", "web", fpNew, delivery.HealthFailed)}}, + {name: "a node with no running have holds no head back and its module's merges are not yet", reached: commitC, waiting: "not running", + notYet: []string{commitA, commitC}, + then: []Entry{haveIn("g14", "web", fpNew, delivery.PhaseApplying, delivery.HealthUnknown)}}, + {name: "an unrelated module not running leaves the head reached (the reviewer's probe)", reached: commitC, + notYet: []string{commitA}, + then: []Entry{haveIn("ace", "mail", fpMail, delivery.PhaseFailed, delivery.HealthFailed)}}, + {name: "a module no node wants holds nothing", reached: commitC, + then: []Entry{wantE(t, "ace", "mail", delivery.Want{Removed: true})}}, + {name: "the indexer not yet having taken the head in", reached: commitB, holds: "has not yet taken in", + then: []Entry{taken(web, commitB)}}, + {name: "a change whose source revision is not known yet holds no head the source named", reached: commitC, + then: []Entry{mod("web", "src2:"+strings.Repeat("9", 64), commitX, 0, true), mod("web", "src2:"+strings.Repeat("9", 64), commitX, 0, false)}}, + } { + t.Run(tc.name, func(t *testing.T) { + m := New() + var written []ReachWrite + var said []DeliveredEvent + m.SetReachWriter(func(w ReachWrite) { written = append(written, w) }) + m.SetDeliveredEvent(func(e DeliveredEvent) { + if e.Repository != "" { + said = append(said, e) + } + }) + fill(m, world(t)...) + if len(written) != 0 { + t.Fatalf("written before every watch handed over: %v", written) + } + for _, b := range reachBuckets { + m.Initial(b) + } + fill(m, tc.then...) + key, _ := delivery.RepositoryKey(web) + r := m.Reach()[key] + if r.Reached != tc.reached { + t.Errorf("reached = %.8s, want %.8s (holds %v)", r.Reached, tc.reached, r.Holds) + } + if got := strings.Join(r.Holds, "; "); (tc.holds == "") != (got == "") || !strings.Contains(got, tc.holds) { + t.Errorf("holds = %q, want words with %q", got, tc.holds) + } + if got := r.Modules["web"].Waiting; (tc.waiting == "") != (got == "") || !strings.Contains(got, tc.waiting) { + t.Errorf("web waits with %q, want words with %q", got, tc.waiting) + } + // A library reaches the head the indexer took in, with no module. + sk, _ := delivery.RepositoryKey(sdk) + if lib := m.Reach()[sk]; lib.Reached != commitS || len(lib.Modules) != 0 { + t.Errorf("the library's reach = %+v", lib) + } + // Each key is written once with its last value; a reached head is said once, for its repository alone. + last := map[string]delivery.Reach{} + for _, w := range written { + v, err := delivery.DecodeReach(w.Value) + if err != nil { + t.Fatalf("written %s: %v", w.Key, err) + } + last[w.Key] = v + } + if !delivery.SameReach(last[key], r) { + t.Errorf("the last value written is %+v, not the reach derived %+v", last[key], r) + } + if tc.reached != "" { + var last DeliveredEvent + for _, e := range said { + if e.Repository == web { + last = e + } + } + if strings.Join(last.NotYet, ",") != strings.Join(tc.notYet, ",") { + t.Errorf("not yet = %v, want %v", last.NotYet, tc.notYet) + } + } + { + n := len(said) + fill(m, tc.then...) + if len(said) != n && tc.reached == commitC { + t.Errorf("said again: %+v", said[n:]) + } + for _, e := range said { + if len(e.Heads) != 1 || e.Group != "" || e.Revision == 0 { + t.Errorf("a reached head said as %+v", e) + } + } + } + }) + } +} diff --git a/modules/mesh-delivery-view/internal/view/read.go b/modules/mesh-delivery-view/internal/view/read.go index 31be53d..0aea653 100644 --- a/modules/mesh-delivery-view/internal/view/read.go +++ b/modules/mesh-delivery-view/internal/view/read.go @@ -20,6 +20,8 @@ import ( const ( ModuleReadTool = "indexer_module" ModulesReadTool = "indexer_modules" + // PlacementReadTool reads the indexer's placement (novox/hq ADR 0549 §2, issue 1126). + PlacementReadTool = "indexer_placement" ) // StateRead reads the indexer's modules state, as the view's reads grant it: its keys and a key's stored value (nil for a key with none). @@ -39,6 +41,39 @@ type Reader struct { WantsState StateRead // the assigner's wants and holds DifferencesState StateRead // the assigner's differences HavesState StateRead // the node-engine's haves + ReachState StateRead // the view's own reach (ADR 0556) + PlacementState StateRead // the indexer's placement +} + +// ReachReadTool reads the view's reach: per repository, how far each module reached and the reached head (ADR 0556). +const ReachReadTool = "view_reach" + +// Reach is the stored reach of one repository, or of every repository when none is named, as the view wrote it. +func (r Reader) Reach(repository string) (any, error) { + if r.ReachState.Get == nil || r.ReachState.Keys == nil { + return nil, fmt.Errorf("the reach state cannot be read here") + } + if repository = strings.TrimSpace(repository); repository != "" { + key, err := delivery.RepositoryKey(repository) + if err != nil { + return nil, err + } + raw, err := r.ReachState.Get(key) + if err != nil { + return nil, err + } + if len(raw) == 0 { + return nil, fmt.Errorf("no reach of %s: the view has derived none (the gate's source does not name it, or the view waits for a watch)", repository) + } + return raw, nil + } + keys, err := r.ReachState.Keys() + if err != nil { + return nil, err + } + sort.Strings(keys) + values, err := r.ReachState.values(keys) + return map[string]any{"reach": values}, err } // Module is the stored value of modules., exactly as the indexer wrote it. @@ -613,3 +648,55 @@ func (r Reader) Differences(node, module string) (KeptAnswer, error) { func (r Reader) Haves(node, module string) (KeptAnswer, error) { return r.kept(r.HavesState, "the haves", node, module) } + +// Placement is the indexer's placement as stored (ADR 0549 §2): one node's key with node, every node placing a module +// with module (each node's key whole, so its pins, ports and bindings are beside the module), and every node's key with +// neither. A key that does not read as a placement is listed as unreadable, never dropped. +func (r Reader) Placement(node, module string) (map[string]any, error) { + node, module = strings.TrimSpace(node), strings.TrimSpace(module) + if r.PlacementState.Keys == nil || r.PlacementState.Get == nil { + return nil, fmt.Errorf("the placement state cannot be read here") + } + if node != "" { + key, err := delivery.PlacementKey(node) + if err != nil { + return nil, err + } + raw, err := r.PlacementState.Get(key) + if err != nil { + return nil, err + } + if len(raw) == 0 { + return nil, fmt.Errorf("no placement of %s: the catalogue holds no placement file for it, or the indexer has read none yet", node) + } + return map[string]any{"placement": map[string]json.RawMessage{node: raw}}, nil + } + keys, err := r.PlacementState.Keys() + if err != nil { + return nil, err + } + sort.Strings(keys) + values, err := r.PlacementState.values(keys) + if err != nil { + return nil, err + } + out := map[string]json.RawMessage{} + var unreadable []string + for i, key := range keys { + n, err := delivery.ParsePlacementKey(key) + if err != nil { + unreadable = append(unreadable, key) + continue + } + p, err := delivery.DecodePlacement(values[i]) + if err != nil { + unreadable = append(unreadable, key) + continue + } + if module != "" && !slices.Contains(p.Assigned(), module) { + continue + } + out[n] = values[i] + } + return map[string]any{"placement": out, "unreadable": unreadable}, nil +} diff --git a/modules/mesh-delivery-view/internal/view/usage.go b/modules/mesh-delivery-view/internal/view/usage.go index 19d43ea..35e1fbc 100644 --- a/modules/mesh-delivery-view/internal/view/usage.go +++ b/modules/mesh-delivery-view/internal/view/usage.go @@ -67,7 +67,8 @@ type UsageAnswer struct { At time.Time `json:"at"` Nodes []NodeUsage `json:"nodes"` Backlog []BacklogRow `json:"backlog"` - // TriesRunning are the tries the checker runs now; their asks are counted under check until the try ends and names them. + // TriesRunning are the tries the checker runs now, each with its asks and where each stands (a checker that names its + // asks as they are put, issue 1890; an older one names them only at the end, and they are counted under check until then). TriesRunning []TryRunning `json:"tries_running"` Notes []string `json:"notes,omitempty"` } @@ -77,6 +78,9 @@ type TryRunning struct { ID string `json:"id"` Repository string `json:"repository"` Started time.Time `json:"started,omitzero"` + Asks []string `json:"asks,omitempty"` + // Steps are the try's own words for each ask: asked, or the node that took it, its state and the phase its step says. + Steps json.RawMessage `json:"steps,omitempty"` } // NodeUsage is one node: each build seat it holds or is set to build for, whether it is eligible to build, and its CPU. @@ -101,7 +105,17 @@ type SeatUsage struct { // Capacity is the asks it runs at once; CapacityFrom where that came from. Capacity int `json:"capacity"` CapacityFrom string `json:"capacity_from"` - Paused bool `json:"paused"` + // Configured is mesh-config's capacity when the module says it judges by a lower one (capacity. of its pauses + // state: the configured capacity bounded by its node's build CPU, issue 1806's follow-up); Capacity is then the + // module's and CapacityFrom its why. + Configured int `json:"configured,omitempty"` + // BuildHours is the node's build window as the module says it (HH:MM-HH:MM and its zone), absent when it builds at any + // hour; OutsideHours says the module judged the time outside it, so it takes nothing new (the operator, 2026-10-26). + BuildHours string `json:"build_hours,omitempty"` + OutsideHours bool `json:"outside_hours,omitempty"` + // BuildHoursFault is why the module cannot read its node's build-hours value, naming it: it then builds at any hour. + BuildHoursFault string `json:"build_hours_fault,omitempty"` + Paused bool `json:"paused"` // Eligible says it takes asks: it holds the seat, has a capacity above 0 and is not paused; WhyNot says why not. // EligibleKnown is false when this tool cannot tell (it cannot read mesh-config's capacities): Eligible is then // absent, never false, and WhyNot starts "not known: ". @@ -384,8 +398,9 @@ func (u UsageReader) Usage() UsageAnswer { } } - // Pauses: paused. of each module. + // Pauses: paused. of each module; and capacity., the capacity the module says it judges by. paused := map[string]bool{} // . + judged := map[string]delivery.Capacity{} for _, lang := range delivery.AskLanguages { module, _ := delivery.BuildModuleOf(lang) pr, ok := u.Pauses[lang] @@ -411,6 +426,13 @@ func (u UsageReader) Usage() UsageAnswer { paused[node+"."+lang] = p.Paused } } + if key, err := delivery.CapacityKey(node); err == nil { + if raw, err := pr.Get(key); err == nil && len(raw) > 0 { + if c, err := delivery.DecodeCapacity(raw); err == nil { + judged[node+"."+lang] = c + } + } + } } } @@ -452,6 +474,18 @@ func (u UsageReader) Usage() UsageAnswer { s.Capacity = (&buildrun.Places{Capacity: caps}).CapacityOf(node) s.CapacityFrom = fmt.Sprintf("the modules' default %q: mesh-config holds no resolved value for %s on %s", buildstep.DefaultCapacities, s.Module, node) } + if c, ok := judged[node+"."+lang]; ok && s.Holds && unknown == "" && !isBad && c.Configured == s.Capacity { + // The module running on the node says its build hours, and judges by less than mesh-config gives when its + // node's build CPU bounds it or the time is outside its build hours. + s.BuildHours, s.OutsideHours = c.BuildHours, c.OutsideHours + if c.BuildHoursFault != "" { + s.BuildHours, s.BuildHoursFault = "any hour", c.BuildHoursFault + } + if c.Effective != c.Configured { + s.Configured, s.Capacity = s.Capacity, c.Effective + s.CapacityFrom = fmt.Sprintf("%s, as %s on %s judges it (said %s)", c.Why, s.Module, node, c.At.UTC().Format(time.RFC3339)) + } + } s.Paused = paused[node+"."+lang] s.EligibleKnown = true switch { @@ -466,6 +500,8 @@ func (u UsageReader) Usage() UsageAnswer { s.WhyNot = "not known: this tool cannot read its capacity (" + unknown + ")" case isBad: s.WhyNot = "its capacity on " + node + " cannot be read, so its puller uses 0" + case s.OutsideHours: + s.WhyNot = "outside its build hours " + s.BuildHours + " on " + node + ": it takes nothing new until they open" case s.Capacity == 0: s.WhyNot = "its capacity on " + node + " is 0" } @@ -517,6 +553,7 @@ func (u UsageReader) tries(now time.Time, ans *UsageAnswer, note func(string, .. want = append(want, k) } } + unnamed := 0 raws, err := u.Tries.values(want) if err != nil { note("the checker's tries cannot all be read: %v", err) @@ -526,11 +563,12 @@ func (u UsageReader) tries(now time.Time, ans *UsageAnswer, note func(string, .. continue } var t struct { - ID string `json:"id"` - Repository string `json:"repository"` - State string `json:"state"` - Started time.Time `json:"started"` - Asks []string `json:"asks"` + ID string `json:"id"` + Repository string `json:"repository"` + State string `json:"state"` + Started time.Time `json:"started"` + Asks []string `json:"asks"` + Steps json.RawMessage `json:"steps"` } if json.Unmarshal(raws[i], &t) != nil { continue @@ -539,11 +577,14 @@ func (u UsageReader) tries(now time.Time, ans *UsageAnswer, note func(string, .. asks[a] = true } if t.State == "running" { - ans.TriesRunning = append(ans.TriesRunning, TryRunning{ID: t.ID, Repository: t.Repository, Started: t.Started}) + ans.TriesRunning = append(ans.TriesRunning, TryRunning{ID: t.ID, Repository: t.Repository, Started: t.Started, Asks: t.Asks, Steps: t.Steps}) + if len(t.Asks) == 0 { + unnamed++ + } } } - if len(ans.TriesRunning) > 0 { - note("a running try's asks are counted under check until the try ends and names them") + if unnamed > 0 { + note("%d running tries name no ask yet (none put yet, or a checker that names them only at the end): their asks are counted under check until then", unnamed) } return asks } diff --git a/modules/mesh-delivery-view/internal/view/usage_test.go b/modules/mesh-delivery-view/internal/view/usage_test.go index 580018a..dc33ff1 100644 --- a/modules/mesh-delivery-view/internal/view/usage_test.go +++ b/modules/mesh-delivery-view/internal/view/usage_test.go @@ -191,4 +191,36 @@ func TestBuildUsageAnswersEveryNodesUseFromTheStates(t *testing.T) { if verdict["shanks"] != "not known" || verdict["ace"] != "not eligible" { t.Errorf("verdicts with mesh-config unreadable = %v, want shanks not known, ace not eligible (paused)", verdict) } + + // Issue 1806's follow-up: the module on shanks says it judges by 1 of mesh-config's 2 (its build CPU bounds it); the + // seat says the module's capacity and why. And issue 1890: a running try that names its asks is said with them. + u = usageFixture() + u.Pauses["go"] = states(map[string]string{"capacity.shanks": `{"v":1,"effective":1,"configured":2,"quota-from":"the cpu.max of /meshyield.slice","quota-cores":6,"cores-per-step":6,"by-cpu":1,"why":"1, bounded by the CPU","at":"2026-10-26T11:00:00Z"}`, + "capacity.ace": `{"v":1,"effective":2,"configured":3,"why":"stale: mesh-config says 2 now"}`}) + u.Tries = states(map[string]string{"try-" + strconv.FormatInt(at("11:58").UnixNano(), 10): `{"id":"try-2","repository":"novox/mesh-delivery","head":"h","trunk":"t","state":"running","asks":["` + tryAsk0 + `"],"steps":[{"ask":"` + tryAsk0 + `","node":"shanks","state":"running","phase":"unit: tests of 3 packages"}]}`}) + bound := u.Usage() + sc := seatOf(t, bound, "shanks", "node-build-go") + if sc.Capacity != 1 || sc.Configured != 2 || !strings.HasPrefix(sc.CapacityFrom, "1, bounded by the CPU, as mesh-build-go on shanks judges it") { + t.Errorf("a seat its CPU bounds: capacity %d of %d, from %q", sc.Capacity, sc.Configured, sc.CapacityFrom) + } + if sa := seatOf(t, bound, "ace", "node-build-go"); sa.Capacity != 2 || sa.Configured != 0 { + t.Errorf("a module's word older than mesh-config's value is not taken: %+v", sa) + } + // The operator, 2026-10-26: outside its build hours the seat takes nothing new, and says so. + u.Pauses["go"] = states(map[string]string{"capacity.shanks": `{"v":1,"effective":0,"configured":2,"build-hours":"18:00-06:00 CET","outside-hours":true,"why":"0, outside its build hours 18:00-06:00 CET until 18:00 (otherwise 2)","at":"2026-10-26T11:00:00Z"}`}) + if so := seatOf(t, u.Usage(), "shanks", "node-build-go"); so.Capacity != 0 || !so.OutsideHours || so.Eligible == nil || *so.Eligible || + so.WhyNot != "outside its build hours 18:00-06:00 CET on shanks: it takes nothing new until they open" { + t.Errorf("a seat outside its build hours: %+v", so) + } + // A build-hours value the module cannot read: it builds at any hour, and the seat says the value and why. + u.Pauses["go"] = states(map[string]string{"capacity.shanks": `{"v":1,"effective":2,"configured":2,"build-hours-fault":"the build hours \"shanks=18:00-6\": \"6\" is not a time HH:MM","why":"2; at any hour, because its build hours cannot be read","at":"2026-10-26T11:00:00Z"}`}) + if sf := seatOf(t, u.Usage(), "shanks", "node-build-go"); sf.BuildHours != "any hour" || !strings.Contains(sf.BuildHoursFault, "shanks=18:00-6") || sf.Eligible == nil || !*sf.Eligible { + t.Errorf("a seat whose build hours cannot be read: %+v", sf) + } + if len(bound.TriesRunning) != 1 || len(bound.TriesRunning[0].Asks) != 1 || !strings.Contains(string(bound.TriesRunning[0].Steps), "unit: tests") || + strings.Contains(strings.Join(bound.Notes, " | "), "name no ask yet") { + t.Errorf("a running try naming its asks: %+v, notes %v", bound.TriesRunning, bound.Notes) + } } + +var tryAsk0 = stepID(at("11:58"), "a00010") diff --git a/modules/mesh-delivery-view/internal/view/view.go b/modules/mesh-delivery-view/internal/view/view.go index 8e3ecff..2c68ba2 100644 --- a/modules/mesh-delivery-view/internal/view/view.go +++ b/modules/mesh-delivery-view/internal/view/view.go @@ -23,6 +23,8 @@ type Entry struct { Deleted bool // Revision is the bucket revision of the change, when the watch gives one (the source revision is the gate's source key's). Revision uint64 + // History says the change comes from a watch with history (ReachHistory): it feeds only the history the reach reads. + History bool } // Row is one module on one node. @@ -240,6 +242,14 @@ type Model struct { // the watchman's buckets, by key presence map[string]presenceEntry stuck map[string]stuckEntry + // how far each repository reached (reach.go, ADR 0556) + modHist map[string][]delivery.Module // the indexer's values of each module, oldest first + srcHist map[uint64]map[string]string // the source's heads, by revision + taken map[string]delivery.Taken // the indexer's taken, by repository key + reach map[string]delivery.Reach // the reach derived last, by repository key + reachSaid map[string]uint64 // the reached head's source revision said last, by repository key + reachNotYet map[string]string // the merges said not yet on the machines with it, by repository key + reachPut func(ReachWrite) } type presenceEntry struct { @@ -348,21 +358,41 @@ type nodeEntry struct { func New() *Model { return &Model{assigns: map[string]assignment{}, builds: map[string]buildEntry{}, groups: map[string]groupEntry{}, modules: map[string]moduleEntry{}, keys: map[string]keyEntry{}, asks: map[string]askRecord{}, wants: map[string]wantEntry{}, told: map[string]bool{}, initial: map[string]bool{}, composed: map[string]composedEntry{}, aasks: map[string]assignerAskRecord{}, chosen: map[string]delivery.ViewAnswer{}, answers: map[string]string{}, answerOf: map[string]delivery.Answer{}, holds: map[string]heldEntry{}, newest: map[string]string{}, haves: map[string]haveEntry{}, nodes: map[string]nodeEntry{}, - presence: map[string]presenceEntry{}, stuck: map[string]stuckEntry{}} + presence: map[string]presenceEntry{}, stuck: map[string]stuckEntry{}, + modHist: map[string][]delivery.Module{}, srcHist: map[uint64]map[string]string{}, taken: map[string]delivery.Taken{}, + reach: map[string]delivery.Reach{}, reachSaid: map[string]uint64{}, reachNotYet: map[string]string{}} } // Apply takes one change of a key. A value it cannot read is kept as an error on its row, never guessed. When the change makes // the source revision delivered, the event of it is said once (SetDeliveredEvent), after the model's lock is released. func (m *Model) Apply(e Entry) { m.mu.Lock() - m.apply(e) - event := m.deliveredNow() - say := m.deliveredEvent - motion, sayMotion := m.motionNow(e), m.motionEvent + var event *DeliveredEvent + var motion []MotionEvent + if e.History { + m.applyHistory(e) + } else { + m.apply(e) + event = m.deliveredNow() + motion = m.motionNow(e) + } + writes, reached := m.reachNow() + say, put := m.deliveredEvent, m.reachPut + sayMotion := m.motionEvent m.mu.Unlock() if event != nil && say != nil { say(*event) } + if put != nil { + for _, w := range writes { + put(w) + } + } + if say != nil { + for _, ev := range reached { + say(ev) + } + } if sayMotion != nil { for _, ev := range motion { sayMotion(ev) @@ -373,10 +403,18 @@ func (m *Model) Apply(e Entry) { // DeliveredEvent is what the view says when the source revision is delivered (event `delivered` of the view, issue 1906 D9, ADR // 0539 §17): the revision of the gate's source key, the group that moved it and every repository's merged head at it. A // consumer takes every earlier revision as covered by it, and takes the same revision twice as once (after a restart of the view). +// +// The same event says a repository's reached head (ADR 0556 clause 3): Repository names it, Heads holds its head alone, Revision +// is the source revision that named that head and Group is empty; a consumer takes every earlier merge of that repository as on +// the machines. type DeliveredEvent struct { - Revision uint64 `json:"revision"` - Group string `json:"group"` - Heads map[string]string `json:"heads"` + Revision uint64 `json:"revision"` + Group string `json:"group"` + Heads map[string]string `json:"heads"` + Repository string `json:"repository,omitempty"` + // NotYet are, in the per-repository form, the merges at or before the reached head that are not yet on the machines: the + // changes of a module a node wants and runs nothing countable of. Said again when they change. + NotYet []string `json:"not_yet,omitempty"` } // SetDeliveredEvent sets where the event of a delivered source revision goes; nothing is said until it is set. @@ -512,6 +550,16 @@ func (m *Model) apply(e Entry) { } d, err := delivery.DecodeComposedDigest(e.Value) m.composed[e.Key] = composedEntry{value: d, err: errText(err)} + case delivery.TakenBucket: + if e.Deleted { + delete(m.taken, e.Key) + return + } + if t, err := delivery.DecodeTaken(e.Value); err == nil { + m.taken[e.Key] = t + } else { + delete(m.taken, e.Key) + } case delivery.SourceBucket: if e.Key != delivery.SourceMainKey { return @@ -1214,10 +1262,22 @@ func (m *Model) Initial(bucket string) { } } say := m.motionEvent + writes, reached := m.reachNow() + put, sayDelivered := m.reachPut, m.deliveredEvent m.mu.Unlock() for _, ev := range out { say(ev) } + if put != nil { + for _, w := range writes { + put(w) + } + } + if sayDelivered != nil { + for _, ev := range reached { + sayDelivered(ev) + } + } } // motionOf is the events to say now about the node and module of a model key, from their want, have and stuck key. diff --git a/modules/mesh-delivery-view/internal/view/view_test.go b/modules/mesh-delivery-view/internal/view/view_test.go index d6ce64e..d228cab 100644 --- a/modules/mesh-delivery-view/internal/view/view_test.go +++ b/modules/mesh-delivery-view/internal/view/view_test.go @@ -110,7 +110,7 @@ func TestWaitsForTheNodeAndForANewerBuild(t *testing.T) { // 1134/AC2 } } -func TestLeftBehindIsNotRunning(t *testing.T) { // 1134/AC3 +func TestLeftBehindIsNotRunningAndPulledInIs(t *testing.T) { // 1134/AC3 (both halves) for name, extra := range map[string]string{"ended": `,"ended":"2026-10-18T11:30:00Z"`, "state": `,"state":"pending"`} { m := New() fill(m, base(t)...) @@ -131,13 +131,11 @@ func TestLeftBehindIsNotRunning(t *testing.T) { // 1134/AC3 if find(t, s, "a", "w").Status != StatusLeftBehind || s.Counted() != 4 { t.Fatalf("a have with no assignment: %+v counted %d", find(t, s, "a", "w"), s.Counted()) } -} - -func TestPulledInIsRunning(t *testing.T) { // 1134/AC3 - m := New() + // A module pulled in by another stands: shown and counted, with what required it. + m = New() fill(m, base(t)...) fill(m, assign("b", "z", "b3", `,"pulled-in":true,"required-by":"x"`)) - s := m.Snapshot() + s = m.Snapshot() r := find(t, s, "b", "z") if r.Status != StatusAssigned || r.RequiredBy != "x" || r.Have != "b3" || s.Counted() != 4 { t.Fatalf("pulled-in row = %+v counted %d, want shown and counted with required-by x", r, s.Counted()) diff --git a/modules/mesh-delivery-view/manifest_test.go b/modules/mesh-delivery-view/manifest_test.go index 6d41a31..ba1fa65 100644 --- a/modules/mesh-delivery-view/manifest_test.go +++ b/modules/mesh-delivery-view/manifest_test.go @@ -3,8 +3,11 @@ package manifest import ( "bytes" "encoding/json" + "io/fs" "os" + "path/filepath" "reflect" + "regexp" "slices" "strings" "testing" @@ -84,7 +87,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 if m.Module != "mesh-delivery-view" { t.Errorf("module = %q", m.Module) } - want := []string{"mesh-build-go.pauses", "mesh-build-go.steps", "mesh-build-image.pauses", "mesh-build-image.steps", "mesh-build-node.pauses", "mesh-build-node.steps", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-assigner.wants", "mesh-delivery-builder.builds", "mesh-delivery-checker.tries", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "node-engine.have"} // build_usage reads the build modules' steps and pauses, mesh-config's capacities and the checker's tries; 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9 + want := []string{"mesh-build-go.pauses", "mesh-build-go.steps", "mesh-build-image.pauses", "mesh-build-image.steps", "mesh-build-node.pauses", "mesh-build-node.steps", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-assigner.wants", "mesh-delivery-builder.builds", "mesh-delivery-checker.tries", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.modules", "mesh-delivery-indexer.placement", "mesh-delivery-indexer.taken", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "node-engine.have"} // build_usage reads the build modules' steps and pauses, mesh-config's capacities and the checker's tries; 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9 got := slices.Clone(m.Reads) slices.Sort(got) if !reflect.DeepEqual(got, want) { @@ -103,8 +106,9 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 if len(m.Uses) != 0 { t.Errorf("uses = %v, want none", m.Uses) } - // The one state the view writes: the operator's answers to the asks that need no proof (ADR 0528 clause 6, 1297/B9), which the teller reads. - if want := []state{{Name: "answers", History: 5}}; !reflect.DeepEqual(m.State, want) { + // The states the view writes: the operator's answers to the asks that need no proof (ADR 0528 clause 6, 1297/B9), which the + // teller reads, and how far each repository reached (ADR 0556 clause 2, issue 1280), which the gate reads. + if want := []state{{Name: "answers", History: 5}, {Name: "reach", History: 1}}; !reflect.DeepEqual(m.State, want) { t.Errorf("state = %+v, want %+v", m.State, want) } if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 { @@ -115,7 +119,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 } // The two read tools of the indexer's modules (issue 1562), carried by a bundle the runtime launches; the indexer // itself serves no tool (ADR 0437 §4). Carrying them gives the tool runner read access to every bucket the view reads. - if want := []string{"indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "builder_build", "builder_builds", "assigner_wants", "assigner_want", "assigner_differences", "node_haves", "build_usage"}; !reflect.DeepEqual(m.Tools, want) { + if want := []string{"indexer_module", "indexer_modules", "indexer_placement", "indexer_candidate", "indexer_candidates", "builder_build", "builder_builds", "assigner_wants", "assigner_want", "assigner_differences", "node_haves", "build_usage", "view_reach"}; !reflect.DeepEqual(m.Tools, want) { t.Errorf("tools = %v, want %v", m.Tools, want) } for _, a := range m.Build.Artifacts { @@ -280,3 +284,42 @@ func TestNoDirectoryBelowRootsTrees(t *testing.T) { // 1134/AC10 t.Error("the socket's directory is /var/lib/mesh-delivery-view-cli, 0755, owned by mesh-delivery-view") } } + +// TestEveryStateTheCommandsReadIsIssued holds every literal stdio.State name in cmd/ to what the mesh issues this +// module: a state of its own by its bare name, another module's by . as `reads` lists it. The tools read +// their own reach as "mesh-delivery-view.reach", which the mesh never issued, and view_reach failed on the machines +// (issue 1280, note 13). +func TestEveryStateTheCommandsReadIsIssued(t *testing.T) { + m, _ := load(t) + issued := map[string]bool{} + for _, s := range m.State { + issued[s.Name] = true + } + for _, r := range m.Reads { + issued[r] = true + } + literal := regexp.MustCompile(`stdio\.State\("([^"]+)"\)`) + seen := 0 + err := filepath.WalkDir("cmd", func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() || !strings.HasSuffix(path, ".go") { + return err + } + src, err := os.ReadFile(path) + if err != nil { + return err + } + for _, match := range literal.FindAllSubmatch(src, -1) { + seen++ + if name := string(match[1]); !issued[name] { + t.Errorf("%s reads the state %q, which the mesh does not issue this module (its own by bare name, another's as `reads` lists it)", path, name) + } + } + return nil + }) + if err != nil { + t.Fatal(err) + } + if seen == 0 { + t.Fatal("no stdio.State literal found under cmd/: the scan reads nothing") + } +} diff --git a/modules/mesh-delivery-view/module.json b/modules/mesh-delivery-view/module.json index 9e10b47..41f42f7 100644 --- a/modules/mesh-delivery-view/module.json +++ b/modules/mesh-delivery-view/module.json @@ -6,6 +6,7 @@ "tools": [ "indexer_module", "indexer_modules", + "indexer_placement", "indexer_candidate", "indexer_candidates", "builder_build", @@ -14,7 +15,8 @@ "assigner_want", "assigner_differences", "node_haves", - "build_usage" + "build_usage", + "view_reach" ], "emits": [ "delivered", @@ -23,7 +25,8 @@ "stuck" ], "state": [ - { "name": "answers", "history": 5 } + { "name": "answers", "history": 5 }, + { "name": "reach", "history": 1 } ], "reads": [ "mesh-controller.assignments", @@ -45,6 +48,8 @@ "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.modules", + "mesh-delivery-indexer.placement", + "mesh-delivery-indexer.taken", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", diff --git a/modules/mesh-grants/go.mod b/modules/mesh-grants/go.mod index 6d8f4e9..19d6941 100644 --- a/modules/mesh-grants/go.mod +++ b/modules/mesh-grants/go.mod @@ -3,7 +3,7 @@ module mesh-grants go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142434-65ee43f07583 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-grants/go.sum b/modules/mesh-grants/go.sum index 4cc1a88..8035b76 100644 --- a/modules/mesh-grants/go.sum +++ b/modules/mesh-grants/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142434-65ee43f07583 h1:ikD/ggf9/5k23sxJOX1PFcGzb4M3+QeC66AT0MyQ9Zs= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142434-65ee43f07583/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117 h1:88CNZOoQgu7YPYKW16qkWT8Gw9Ucg07aTrIImA2nYUk= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-grants/grants_test.go b/modules/mesh-grants/grants_test.go index 77842e3..3e72e77 100644 --- a/modules/mesh-grants/grants_test.go +++ b/modules/mesh-grants/grants_test.go @@ -708,6 +708,43 @@ func TestGrants(t *testing.T) { } }) + // 1126 (ADR 0549 §1, §3): once the catalogue places the nodes, a seat held once for the mesh is held where placement + // says, not on every machine running a module that claims it; what placement adds is composed beside the assignments. + t.Run("a mesh seat is held where placement says", func(t *testing.T) { + store := catalogue.Manifest{Module: "store", Version: "1", + DefinesSeats: []catalogue.SeatDeclaration{{Name: "store-seat", Scope: catalogue.ScopeMesh, Serves: []catalogue.Verb{{Name: "databases"}}}}, + Claims: []catalogue.Claim{{Name: "store-seat", Scope: catalogue.ScopeMesh}}} + in := grants.Inputs{Nodes: []string{"a", "b"}, Assigned: map[string][]string{"a": {"store"}, "b": {"store"}}, + Manifests: map[string]catalogue.Manifest{"store": store}} + holds := func(recs buscompose.Records, node string) int { + n := 0 + for _, d := range recs.Assigned[node] { + n += len(d.Holds) + } + return n + } + if recs, _ := grants.Records(in); holds(recs, "a") != 1 || holds(recs, "b") != 1 { + t.Fatalf("before placement the seat is held on a %d and b %d times, want both", holds(recs, "a"), holds(recs, "b")) + } + in.Placed = map[string]delivery.Placement{ + "a": {Node: "a", Modules: []delivery.PlacedModule{{Module: "store"}}, Holds: []delivery.Holding{{Claim: "store-seat", Scope: "mesh", Module: "store"}}}, + "b": {Node: "b", Modules: []delivery.PlacedModule{{Module: "store"}}}} + if recs, _ := grants.Records(in); holds(recs, "a") != 1 || holds(recs, "b") != 0 { + t.Errorf("placed, the seat is held on a %d and b %d times, want a alone", holds(recs, "a"), holds(recs, "b")) + } + b := fixture() + pl, _ := delivery.EncodePlacement(delivery.Placement{Node: "b", Modules: []delivery.PlacedModule{{Module: "mod-y"}, {Module: "mod-x"}}}) + b.states[grants.ReadPlacement] = map[string][]byte{"placement.b": pl} + read := grants.ReadInputs(b, &grants.Manifests{}) + if !slices.Contains(read.Assigned["b"], "mod-x") || read.Placed["b"].Node != "b" || slices.ContainsFunc(read.Gaps, func(g grants.Gap) bool { return g.Input == "seat holdings" }) { + t.Errorf("read: assigned %v, placed %v, gaps %v", read.Assigned, read.Placed, read.Gaps) + } + b.refused[grants.ReadPlacement] = true + if read := grants.ReadInputs(b, &grants.Manifests{}); slices.ContainsFunc(read.Gaps, func(g grants.Gap) bool { return g.Input == grants.ReadPlacement && g.Failed }) { + t.Errorf("placement not on the bus is a failed read: %v", read.Gaps) + } + }) + t.Run("compare excuses a retired verb and a multiply claimed mesh seat's added subscriptions on its machines, nothing else", func(t *testing.T) { const retired, store, storeInfo = "mesh.seat.mesh-delivery.tool.mark-ready", "mesh.seat.mesh-store.tool.databases", "$SRV.INFO.mesh-store.>" two := map[string][]string{"mesh-store": {"a", "n"}} diff --git a/modules/mesh-grants/internal/grants/follow.go b/modules/mesh-grants/internal/grants/follow.go index 938a243..3d19217 100644 --- a/modules/mesh-grants/internal/grants/follow.go +++ b/modules/mesh-grants/internal/grants/follow.go @@ -18,7 +18,7 @@ type Watcher interface { // Followed are the states whose change composes again: placement, the indexer's manifests by module, the resolved // config values, the node agents' haves (a machine's first have, and the bus node's loaded list) and the assigner's // wants (the manifest each machine is wanted to run), and the vault's hashes of the bus accounts. -var Followed = []string{ReadAssignments, ReadModules, ReadResolved, ReadHave, ReadWants, ReadAccounts} +var Followed = []string{ReadAssignments, ReadModules, ReadResolved, ReadHave, ReadWants, ReadAccounts, ReadPlacement} // Follower composes again on every change it hears, never on a timer. It never stops for a refusal: a watch it cannot // open is said, composed without, and opened again at the next wake (any change it does hear, or a reconnect). diff --git a/modules/mesh-grants/internal/grants/inputs.go b/modules/mesh-grants/internal/grants/inputs.go index fdb6c03..0634fe5 100644 --- a/modules/mesh-grants/internal/grants/inputs.go +++ b/modules/mesh-grants/internal/grants/inputs.go @@ -34,11 +34,14 @@ const ( ReadBuilds = "mesh-delivery-builder.builds" // ReadAccounts is the vault's bcrypt hashes of the bus accounts it holds, by bus user (ADR 0547 clause 9). ReadAccounts = "mesh-vault.bus-accounts" + // ReadPlacement is the indexer's placement, read from the catalogue (ADR 0549 §2, §3): each node's modules and the seats + // they hold. + ReadPlacement = "mesh-delivery-indexer.placement" ) // Reads is every state the part reads, in the manifest's order. var Reads = []string{ReadAssignments, ReadControllerGrant, ReadModules, ReadManifests, ReadResolved, ReadHave, ReadWants, ReadBuilds, - ReadAccounts} + ReadAccounts, ReadPlacement} // The bus module's tools the part calls, as its manifest's invokes name them: read-only, each. const ( @@ -103,8 +106,12 @@ type resolved struct { type Inputs struct { // Nodes is every machine seen: in a node agent's have or a standing assignment. Nodes []string - // Assigned is each machine's standing assignments, sorted. + // Assigned is each machine's standing assignments and what the catalogue's placement puts there, sorted: while both + // stand (ADR 0549 §5) a module either names keeps its membership. Assigned map[string][]string + // Placed is each node's placement as the indexer read it from the catalogue (ADR 0549 §2); empty before any is read. + // Where it is, a seat held once for the mesh is held where placement says, not wherever a module claiming it runs. + Placed map[string]delivery.Placement // Manifests is each module's manifest at the indexer's trunk, by module. Manifests map[string]catalogue.Manifest // Digests is each module's manifest digest, by module. @@ -230,6 +237,38 @@ func ReadInputs(r Reader, cache *Manifests) Inputs { in.BusNode = a.Node } } + // The catalogue's placement (ADR 0549 §3): what it places is composed beside the assignments, and the seats held once + // for the mesh are held where it says. Not on the bus yet is a standing gap, never a failed read, so the list is still + // composed and sealed from the assignments alone. + if keys, err := r.Keys(ReadPlacement); err != nil { + in.Gaps = append(in.Gaps, Gap{Input: ReadPlacement, Why: "not read yet: the assignments alone place modules (" + err.Error() + ")"}) + } else { + for _, k := range keys { + raw, found, err := r.Get(ReadPlacement, k) + if err != nil || !found { + continue + } + p, err := delivery.DecodePlacement(raw) + node, kerr := delivery.ParsePlacementKey(k) + if err != nil || kerr != nil || p.Node != node { + in.Gaps = append(in.Gaps, Gap{Input: ReadPlacement + " " + k, Why: "not a placement, so the node is composed from its assignments alone"}) + continue + } + if in.Placed == nil { + in.Placed = map[string]delivery.Placement{} + } + in.Placed[node] = p + nodes[node] = true + for _, m := range p.Assigned() { + if !slices.Contains(in.Assigned[node], m) { + in.Assigned[node] = append(in.Assigned[node], m) + } + } + } + } + if len(in.Placed) > 0 { + in.Gaps = slices.DeleteFunc(in.Gaps, func(g Gap) bool { return g.Input == "seat holdings" }) + } haveKeys, err := r.Keys(ReadHave) if err != nil { in.Gaps = append(in.Gaps, Gap{Input: ReadHave, Why: "refused or absent: " + err.Error(), Failed: true}) diff --git a/modules/mesh-grants/internal/grants/records.go b/modules/mesh-grants/internal/grants/records.go index e53beab..e92e11c 100644 --- a/modules/mesh-grants/internal/grants/records.go +++ b/modules/mesh-grants/internal/grants/records.go @@ -7,6 +7,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" ) // Records is what the bus composer composes from, rebuilt from the bus (mesh-controller internal/inventory busrecords.go, @@ -59,6 +60,9 @@ func Records(in Inputs) (buscompose.Records, []Gap) { continue } d := declaredFor(m, seats, declarers) + if len(in.Placed) > 0 { + d.Holds = heldWherePlaced(d.Holds, in.Placed[node], m.Module) + } for _, sr := range catalogue.ReadsGranted(m, here, node) { d.KeyedReads = append(d.KeyedReads, buscompose.KeyedRead{Read: sr.Read, Key: sr.Key}) } @@ -299,3 +303,22 @@ func hearsOf(m catalogue.Manifest, declared map[string]catalogue.Manifest, node } return out, touches, onEvents } + +// heldWherePlaced is a module's holds on a node as the catalogue's placement says them (ADR 0549 §1, §3): a seat held once +// for the mesh is held there only when the node's placement names this module holding it, so a module claiming such a seat +// on two machines holds it on the one placement names, not on both; a seat of every node is held wherever the module runs. +func heldWherePlaced(holds []buscompose.Seat, p delivery.Placement, module string) []buscompose.Seat { + var out []buscompose.Seat + for _, h := range holds { + if h.Scope != "" && h.Scope != catalogue.ScopeMesh { + out = append(out, h) + continue + } + if slices.ContainsFunc(p.Holds, func(x delivery.Holding) bool { + return x.Claim == h.Name && x.Module == module && x.Kind == h.Kind + }) { + out = append(out, h) + } + } + return out +} diff --git a/modules/mesh-grants/module.json b/modules/mesh-grants/module.json index 710c0ce..65d06b0 100644 --- a/modules/mesh-grants/module.json +++ b/modules/mesh-grants/module.json @@ -10,6 +10,7 @@ "mesh-delivery-builder.builds", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", + "mesh-delivery-indexer.placement", "node-engine.have", "mesh-vault.bus-accounts" ],