From 416e270899e71d2de02f6199f1529cb441a0880c Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 10 Nov 2026 02:59:53 +0100 Subject: [PATCH 01/22] =?UTF-8?q?D1:=20the=20assigner,=20builder=20and=20i?= =?UTF-8?q?ndexer=20call=20every=20node's=20manifest-reads=20again=20at=20?= =?UTF-8?q?their=20consumer's=20bind=20(issue=201966,=20ADR=200668=20?= =?UTF-8?q?=C2=A74)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Note 155: a manifest-reads-changed said before the assigner's consumer existed was never heard, and ace's bootstrapped statement stood. Each part registers the engine-reads keeper's AtBind on its hub and calls the verb as a kept call (CallKeptOn). The asks-at-bind half (Q7) is not in this commit. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-assigner/main.go | 6 +++++- .../cmd/mesh-delivery-assigner/pass_test.go | 3 +++ .../cmd/mesh-delivery-assigner/puts.go | 2 ++ .../cmd/mesh-delivery-builder/main.go | 3 ++- .../cmd/mesh-delivery-builder/reads.go | 13 ++++++++++++- .../cmd/mesh-delivery-builder/refused_test.go | 12 ++++++++++++ .../cmd/mesh-delivery-indexer/main.go | 3 ++- .../cmd/mesh-delivery-indexer/reads.go | 12 +++++++++++- 8 files changed, 49 insertions(+), 5 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 9e72e33e..48ced1a4 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -16,6 +16,7 @@ import ( "mesh-delivery-assigner/internal/buckets" "os" "os/signal" + "maps" "slices" "syscall" "time" @@ -315,7 +316,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, // from the node's record where the engine does not answer (the fallback, said in the journal). reads := &enginereads.Keeper{Fallback: w.recordReads, Changed: wake, Logf: logger.Printf} if bus != nil { - reads.Call = bus.CallOn + reads.Call = bus.CallKeptOn // a kept call, called again through a transport fault (ADR 0668 §1) } if kv := kvs[engineReadsBucket]; kv != nil { reads.Put = func(node string, v []byte) error { _, err := kv.Put(ctx, node, v); return err } @@ -570,6 +571,9 @@ func hearReadsChanged(ctx context.Context, js jetstream.JetStream, hub *process. } } err := hub.Handle(enginereads.SubjectPrefix+"*", handle) + // At each bind every node the assigner holds a statement of is asked again: an event said before its consumer existed + // is not delivered to it (issue 1966 note 155: ace's bootstrapped statement stood; ADR 0668 §4). + hub.OnBound(reads.AtBind(func() []string { return slices.Sorted(maps.Keys(reads.Statements())) })) if err == nil { err = hub.Start(ctx, js, account) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index 8d116ce6..ada4fe06 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -316,6 +316,9 @@ func (b *fakeBus) Call(key string, body any) (json.RawMessage, error) { func (b *fakeBus) CallOn(node, key string, body any) (json.RawMessage, error) { return b.Call(node+"/"+key, body) } +func (b *fakeBus) CallKeptOn(node, key string, body any) (json.RawMessage, error) { + return b.CallOn(node, key, body) +} func (b *fakeBus) Emit(eventType string, body any) error { b.events = append(b.events, fmt.Sprintf("%s %+v", eventType, body)) return nil diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go index 492f4219..18540885 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go @@ -41,6 +41,8 @@ const PutEvent = "put" type caller interface { Call(key string, body any) (json.RawMessage, error) CallOn(node, key string, body any) (json.RawMessage, error) + // CallKeptOn is CallOn for a kept call, safe to call again (process.Conn.CallKeptOn; ADR 0668). + CallKeptOn(node, key string, body any) (json.RawMessage, error) Emit(eventType string, body any) error } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 224b428b..24f16218 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -263,7 +263,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro watch.put = gate.Toolchain // What each counted node's engine reads (ADR 0618 §2, ADR 0624 §1): asked through the node-engine seat's verb as the // builder's own account, for every node the assignments names, and kept in the state engine-reads once it is bound. - reads := newEngineReads(calls.CallOn, gate, logger.Printf) + reads := newEngineReads(calls.CallKeptOn, gate, logger.Printf) // The old reference form's window over them (DLV-B9), kept in the state old-form once it is bound; builder_old_form // reads it and drops a retired node by hand. reads.ctx = context.WithoutCancel(ctx) @@ -351,6 +351,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := hub.Handle("mesh.seat.*.event.>", events); err != nil { return err } + hub.OnBound(reads.atBind) // what a node's engine said before the consumer was bound (ADR 0668 §4) if stopped != nil { return stopped } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reads.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reads.go index 4475e766..b014f154 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reads.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reads.go @@ -4,6 +4,7 @@ import ( "context" "slices" "sync" + "time" "github.com/nats-io/nats.go/jetstream" @@ -35,7 +36,7 @@ type engineReads struct { ctx context.Context } -// newEngineReads asks through call (process.Conn.CallOn) and keeps each answer in the state once it is bound. +// newEngineReads asks through call (process.Conn.CallKeptOn, a kept call; ADR 0668) and keeps each answer in the state once it is bound. func newEngineReads(call enginereads.Caller, gate *buildrun.ToolchainGate, logf func(string, ...any)) *engineReads { r := &engineReads{gate: gate, keeper: &enginereads.Keeper{Call: call, Logf: logf}} r.keeper.Changed = r.stepWindow @@ -107,6 +108,16 @@ func (r *engineReads) placed() { r.stepWindow() } +// atBind asks every counted node again when the builder's consumer is bound (the hub's OnBound): a manifest-reads-changed +// said before the consumer existed is not delivered to it, so the statement it would have ended is ended here (novox/hq +// ADR 0668 §4; issue 1966 note 155). +func (r *engineReads) atBind(created time.Time) { + r.mu.Lock() + nodes := slices.Clone(r.counted) + r.mu.Unlock() + r.keeper.AtBind(func() []string { return nodes })(created) +} + // heard takes an event of the builder's consumer on EVENTS: manifest-reads-changed asks its node again, and answers true. func (r *engineReads) heard(m jetstream.Msg, logf func(string, ...any)) bool { node, ours, err := enginereads.NodeOf(m.Subject(), m.Data()) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go index f4d30f13..0994e9e9 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go @@ -414,6 +414,18 @@ func aFailedTrunkBuildIsAFinding(t *testing.T) { said: func(b versions.OldFormClosedBody) { closings = append(closings, b) }, log: func(e builds.BuildLogEntry) { logged = append(logged, e) }} reads := &engineReads{keeper: &enginereads.Keeper{}, window: window, counted: []string{"a", "b"}} + // Issue 1966 note 155, ADR 0668 §4: at the consumer's bind every counted node is called once again. + var boundCalls []string + reads.keeper.Sync = true + reads.keeper.Call = func(node, _ string, _ any) (json.RawMessage, error) { + boundCalls = append(boundCalls, node) + return nil, errors.New("not served") + } + reads.atBind(now) + if !slices.Equal(boundCalls, []string{"a", "b"}) { + t.Fatalf("at the bind called %v, want every counted node once", boundCalls) + } + reads.keeper = &enginereads.Keeper{} newest := []string{forms.FormNeed("oci")} counted := map[string][]string{"a": newest, "b": nil} window.step(context.Background(), []string{"a", "b"}, counted) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 00ef9a3d..b2ae8c70 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -489,7 +489,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // What each counted node's engine reads (ADR 0624 §1), for the mesh-wide hold of a manifest every node reads (ADR 0632 // §2): asked through the node-engine seat's verb as the indexer's own account, for every node the assignments names, // kept in the state engine-reads once it is bound; each change weighs the holds again. - reads := newEngineReads(calls.CallOn, gate, func() { ix.KickHolds(ctx) }, logger.Printf) + reads := newEngineReads(calls.CallKeptOn, gate, func() { ix.KickHolds(ctx) }, logger.Printf) gate.OnPlaced = reads.placed ix.Counted, ix.AskEngines = reads.counted, reads.ask // The holds, new with ADR 0632 (issue 1790's kind), bound when they appear: until then a merge that would hold a @@ -648,6 +648,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro }); err != nil { return err } + hub.OnBound(reads.atBind) // what a node's engine said before the consumer was bound (ADR 0668 §4) versions := func(m jetstream.Msg) { if !heardVersion(ctx, ix, m, logger.Printf) { _ = m.Ack() // nothing else the indexer consumes rides the shared stream diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/reads.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/reads.go index 29a7c890..890a8f10 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/reads.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/reads.go @@ -4,6 +4,7 @@ import ( "context" "slices" "sync" + "time" "github.com/nats-io/nats.go/jetstream" @@ -29,7 +30,7 @@ type engineReads struct { nodes []string } -// newEngineReads asks through call (process.Conn.CallOn) and tells changed after each statement changed. +// newEngineReads asks through call (process.Conn.CallKeptOn, a kept call; ADR 0668) and tells changed after each statement changed. func newEngineReads(call enginereads.Caller, gate *buildrun.ToolchainGate, changed func(), logf func(string, ...any)) *engineReads { return &engineReads{gate: gate, keeper: &enginereads.Keeper{Call: call, Logf: logf, Changed: changed}} } @@ -77,6 +78,15 @@ func (r *engineReads) placed() { } } +// atBind asks every counted node again when the indexer's consumer is bound (the hub's OnBound): a manifest-reads-changed +// said before the consumer existed is not delivered to it (novox/hq ADR 0668 §4; issue 1966 note 155). +func (r *engineReads) atBind(created time.Time) { + r.mu.Lock() + nodes := slices.Clone(r.nodes) + r.mu.Unlock() + r.keeper.AtBind(func() []string { return nodes })(created) +} + // heard takes an event of the indexer's consumer on EVENTS: manifest-reads-changed asks its node again, and answers true. func (r *engineReads) heard(m jetstream.Msg, logf func(string, ...any)) bool { node, ours, err := enginereads.NodeOf(m.Subject(), m.Data()) From 7bdc41c0d3dacfc9bdd771a537a41f1a86d091b3 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 10 Nov 2026 03:01:38 +0100 Subject: [PATCH 02/22] =?UTF-8?q?D5:=20indexer=5Fdrop=5Fpending=20refuses?= =?UTF-8?q?=20a=20merge=20no=20later=20merge=20covers,=20unless=20abandon?= =?UTF-8?q?=20(issue=201966,=20ADR=200633=20=C2=A75,=20ADR=200668=20=C2=A7?= =?UTF-8?q?5)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #650, its repository's head, was dropped on the advice of a line that named a drop where nothing covered it; its changes would never have been taken in. The rule lived only in the record and the advice; the tool now checks it, and the advice names the read again. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-indexer/takein.go | 4 +-- .../internal/index/pending.go | 29 +++++++++++++++---- .../internal/index/pending_test.go | 15 +++++++++- .../internal/index/pending_tools.go | 12 ++++---- .../internal/index/takein.go | 12 ++++++-- 5 files changed, 56 insertions(+), 16 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go index 8585099d..af5f2e8a 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go @@ -20,7 +20,7 @@ type takerIn interface { SeatDefinitionsAgain(ctx context.Context, module string, pull int, head string) index.TakenIn HoldAgain(ctx context.Context, module string, release bool, by string) index.TakenIn TakeInPendingByHand(ctx context.Context, key, by, why string) index.TakenIn - DropPending(ctx context.Context, key, by, why string) index.TakenIn + DropPending(ctx context.Context, key, by, why string, abandon bool) index.TakenIn TakeInMerge(ctx context.Context, repository, merge string, seq uint64, by, why string) index.TakenIn DropTryCandidate(ctx context.Context, id, by, why string) index.TakenIn RecoverTriesByHand(ctx context.Context, by, why string) index.TakenIn @@ -55,7 +55,7 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream asked, r = "hand take-in of a merge", ix.TakeInMerge(ctx, a.MergeRepository, a.Merge, a.Sequence, a.By, a.Why) a.Module = r.Module } else if a.Pending != "" && a.Drop { - asked, r = "drop of a pending merge", ix.DropPending(ctx, a.Pending, a.By, a.Why) + asked, r = "drop of a pending merge", ix.DropPending(ctx, a.Pending, a.By, a.Why, a.Abandon) a.Module = a.Pending } else if a.Pending != "" { asked, r = "hand take-in of what waits", ix.TakeInPendingByHand(ctx, a.Pending, a.By, a.Why) diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index d92c6956..ab039baa 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -760,8 +760,8 @@ func (ix *Indexer) holdLost(ctx context.Context, ready, lost []PendingEntry, rea return nil } for _, g := range lost { - ix.logf("take-in: %s (%s#%d) is not taken in and holds its revision, the forge bridge could not give its message: %s; a newer merge of %s covers it, indexer_drop_pending drops it", - g.Key, g.Forge.Repository, g.Forge.Number, reasons[g.Key], g.Forge.Repository) + ix.logf("take-in: %s (%s#%d) is not taken in and holds its revision, the forge bridge could not give its message: %s; %s reads it again; indexer_drop_pending drops it once a later merge of %s covers it", + g.Key, g.Forge.Repository, g.Forge.Number, reasons[g.Key], TakeInPendingTool, g.Forge.Repository) } if !ix.sayTakeIn(ctx, t) { return nil @@ -1226,7 +1226,7 @@ func (ix *Indexer) TakeInPendingByHand(ctx context.Context, key, by, why string) } take, lost, reasons := ix.loadNewest(ctx, take) if len(lost) > 0 { - r.Answer = fmt.Sprintf("nothing is taken in: the message of %s is not read (%s); a newer merge of its repository covers it, indexer_drop_pending drops it", lost[0].Key, reasons[lost[0].Key]) + r.Answer = fmt.Sprintf("nothing is taken in: the message of %s is not read (%s); asked again, it is read again; indexer_drop_pending drops it once a later merge of its repository covers it", lost[0].Key, reasons[lost[0].Key]) return r } ix.logf("take-in: a hand take-in of %s (%d merges) asked by %s: %s", key, len(take), by, why) @@ -1245,8 +1245,10 @@ func (ix *Indexer) TakeInPendingByHand(ctx context.Context, key, by, why string) // DropPending is the hand drop (indexer_drop_pending, ADR 0633 §5): the one pending merge under key, abandoned // because a later merge replaced it, is removed and nothing of it is taken in. Its wait is said taken, naming the drop; -// the merges left under that wait are said pending again by the evaluation that follows. -func (ix *Indexer) DropPending(ctx context.Context, key, by, why string) TakenIn { +// the merges left under that wait are said pending again by the evaluation that follows. A merge that no later pending +// merge and no taken merge of its repository covers is refused unless abandon (issue 1966: #650, its repository's head, +// was dropped on the advice of a line that named a drop where nothing covered it; ADR 0668 §5). +func (ix *Indexer) DropPending(ctx context.Context, key, by, why string, abandon bool) TakenIn { r := TakenIn{Module: key} if ix.Pending == nil { r.Answer = "the pending state is not on the bus: nothing waits" @@ -1266,6 +1268,23 @@ func (ix *Indexer) DropPending(ctx context.Context, key, by, why string) TakenIn return r } e := entries[at] + if !abandon { + covered := slices.ContainsFunc(entries[at+1:], func(o PendingEntry) bool { return o.Forge.Repository == e.Forge.Repository }) + if !covered { + taken, err := ix.takenAlready(ctx, forgemsg.GroupMerge{Repository: e.Forge.Repository, Number: e.Forge.Number, Merge: e.Forge.Merge}) + if err != nil { + r.Answer = fmt.Sprintf("%s is not dropped: whether a taken merge of %s covers it is not read: %v", key, e.Forge.Repository, err) + return r + } + covered = taken + } + if !covered { + r.Answer = fmt.Sprintf("%s is not dropped: no later merge of %s is pending and its taken merge does not cover %s, so its "+ + "changes would never be taken in (ADR 0633 §5); %s reads its message again, or abandon: true drops it all the same", + key, e.Forge.Repository, e.Forge.Merge, TakeInPendingTool) + return r + } + } if err := ix.Pending.Delete(ctx, e.Key, e.Revision); err != nil { r.Answer = fmt.Sprintf("%s.%s is not removed: %v", PendingState, key, err) return r diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index 0ecaef44..0002e64e 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -679,12 +679,18 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) t.Fatalf("indexer_pending = %+v", answer) } b.said = nil - if r := b.ix.DropPending(ctx, takein.PendingKey(2), "lead@shanks", "replaced by m.3"); !r.Stored { + if r := b.ix.DropPending(ctx, takein.PendingKey(2), "lead@shanks", "replaced by m.3", false); !r.Stored { t.Fatalf("drop: %+v", r) } if len(b.said) < 2 || !b.said[0].Dropped || b.said[0].Key != takein.PendingKey(2) || b.said[0].By != "lead@shanks" || b.said[1].Key != takein.PendingKey(3) { t.Fatalf("drop said %+v: want m.2 taken (dropped, by) and m.3 pending", b.said) } + // Issue 1966 (#650), ADR 0633 §5: the newest pending merge of its repository, which no taken merge covers, is not + // dropped; the answer names the read again. abandon drops it all the same (below). + if r := b.ix.DropPending(ctx, takein.PendingKey(3), "lead@shanks", "lost message", false); r.Stored || + !strings.Contains(r.Answer, TakeInPendingTool) || len(b.keys()) != 1 { + t.Fatalf("a drop nothing covers: %+v, pending %v", r, b.keys()) + } b.said = nil if r := b.ix.TakeInPendingByHand(ctx, takein.PendingKey(3), "lead@shanks", "the named indexer can never run"); !r.Stored { t.Fatalf("hand take-in: %+v", r) @@ -692,6 +698,13 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('d') || len(b.said) != 1 || b.said[0].Kind != takein.TakeInTaken || b.said[0].By != "lead@shanks" { t.Errorf("hand take-in: pending %v, avahi at %s, said %+v", b.keys(), b.commit(t, "avahi"), b.said) } + b.arrive(t, 4, b.merge(23, sha('e'), "tree-avahi-5", "tree-indexer-2")) + if r := b.ix.DropPending(ctx, takein.PendingKey(4), "lead@shanks", "abandoned on the operator's word", true); !r.Stored || len(b.keys()) != 0 { + t.Errorf("an abandoned drop: %+v, pending %v", r, b.keys()) + } + if _, _, err := NewPendingAsk(takein.PendingKey(4), "x", "lead@shanks", false, true, time.Unix(0, 0)); err == nil { + t.Errorf("abandon on a take-in was asked") + } }) t.Run("1585 a merge over the value cap is held by reference, a new indexer with an empty memory takes it in through the forge bridge, and a covered pull request leaves its candidate", func(t *testing.T) { big := func(f forgemsg.Forge) forgemsg.Forge { diff --git a/modules/mesh-delivery-indexer/internal/index/pending_tools.go b/modules/mesh-delivery-indexer/internal/index/pending_tools.go index 189cff8a..4ab76300 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_tools.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_tools.go @@ -21,7 +21,7 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T str := func(args map[string]any, k string) string { s, _ := args[k].(string); return s } hand := func(drop bool) func(map[string]any) (any, error) { return func(args map[string]any) (any, error) { - key, value, err := NewPendingAsk(str(args, "key"), str(args, "why"), stdio.CallerOf(args), drop, now()) + key, value, err := NewPendingAsk(str(args, "key"), str(args, "why"), stdio.CallerOf(args), drop, args["abandon"] == true, now()) if err != nil { return nil, err } @@ -93,11 +93,13 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T Run: hand(false)}, {Name: DropPendingTool, Description: "Drop one pending merge by hand, abandoned because a later merge replaced it (ADR 0633 §5): it is removed from " + - "the pending state and nothing of it is taken in; what waited with it is worked out again. A hand act: who and why are " + - "said in the journal and on the take-in.taken event (dropped).", + "the pending state and nothing of it is taken in; what waited with it is worked out again. A merge no later pending merge " + + "and no taken merge of its repository covers is refused, naming indexer_take_in_pending, which reads its message again; " + + "abandon: true drops it all the same. A hand act: who and why are said in the journal and on the take-in.taken event (dropped).", Input: map[string]any{"type": "object", "required": []string{"key", "why"}, "properties": map[string]any{ - "key": map[string]any{"type": "string", "description": "the pending merge's key, m., as indexer_pending names its merges' wait"}, - "why": map[string]any{"type": "string", "description": "why it is dropped, in plain words"}}}, + "key": map[string]any{"type": "string", "description": "the pending merge's key, m., as indexer_pending names its merges' wait"}, + "why": map[string]any{"type": "string", "description": "why it is dropped, in plain words"}, + "abandon": map[string]any{"type": "boolean", "description": "drop it though no later merge covers it; its changes are then never taken in"}}}, Run: hand(true)}, {Name: TakeInMergeTool, Description: "Take one merge in by hand (ADR 0633 §5, issue 1585): the indexer asks the forge bridge for the merge's merged message " + diff --git a/modules/mesh-delivery-indexer/internal/index/takein.go b/modules/mesh-delivery-indexer/internal/index/takein.go index b20c7006..89505157 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein.go +++ b/modules/mesh-delivery-indexer/internal/index/takein.go @@ -49,6 +49,9 @@ type TakeInAsk struct { // Pending asks the wait under this take-in key to be taken in by hand (indexer_take_in_pending), or with Drop the // one pending merge under it to be dropped (indexer_drop_pending), instead of a module (pending.go, ADR 0633 §5). Pending string `json:"pending,omitempty"` + // Abandon, with Drop, drops a pending merge no later merge covers (ADR 0633 §5 asks a later merge; issue 1966, ADR + // 0668 §5): the hand act says why it is abandoned all the same. + Abandon bool `json:"abandon,omitempty"` // Merge asks the merge of MergeRepository at this commit to be taken in by hand (indexer_take_in_merge): its merged message // is asked of the forge bridge and run through the take-in as a delivered one, under its stream sequence Sequence. Merge string `json:"merge,omitempty"` @@ -137,8 +140,8 @@ func NewMergeAsk(repository, merge string, sequence uint64, why, by string, at t // NewPendingAsk is the value indexer_take_in_pending (drop false) or indexer_drop_pending (drop true) writes: its key // in takeins, `pending..`, and the value. key is a wait's key as indexer_pending names it, for a -// drop one pending merge's. -func NewPendingAsk(key, why, by string, drop bool, at time.Time) (string, []byte, error) { +// drop one pending merge's; abandon drops one no later merge covers. +func NewPendingAsk(key, why, by string, drop, abandon bool, at time.Time) (string, []byte, error) { key = strings.TrimSpace(key) if !strings.HasPrefix(key, "m.") && !(strings.HasPrefix(key, "g.") && !drop) { if drop { @@ -153,6 +156,9 @@ func NewPendingAsk(key, why, by string, drop bool, at time.Time) (string, []byte if drop { act = "drop" } - data, err := json.Marshal(TakeInAsk{V: 1, Pending: key, Drop: drop, Why: strings.TrimSpace(why), By: by, At: at.UTC()}) + if abandon && !drop { + return "", nil, errors.New("abandon is a drop's: indexer_take_in_pending takes a wait in") + } + data, err := json.Marshal(TakeInAsk{V: 1, Pending: key, Drop: drop, Abandon: abandon, Why: strings.TrimSpace(why), By: by, At: at.UTC()}) return "pending." + act + "." + key, data, err } From 3f2482716b9ed5e714f6ccdcb28477dbb8aff597 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 10 Nov 2026 03:10:33 +0100 Subject: [PATCH 03/22] =?UTF-8?q?Q7:=20the=20assigner=20reads=20the=20oper?= =?UTF-8?q?ator-channel=20seat's=20asks=20at=20its=20consumer's=20bind=20a?= =?UTF-8?q?nd=20asks=20again=20each=20held=20asking=20that=20lapsed=20or?= =?UTF-8?q?=20was=20cancelled=20(issue=201966,=20ADR=200668=20=C2=A74)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An ask-ended the teller said before the assigner's consumer existed is not delivered to it (note 155). At each bind the assigner calls the seat verb operator-channel.asks once, a kept call, for the ended asks since its earliest held asking, and writes again each one that ended expired or cancelled, by the ask-ended rules and their compare-and-set. A seat verb: no module named. One invoke added. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-assigner/askended.go | 72 ++++++++++++++++++ .../integration_test.go | 30 +++++++- .../cmd/mesh-delivery-assigner/main.go | 3 +- .../cmd/mesh-delivery-assigner/pass_test.go | 1 + .../cmd/mesh-delivery-assigner/puts.go | 2 + .../internal/assign/asks_test.go | 45 +++++++++++ .../internal/assign/ended.go | 74 +++++++++++++++++++ .../mesh-delivery-assigner/manifest_test.go | 3 +- modules/mesh-delivery-assigner/module.json | 3 +- 9 files changed, 229 insertions(+), 4 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/askended.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/askended.go index e121b991..893232ee 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/askended.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/askended.go @@ -11,6 +11,8 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents" "git.novox.be/novox/mesh-sdk/go/process" + oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" + "mesh-delivery-assigner/internal/assign" "mesh-delivery-assigner/internal/buckets" "mesh-delivery-assigner/internal/puts" @@ -40,6 +42,76 @@ func hearAskEnded(ctx context.Context, js jetstream.JetStream, hub *process.Even }) } +// channelAsksVerb is the operator-channel seat's verb asks, read at the assigner's consumer's bind (a seat verb: no module +// named). +const channelAsksVerb = "seat:" + oc.Seat + "." + oc.VerbAsks + +// readEndedAtBind registers on the hub, before it starts, one read of the operator-channel seat's asks at each bind of the +// assigner's consumer: an ask-ended the teller said before the consumer existed is not delivered to it (issue 1966 note 155, +// ADR 0668 §4), so each held ask whose asking ended there expired or cancelled is written again as that event writes it +// (askAgain, with every rule and its compare-and-set). The read is a kept call (CallKept), made off the hub's start. +func readEndedAtBind(ctx context.Context, js jetstream.JetStream, hub *process.EventHub, asksKV jetstream.KeyValue, bus caller, logf func(string, ...any)) { + if asksKV == nil || bus == nil { + return + } + hub.OnBound(func(created time.Time) { + go func() { + if err := endedAtBind(ctx, js, asksKV, bus, created, time.Now(), logf); err != nil { + logf("the operator channel's ended asks are not read at the bind of %s (%v): an ask that ended there before is asked again when the teller says it ended", created.Format(time.RFC3339), err) + } + }() + }) +} + +// endedAtBind is readEndedAtBind's one read: the held asks, the seat's verb, and each ended asking written again. +func endedAtBind(ctx context.Context, js jetstream.JetStream, asksKV jetstream.KeyValue, bus caller, created, now time.Time, logf func(string, ...any)) error { + keys, err := asksKV.Keys(ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil + } + if err != nil { + return err + } + held := map[string][]byte{} + for _, key := range keys { + if e, err := asksKV.Get(ctx, key); err == nil { + held[key] = e.Value() + } + } + query, ok := assign.ChannelAsksQuery(held) + if !ok { + return nil + } + answer, err := bus.CallKept(channelAsksVerb, query) + if err != nil { + return err + } + ended, cut, err := assign.EndedAtChannel(held, answer) + if err != nil { + return err + } + if cut != "" { + logf("%s at the bind: %s", channelAsksVerb, cut) + } + var putsKV jetstream.KeyValue + if js != nil { + if kv, err := js.KeyValue(ctx, buckets.Puts); err == nil { + putsKV = kv + } + } + for _, e := range ended { + body, err := tellerevents.EncodeEnded(e) + if err == nil { + err = askAgain(ctx, asksKV, putsKV, body, now, logf) + } + if err != nil { + return err + } + } + logf("%s at the bind of %s: %d of %d held asks ended at the channel without an answer", channelAsksVerb, created.Format(time.RFC3339), len(ended), len(held)) + return nil +} + // askAgain writes the ask the teller said ended again under a new asking, by compare-and-set at the revision that holds the // asking that ended (assign.AskedAgain), and says in the journal what it wrote or why it wrote nothing. Only a bus that // could not be read or written is an error, so the event is heard again; a body that does not read is said and dropped. diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go index b4267224..b6e8c91e 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go @@ -1103,7 +1103,7 @@ func TestAHoldWrittenAgainAfterALiftAndARestartIsAskedAgain(t *testing.T) { time.Sleep(1100 * time.Millisecond) // the next hold is written in another second b.answerTo(t, akey, "keep", first.Ask.ID) // an answer to the first hold, which the later hold must not take b.putJSON(t, buckets.Have, "ace.web", failed) - b.start(t) + _, stop = b.startStoppable(t) again, err := decodeAsk(akey, b.get(t, buckets.Asks, akey)) if err != nil { t.Fatal(err) @@ -1111,6 +1111,34 @@ func TestAHoldWrittenAgainAfterALiftAndARestartIsAskedAgain(t *testing.T) { if again.Ask.ID == first.Ask.ID { t.Fatalf("the hold written again is asked under the first id %s", first.Ask.ID) } + // Issue 1966 note 155, ADR 0668 §4: that asking lapses at the operator channel while no assigner runs, so the teller's + // ask-ended is said before the next one's consumer binds; at its bind the next reads the operator-channel seat's asks and + // writes the hold again under a new asking. + stop() + nc, err := nats.Connect(b.srv.ClientURL()) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + for _, subject := range []string{"mesh.seat.operator-channel.tool.asks", "mesh.seat.operator-channel.call.asks.*"} { + if _, err := nc.Subscribe(subject, func(m *nats.Msg) { + answer, _ := json.Marshal(map[string]any{"asks": []map[string]string{{"key": "mesh-delivery-teller." + again.Ask.ID, "state": "expired"}}, "matched": 1}) + reply, _ := json.Marshal(map[string]any{"result": json.RawMessage(answer)}) + _ = m.Respond(reply) + }); err != nil { + t.Fatal(err) + } + } + subscribed(t, nc) + b.start(t) + for deadline := time.Now().Add(15 * time.Second); ; time.Sleep(50 * time.Millisecond) { + if third, err := decodeAsk(akey, b.get(t, buckets.Asks, akey)); err == nil && third.Ask.ID != again.Ask.ID { + break + } + if time.Now().After(deadline) { + t.Fatalf("the hold whose asking %s lapsed before the assigner's bind was not asked again", again.Ask.ID) + } + } b.putJSON(t, buckets.Have, "ace.web", healthy) for deadline := time.Now().Add(15 * time.Second); b.hasKey(buckets.Wants, hkey); time.Sleep(100 * time.Millisecond) { if time.Now().After(deadline) { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 48ced1a4..1f51d4bc 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -12,11 +12,11 @@ import ( "fmt" "io" "log" + "maps" "mesh-delivery-assigner/internal/assignment" "mesh-delivery-assigner/internal/buckets" "os" "os/signal" - "maps" "slices" "syscall" "time" @@ -483,6 +483,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, if err := hearAskEnded(ctx, js, hub, kvs[buckets.Asks], logger.Printf); err != nil { logger.Printf("%s is not heard, so an ask that ends at the operator channel without an answer is not asked again until the assigner starts again: %v", askEndedSubject, err) } + readEndedAtBind(ctx, js, hub, kvs[buckets.Asks], bus, logger.Printf) hearReadsChanged(ctx, js, hub, reads, findings, gate.eventsLive, logger.Printf) } if bus != nil && kvs[buckets.Puts] != nil { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index ada4fe06..f760fc08 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -316,6 +316,7 @@ func (b *fakeBus) Call(key string, body any) (json.RawMessage, error) { func (b *fakeBus) CallOn(node, key string, body any) (json.RawMessage, error) { return b.Call(node+"/"+key, body) } +func (b *fakeBus) CallKept(key string, body any) (json.RawMessage, error) { return b.Call(key, body) } func (b *fakeBus) CallKeptOn(node, key string, body any) (json.RawMessage, error) { return b.CallOn(node, key, body) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go index 18540885..929ecbdb 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go @@ -41,6 +41,8 @@ const PutEvent = "put" type caller interface { Call(key string, body any) (json.RawMessage, error) CallOn(node, key string, body any) (json.RawMessage, error) + // CallKept is Call for a kept call, safe to call again (process.Conn.CallKept; ADR 0668). + CallKept(key string, body any) (json.RawMessage, error) // CallKeptOn is CallOn for a kept call, safe to call again (process.Conn.CallKeptOn; ADR 0668). CallKeptOn(node, key string, body any) (json.RawMessage, error) Emit(eventType string, body any) error diff --git a/modules/mesh-delivery-assigner/internal/assign/asks_test.go b/modules/mesh-delivery-assigner/internal/assign/asks_test.go index 8ac49816..d550f1fa 100644 --- a/modules/mesh-delivery-assigner/internal/assign/asks_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/asks_test.go @@ -28,6 +28,7 @@ func TestTheHoldAskIsValid(t *testing.T) { t.Run("hold", theHoldAskIsValid) t.Run("restore", theRestoreAskIsValid) t.Run("ended at the channel", anAskEndedAtTheChannelIsWrittenAgain) + t.Run("ended at the channel, read at bind", anAskEndedAtTheChannelIsReadAtBind) } // Issue 1966, note 151: an ask the teller says ended at the operator channel without an answer (its event ask-ended) is @@ -122,6 +123,50 @@ func anAskEndedAtTheChannelIsWrittenAgain(t *testing.T) { } } +// Issue 1966 note 155, ADR 0668 §4: an ended ask the teller said before the assigner's consumer existed is not delivered to +// it, so at bind the assigner reads the operator-channel seat's asks: the query reaches back to the earliest held asking, +// and of the answer only the held askings that lapsed or were cancelled are taken, as the teller's event says them; an +// answered or another asker's ask, and one the key no longer holds, are not. +func anAskEndedAtTheChannelIsReadAtBind(t *testing.T) { + asked := time.Date(2026, 11, 9, 22, 0, 0, 0, time.UTC) + holdKey, _ := asks.HoldAskKey("nats", fNew) + restoreKey, _ := asks.RestoreAskKey("novox", "nats", fNew) + backupKey, _ := asks.BackupAskKey("novox", "nats", fNew) + hold := HoldAsk(holdKey, "nats", fNew, "novox", "w", AskingSeconds(asked), asked) + restore := RestoreAsk(restoreKey, "novox", "nats", fNew, "w", AskingSeconds(asked.Add(time.Hour)), asked.Add(time.Hour)) + backup := BackupAsk(backupKey, "novox", "nats", fNew, "sha256:"+strings.Repeat("e", 64), "w", 0, asked.Add(-time.Hour)) // no asking + held := map[string][]byte{} + for key, a := range map[string]asks.Ask{holdKey: hold, restoreKey: restore, backupKey: backup} { + v, err := EncodeAsk(key, a) + if err != nil { + t.Fatal(err) + } + held[key] = v + } + if _, ok := ChannelAsksQuery(nil); ok { + t.Error("with no ask held the channel is asked") + } + q, ok := ChannelAsksQuery(held) + if since, _ := q["since"].(string); !ok || q["state"] != "ended" || since != asked.Add(-time.Hour-time.Minute).Format(time.RFC3339) { + t.Errorf("the query is %v, want the ended asks since %s", q, asked.Add(-time.Hour-time.Minute).Format(time.RFC3339)) + } + at := func(key string, a asks.Ask, state string) string { + return fmt.Sprintf(`{"key":"t.%s","state":%q}`, mustAsk(t, a).ID, state) + } + answer := `{"asks":[` + strings.Join([]string{at(holdKey, hold, occ.OutcomeExpired), at(restoreKey, restore, occ.OutcomeChosen), + at(backupKey, backup, occ.OutcomeCancelled), `{"key":"t.hold-gone","state":"expired"}`, `{"key":"open","state":"open"}`}, ",") + `],"cut":"c"}` + ended, cut, err := EndedAtChannel(held, []byte(answer)) + want := []tellerevents.Ended{{Key: backupKey, Ask: mustAsk(t, backup).ID, Outcome: occ.OutcomeCancelled}, + {Key: holdKey, Ask: mustAsk(t, hold).ID, Outcome: occ.OutcomeExpired}} + slices.SortFunc(want, func(a, b tellerevents.Ended) int { return strings.Compare(a.Key, b.Key) }) + if err != nil || cut != "c" || !slices.Equal(ended, want) { + t.Errorf("EndedAtChannel = %+v, %q, %v; want %+v and the answer's cut", ended, cut, err, want) + } + if _, _, err := EndedAtChannel(held, []byte(`[1]`)); err == nil { + t.Error("an answer that does not read is taken") + } +} + func ptr(a asks.Ask) *asks.Ask { return &a } func theHoldAskIsValid(t *testing.T) { diff --git a/modules/mesh-delivery-assigner/internal/assign/ended.go b/modules/mesh-delivery-assigner/internal/assign/ended.go index 107db2b2..8b5e4e7f 100644 --- a/modules/mesh-delivery-assigner/internal/assign/ended.go +++ b/modules/mesh-delivery-assigner/internal/assign/ended.go @@ -1,10 +1,14 @@ package assign import ( + "encoding/json" "fmt" + "slices" + "strings" "time" occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" + oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks" @@ -71,3 +75,73 @@ func AskedAgain(e tellerevents.Ended, current []byte, put *puts.Put, now time.Ti } return value, "" } + +// ChannelAsksBound is the most bytes of asks the assigner asks the operator-channel seat's verb asks for at its consumer's +// bind: every ended asking of its held asks fits. +const ChannelAsksBound = 1 << 20 + +// ChannelAsksQuery is the arguments of the operator-channel seat's verb asks that answer, at the assigner's consumer's bind, +// every asking of its held asks (key to value, as the asks state holds them) that ended at the channel: the ended asks +// opened since the earliest of them could have been opened, less a minute (issue 1966 note 155, ADR 0668 §4). ok is false +// when no held ask reads, and then nothing is asked. +func ChannelAsksQuery(held map[string][]byte) (query map[string]any, ok bool) { + var since time.Time + for key, value := range held { + a, ask, err := DecodeAsk(key, value) + if err != nil { + continue + } + opened := ExpiresAt(ask).Add(-AskLasts) + if a.Asking >= 2 { + if at := time.Unix(int64(a.Asking), 0); at.Before(opened) { + opened = at + } + } + if !ok || opened.Before(since) { + since, ok = opened, true + } + } + if !ok { + return nil, false + } + return map[string]any{"state": "ended", "since": since.Add(-time.Minute).UTC().Format(time.RFC3339), "bound": ChannelAsksBound}, true +} + +// channelAsks is what the assigner reads of the verb asks' answer: each ask's key (".", the seat's state asks) +// and the state it ended with. +type channelAsks struct { + Asks []struct { + Key string `json:"key"` + State string `json:"state"` + } `json:"asks"` + Cut string `json:"cut"` +} + +// EndedAtChannel is, of the operator-channel seat's answer of its verb asks, each held ask whose asking ended there +// expired or cancelled, as the teller says it in its event ask-ended, so the assigner writes it again by the rules of that +// event (AskedAgain), with its compare-and-set: an ended ask said before the assigner's consumer existed is not delivered +// to it (issue 1966 note 155, ADR 0668 §4). cut is the answer's own cut, said by the caller; an answer that does not read +// is the error. +func EndedAtChannel(held map[string][]byte, answer []byte) (ended []tellerevents.Ended, cut string, err error) { + var got channelAsks + if err := json.Unmarshal(answer, &got); err != nil { + return nil, "", fmt.Errorf("assign: the operator channel's asks do not read: %w", err) + } + byID := map[string]string{} + for key, value := range held { + if _, ask, err := DecodeAsk(key, value); err == nil { + byID[ask.ID] = key + } + } + for _, v := range got.Asks { + if v.State != occ.OutcomeExpired && v.State != occ.OutcomeCancelled { + continue + } + _, id, ok := oc.ParseAsksByAskerIDKey(v.Key) + if key, held := byID[id]; ok && held { + ended = append(ended, tellerevents.Ended{Key: key, Ask: id, Outcome: v.State}) + } + } + slices.SortFunc(ended, func(a, b tellerevents.Ended) int { return strings.Compare(a.Key, b.Key) }) + return ended, got.Cut, nil +} diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index 4438804c..7c0a697c 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -130,7 +130,8 @@ func TestManifest(t *testing.T) { "seat:mesh-catalog.seat-definitions", // ADR 0636 §4: the seat list "seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", // ADR 0565 §9, §11: the seat alone "seat:node-backup.backed-up", "seat:node-backup.now", - "seat:node-engine.manifest-reads"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) { // ADR 0618 §3 + "seat:node-engine.manifest-reads", // ADR 0618 §3 + "seat:operator-channel.asks"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) { // ADR 0668 §4: the ended asks at bind t.Errorf("invokes = %v, want %v", m.Invokes, want) } if want := []string{"assignments-copied", "put", "views.asks", "views.copies", "views.differences", "views.findings", "views.assignments", "views.seat-holders", "views.wants"}; !slices.Equal(m.Emits, want) { diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index 63ae125a..bb459b24 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -57,7 +57,8 @@ "seat:mesh-hand-acts.warranted", "seat:node-backup.backed-up", "seat:node-backup.now", - "seat:node-engine.manifest-reads" + "seat:node-engine.manifest-reads", + "seat:operator-channel.asks" ], "emits": [ "assignments-copied", From c3b240bdfaf79b0812faa1db859c469f2c096747 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 10 Nov 2026 03:14:12 +0100 Subject: [PATCH 04/22] =?UTF-8?q?D4:=20a=20merged=20message=20the=20forge?= =?UTF-8?q?=20bridge=20could=20not=20give=20for=20a=20transport=20fault=20?= =?UTF-8?q?is=20unread,=20not=20lost,=20and=20read=20again=20on=20the=20ne?= =?UTF-8?q?xt=20message=20heard=20on=20the=20forge=20(issue=201966=20note?= =?UTF-8?q?=20153,=20ADR=200668=20=C2=A71,=20=C2=A74)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The indexer's call for a merged message is a kept call (CallKept). A load that met a deadline or no responders keeps the entry as unread (Pending.Unread, indexer_pending's unread), with a line that names the read again rather than a drop; after every message heard on FORGE, of any kind, an unread merge is evaluated again. The bridge's own answer stays lost. indexer_take_in_pending already reads the message again (Q8). Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../mesh-delivery-indexer/bridgemessage.go | 8 +++ .../cmd/mesh-delivery-indexer/main.go | 8 ++- .../internal/index/pending.go | 60 +++++++++++++++---- .../internal/index/pending_test.go | 35 ++++++++++- .../mesh-delivery-indexer/takein/takein.go | 3 + 5 files changed, 100 insertions(+), 14 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/bridgemessage.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/bridgemessage.go index 1bf9196e..f99bb36a 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/bridgemessage.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/bridgemessage.go @@ -34,3 +34,11 @@ func messageVia(call toolCaller) func(ctx context.Context, repository, merge str return a.Message, nil } } + +// unreadAgain reads again, after a message heard on FORGE, a pending merge whose message was unread for a transport fault +// (index.Indexer.UnreadAgain; issue 1966 note 153): the forge is heard again, so the bridge is asked again. A failure is said. +func unreadAgain(ctx context.Context, ix *index.Indexer, logger interface{ Printf(string, ...any) }) { + if err := ix.UnreadAgain(ctx); err != nil { + logger.Printf("take-in: a merge unread for a transport fault is not read again after this message: %v", err) + } +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index b2ae8c70..acd2104c 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -430,7 +430,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro ReadOpen: openVia(calls.Call), Packages: awaitedStore{b: packages, js: js, name: index.PackagesBucket}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call), OwnBuild: ownBuildVia(calls.Call, cred.Node), OpenTries: openTriesVia(calls.Call), OwnBuildState: awaitedStore{b: ownBuildState}, TryRecovery: awaitedStore{b: tryRecovery}, Graph: awaitedStore{b: graph}, SeatDefinitions: awaitedStore{b: seatDefs}, ReadFile: readFileVia(calls.Call), SaySeatDefinition: saySeats, - Message: messageVia(calls.Call), + Message: messageVia(calls.CallKept), // a kept call: a read, called again through a transport fault (ADR 0668 §1) // The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear; // until then every merge is taken in at once, as before. Pending: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.PendingBucket, "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)}, @@ -719,7 +719,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro index.ConfigCandidatesState: awaited(configCandidates), index.PackagesState: awaited(packages), graphState: awaited(graph), index.HeardState: awaited(heard.b), "taken": awaited(taken), "findings": awaited(findings), index.SeatDefinitionsState: awaited(seatDefs)}, errs, logger.Printf) - cc, err := process.HearEvents(cons, func(m jetstream.Msg) { handle(ctx, m, ix, heard, logger); ix.KickFill(ctx) }, logger.Printf) + cc, err := process.HearEvents(cons, func(m jetstream.Msg) { + handle(ctx, m, ix, heard, logger) + ix.KickFill(ctx) + unreadAgain(ctx, ix, logger) + }, logger.Printf) if err != nil { return err } diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index ab039baa..9999039f 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -17,6 +17,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" "git.novox.be/novox/mesh-delivery/sdk/trunk" + "git.novox.be/novox/mesh-sdk/go/busretry" "git.novox.be/novox/mesh-sdk/go/keys" "git.novox.be/novox/mesh-sdk/go/process" @@ -87,6 +88,10 @@ type Pending struct { SaidKind string `json:"said-kind,omitempty"` SaidAt time.Time `json:"said-at,omitzero"` Taking string `json:"taking,omitempty"` + // Unread is why the merged message could not be read for a transport fault (the forge bridge restarting, a deadline; + // ADR 0664's classes): not lost, it is read again on the next message heard on the forge, at the start and by + // indexer_take_in_pending (issue 1966 note 153, ADR 0668 §1). + Unread string `json:"unread,omitempty"` } // PendingEntry is one key of the pending state as read: its key, revision, value and the merged message decoded. For a @@ -244,9 +249,9 @@ func (ix *Indexer) load(ctx context.Context, e PendingEntry) (PendingEntry, erro // loadNewest prepares the ready merges of one revision, in merge order. Of the merges of one repository only the newest is // loaded: applying it writes every module whose manifest differs from what is held, so it covers the older ones, which are // marked Covered and never read. A newest merge the forge bridge cannot give is answered in lost with its reason, by key, and its revision is held (holdLost). -func (ix *Indexer) loadNewest(ctx context.Context, entries []PendingEntry) (out, lost []PendingEntry, reasons map[string]string) { +func (ix *Indexer) loadNewest(ctx context.Context, entries []PendingEntry) (out, lost []PendingEntry, reasons map[string]string, unread map[string]bool) { out = slices.Clone(entries) - reasons = map[string]string{} + reasons, unread = map[string]string{}, map[string]bool{} newest := map[string]bool{} for i := len(out) - 1; i >= 0; i-- { repo := strings.ToLower(out[i].Forge.Repository) @@ -267,12 +272,14 @@ func (ix *Indexer) loadNewest(ctx context.Context, entries []PendingEntry) (out, if err != nil { lost = append(lost, out[i]) reasons[out[i].Key] = err.Error() + // A transport fault is not the bridge's answer: the message is unread, never lost (ADR 0668 §1). + unread[out[i].Key] = busretry.Undelivered(err) || busretry.TimedOut(err) continue } out[i] = full } slices.Reverse(lost) - return out, lost, reasons + return out, lost, reasons, unread } // Waits is the pending state read as what waits (ADR 0633 §3, §4): Ready are the merges no group still being gathered @@ -670,6 +677,24 @@ func (ix *Indexer) TakeInAtStart(ctx context.Context) (Applied, string, error) { return a, words, err } +// UnreadAgain evaluates pending again when a merge in it is unread for a transport fault (Pending.Unread): told after +// every message heard on the forge, of any kind, so a merge whose message the forge bridge could not give while it +// restarted is read again once the forge is heard again, with no timer (issue 1966 note 153, ADR 0668 §4). It reads +// pending once and does nothing when no merge is unread. +func (ix *Indexer) UnreadAgain(ctx context.Context) error { + if ix.Pending == nil || !ix.pendingOnBus(ctx) { + return nil + } + entries, err := ReadPending(ctx, ix.Pending, ix.logf) + if err != nil { + return err + } + if !slices.ContainsFunc(entries, func(e PendingEntry) bool { return e.Value.Unread != "" }) { + return nil + } + return ix.PendingAgain(ctx) +} + // PendingAgain evaluates pending again after something a fingerprint is computed from changed, a toolchain report // (ADR 0633 §1, §3): a revision that now names the running build is taken in, and one that names another build is held // for that one. It does nothing while the pending state is not on the bus or the build this program runs is not known. @@ -722,9 +747,9 @@ func (ix *Indexer) evaluate(ctx context.Context, own string, start bool) (Applie // Of each repository's merges only the newest is read (from memory, else asked of the forge bridge: a pending value // keeps a reference); the older ones are covered by it. A newest merge that cannot be read is said and // holds its revision: nothing of it is taken in silently. - ready, lost, reasons := ix.loadNewest(ctx, w.Ready) + ready, lost, reasons, unread := ix.loadNewest(ctx, w.Ready) if len(lost) > 0 { - return Applied{}, ix.waitingLine(w), ix.holdLost(ctx, w.Ready, lost, reasons) + return Applied{}, ix.waitingLine(w), ix.holdLost(ctx, w.Ready, lost, reasons, unread) } names, dry, err := ix.computeOver(ctx, ready) if err != nil { @@ -743,8 +768,9 @@ func (ix *Indexer) evaluate(ctx context.Context, own string, start bool) (Applie // holdLost holds a revision whose newest merge of a repository cannot be read (the forge bridge's stream no longer holds // it, or the bus refuses the read): said once on take-in.pending with the lost keys (TakeIn.Lost), and on each lost entry // (Pending.Lost, the reason, which indexer_pending shows), until a newer merge of its repository covers it or -// indexer_drop_pending drops it. Nothing of the revision is taken in meanwhile. -func (ix *Indexer) holdLost(ctx context.Context, ready, lost []PendingEntry, reasons map[string]string) error { +// indexer_drop_pending drops it. A merge unread for a transport fault (unread) is kept as Pending.Unread, not Lost, and read +// again on the next message heard on the forge (UnreadAgain). Nothing of the revision is taken in meanwhile. +func (ix *Indexer) holdLost(ctx context.Context, ready, lost []PendingEntry, reasons map[string]string, unread map[string]bool) error { t := revisionWait(takein.TakeInPending, ready, "") for _, g := range lost { t.Lost = append(t.Lost, g.Key) @@ -752,7 +778,7 @@ func (ix *Indexer) holdLost(ctx context.Context, ready, lost []PendingEntry, rea said := true for _, e := range ready { isLost := slices.Contains(t.Lost, e.Key) - if e.Value.Said != t.Key || e.Value.SaidKind != takein.TakeInPending || (isLost && e.Value.Lost == "") { + if e.Value.Said != t.Key || e.Value.SaidKind != takein.TakeInPending || (isLost && (unread[e.Key] && e.Value.Unread == "" || !unread[e.Key] && e.Value.Lost == "")) { said = false } } @@ -760,6 +786,11 @@ func (ix *Indexer) holdLost(ctx context.Context, ready, lost []PendingEntry, rea return nil } for _, g := range lost { + if unread[g.Key] { + ix.logf("take-in: %s (%s#%d) is not taken in and holds its revision, its message is unread for a transport fault: %s; it is read again on the next message heard on the forge, at the start, and by %s", + g.Key, g.Forge.Repository, g.Forge.Number, reasons[g.Key], TakeInPendingTool) + continue + } ix.logf("take-in: %s (%s#%d) is not taken in and holds its revision, the forge bridge could not give its message: %s; %s reads it again; indexer_drop_pending drops it once a later merge of %s covers it", g.Key, g.Forge.Repository, g.Forge.Number, reasons[g.Key], TakeInPendingTool, g.Forge.Repository) } @@ -773,8 +804,12 @@ func (ix *Indexer) holdLost(ctx context.Context, ready, lost []PendingEntry, rea } } for _, g := range lost { - reason := reasons[g.Key] - ix.markSaid(ctx, []PendingEntry{g}, &t, func(p *Pending) { p.Lost = reason }) + reason, isUnread := reasons[g.Key], unread[g.Key] + ix.markSaid(ctx, []PendingEntry{g}, &t, func(p *Pending) { + if p.Lost, p.Unread = reason, ""; isUnread { + p.Lost, p.Unread = "", reason + } + }) } ix.markSaid(ctx, kept, &t, nil) return nil @@ -1179,6 +1214,9 @@ func waitOf(entries []PendingEntry, what, behind string) takein.PendingWait { if e.Value.Lost != "" { pw.Lost = append(pw.Lost, e.Key) } + if e.Value.Unread != "" { + pw.Unread = append(pw.Unread, e.Key) + } if e.Value.Refused != "" { pw.Refused = e.Value.Refused } @@ -1224,7 +1262,7 @@ func (ix *Indexer) TakeInPendingByHand(ctx context.Context, key, by, why string) r.Answer = fmt.Sprintf("nothing waits under %s: indexer_pending names the waits", key) return r } - take, lost, reasons := ix.loadNewest(ctx, take) + take, lost, reasons, _ := ix.loadNewest(ctx, take) if len(lost) > 0 { r.Answer = fmt.Sprintf("nothing is taken in: the message of %s is not read (%s); asked again, it is read again; indexer_drop_pending drops it once a later merge of its repository covers it", lost[0].Key, reasons[lost[0].Key]) return r diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index 0002e64e..c0bd87e8 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -17,8 +17,10 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" "git.novox.be/novox/mesh-delivery/sdk/trunk" "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/busretry" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/process" + "github.com/nats-io/nats.go" ) // takeInBed is an indexer holding the catalogue's avahi and the indexer's own module, both with a fingerprint, its @@ -554,6 +556,37 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('e') { t.Errorf("after a lost start: pending %v, avahi at %s: want all taken in by the next merged message", b.keys(), b.commit(t, "avahi")) } + // Issue 1966 note 153, ADR 0668 §1, §4: a transport fault (the bridge restarting past the call's bound) leaves the + // newest merge unread, not lost, and indexer_pending says so; the next message heard on the forge, of any kind, reads + // it again. The bridge's own answer (it holds no such merge) is lost, as before, and a heard message does not ask it. + for _, c := range []struct { + name string + fault error + unread bool + }{ + {"a deadline", fmt.Errorf("the bridge: %w", nats.ErrTimeout), true}, + {"no responders", fmt.Errorf("the bridge: %w", nats.ErrNoResponders), true}, + {"the bridge's answer", &busretry.Answered{Words: "no merge at that commit"}, false}, + } { + b = held() + answers := b.ix.Message + b.ix.held = nil + b.ix.Message = func(context.Context, string, string, int) ([]byte, error) { return nil, c.fault } + if _, _, err := b.ix.TakeInAtStart(ctx); err != nil || len(b.keys()) != 2 { + t.Fatalf("%s at the start: err %v, pending %v", c.name, err, b.keys()) + } + waits := PendingWaits(mustReadPending(t, b)) + if len(waits) != 1 || (len(waits[0].Unread) == 1) != c.unread || (len(waits[0].Lost) == 1) == c.unread { + t.Fatalf("%s: indexer_pending %+v: want unread %v, lost %v", c.name, waits, c.unread, !c.unread) + } + b.ix.Message = answers + if err := b.ix.UnreadAgain(ctx); err != nil { + t.Fatal(err) + } + if taken := len(b.keys()) == 0 && b.commit(t, "avahi") == sha('d'); taken != c.unread { + t.Errorf("%s, a message heard after: pending %v, avahi at %s: want taken in %v", c.name, b.keys(), b.commit(t, "avahi"), c.unread) + } + } }) t.Run("T5 two take-ins of one pending revision: the one that loses the claim writes nothing", func(t *testing.T) { b := newTakeInBed(t) @@ -643,7 +676,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) key, _ := ToolchainKey(node, "go") x.PutToolchain("mesh-build-go", key, toolchainValue(t, "go", "go1.27.2", "1"), false) } - ready, _, _ := b.ix.loadNewest(ctx, WaitsOf(entries).Ready) + ready, _, _, _ := b.ix.loadNewest(ctx, WaitsOf(entries).Ready) next, _, err := x.computeOver(ctx, ready) if err != nil || next == "" || next == b.said[0].WaitsFor { t.Fatalf("the build under the next toolchain: %q %v (the revision names %s)", next, err, b.said[0].WaitsFor) diff --git a/modules/mesh-delivery-indexer/takein/takein.go b/modules/mesh-delivery-indexer/takein/takein.go index 645535c2..59141f58 100644 --- a/modules/mesh-delivery-indexer/takein/takein.go +++ b/modules/mesh-delivery-indexer/takein/takein.go @@ -243,6 +243,9 @@ type PendingWait struct { SaidAt time.Time `json:"said-at,omitzero"` // Lost are the keys of its merges whose message the forge bridge's stream no longer holds (TakeIn.Lost). Lost []string `json:"lost,omitempty"` + // Unread are the keys of its merges whose message is unread for a transport fault: read again on the next message + // heard on the forge, at the start and by indexer_take_in_pending, never dropped for it (issue 1966, ADR 0668 §1). + Unread []string `json:"unread,omitempty"` // Refused is why the manifest of the indexer it names was refused (TakeIn.Refused). Refused string `json:"refused,omitempty"` } From f02a740b10dc471f96a953106317d85d0fba69da Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 10 Nov 2026 03:16:28 +0100 Subject: [PATCH 05/22] =?UTF-8?q?D2:=20assigner=5Fask=5Fagain=20asks=20one?= =?UTF-8?q?=20of=20the=20assigner's=20asks=20of=20the=20operator=20again?= =?UTF-8?q?=20by=20hand=20(issue=201966,=20ADR=200668=20=C2=A75)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The recovery of an ask that ended at the channel unheard needed a restart. The tool writes the key again under a new asking by the ask-ended rules (a proposal's authorising ask and a put's record ask are refused), refuses an asking the operator answered, naming the answer, and records the hand act through the seat mesh-hand-acts. One tool added to the manifest. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../mesh-delivery-assigner/askagain_tool.go | 100 ++++++++++++++++++ .../cmd/mesh-delivery-assigner/pass_test.go | 26 +++++ .../cmd/mesh-delivery-assigner/tools.go | 1 + .../mesh-delivery-assigner/manifest_test.go | 4 +- modules/mesh-delivery-assigner/module.json | 3 +- 5 files changed, 131 insertions(+), 3 deletions(-) create mode 100644 modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/askagain_tool.go diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/askagain_tool.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/askagain_tool.go new file mode 100644 index 00000000..762a44b1 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/askagain_tool.go @@ -0,0 +1,100 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents" + stdio "git.novox.be/novox/mesh-sdk/go" + + "mesh-delivery-assigner/internal/assign" + "mesh-delivery-assigner/internal/buckets" + "mesh-delivery-assigner/internal/puts" +) + +// askAgainTool asks one of the assigner's asks again by hand (issue 1966, ADR 0668 §5): the recovery of an ask that ended at +// the operator channel without an answer when the event that would have asked it again was not heard. +const askAgainTool = "assigner_ask_again" + +// asksState and answersState are the assigner's asks and the teller's answers, as the runtime names them. +const ( + asksState = "asks" + answersState = buckets.TellerModule + ".answers" +) + +// askAgainByHand writes the ask at key again under a new asking, by the rules the teller's ask-ended writes it by (assign.AskedAgain: +// a proposal's authorising ask and a put's record ask are not), so the teller asks it again; an asking the operator answered +// is refused, naming the answer. The act is recorded through the seat mesh-hand-acts. +func askAgainByHand(st store, ask func(string, any) (json.RawMessage, error), key, why, by string, now time.Time) (any, error) { + switch { + case key == "": + return nil, errors.New("name the ask's key, as assigner_read_asks lists it") + case why == "": + return nil, errors.New("a hand ask again says why") + case by == "" || by == stdio.Unproven: + return nil, errors.New("a hand ask again names who asks it, and the runtime proved no caller") + } + current, _, found, err := st.get(asksState, key) + if err != nil { + return nil, err + } + if !found { + return nil, fmt.Errorf("the assigner holds no ask under %s: assigner_read_asks lists them", key) + } + _, routerAsk, err := assign.DecodeAsk(key, current) + if err != nil { + return nil, err + } + if raw, _, found, err := st.get(answersState, key); err == nil && found { + if a, err := meshdeliveryclient.DecodeAnswer(raw); err == nil && strings.HasPrefix(a.Warrant, routerAsk.ID+"@") { + return nil, fmt.Errorf("the asking %s was answered %q (%s): it is not asked again", routerAsk.ID, a.Option, a.Words) + } + } + var put *puts.Put + if k, err := asks.ParseAnyAskKey(key); err == nil && k.Kind == asks.AskKindRecord { + if pkey, err := puts.Key(k.Node, k.Module); err == nil { + if raw, _, found, err := st.get(putsState, pkey); err == nil && found { + if p, err := puts.Decode(raw); err == nil { + put = &p + } + } + } + } + value, refused := assign.AskedAgain(tellerevents.Ended{Key: key, Ask: routerAsk.ID, Outcome: occ.OutcomeCancelled}, current, put, now) + if refused != "" { + return nil, errors.New(refused) + } + if err := st.put(asksState, key, value); err != nil { + return nil, fmt.Errorf("the ask %s is not written again: %w", key, err) + } + _, next, _ := assign.DecodeAsk(key, value) + taken, rerr := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: askAgainTool + " " + key, Why: why + " (" + by + ")", Cause: "ask-again"}) + recorded := taken.Log + if rerr != nil { + recorded = "not recorded: " + rerr.Error() + } + return map[string]any{"key": key, "ended": routerAsk.ID, "asked": next.ID, "by": by, "hand-act": recorded}, nil +} + +// askAgainTools is the tool over the runtime's states. +func askAgainTools(st store, ask func(string, any) (json.RawMessage, error), now func() time.Time) []stdio.Tool { + return []stdio.Tool{{Name: askAgainTool, Description: "Ask one of the assigner's asks of the operator again (issue 1966): the ask " + + "under key is written again under a new asking, which the teller asks on the operator channel, as when the channel says it " + + "ended without an answer. Refused for an asking the operator answered, a proposal's authorising ask (the seat's verb again " + + "asks it) and a record ask a put asked. Recorded as a hand act.", + Input: schema(map[string]string{"key": "the ask's key, as assigner_read_asks lists it", "why": "why it is asked again"}, + []string{"key", "why"}), + Run: func(a map[string]any) (any, error) { + key, _ := a["key"].(string) + why, _ := a["why"].(string) + return askAgainByHand(st, ask, key, why, stdio.CallerOf(a), now()) + }}} +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index f760fc08..64b17792 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + stdio "git.novox.be/novox/mesh-sdk/go" "mesh-delivery-assigner/internal/buckets" "mesh-delivery-assigner/internal/puts" "slices" @@ -127,6 +128,31 @@ func TestAHoldAskedAgainAfterALiftAndARestartHasANewID(t *testing.T) { if a2.Ask.ID == a1.Ask.ID { t.Errorf("the ask written again after a lift and a restart has the id %s of the first", a1.Ask.ID) } + // Issue 1966, ADR 0668 §5: assigner_ask_again writes the hold again under a new asking and records the hand act; the + // asking the operator answered is refused, naming the answer; a proposal's authorising ask is refused by its rule. + answersKV := &memKV{values: map[string][]byte{}} + st := memStore{asksState: asksKV, answersState: answersKV} + var recorded []string + record := func(key string, body any) (json.RawMessage, error) { + raw, _ := json.Marshal(body) + recorded = append(recorded, key+" "+string(raw)) + return json.RawMessage(`{}`), nil + } + now := time.Date(2026, 10, 25, 3, 0, 0, 0, time.UTC) + out, err := askAgainByHand(st, record, key, "the channel lost it", "lead@shanks", now) + a3, derr := decodeAsk(key, asksKV.values[key]) + if err != nil || derr != nil || a3.Ask.ID == a2.Ask.ID || len(recorded) != 1 || !strings.Contains(recorded[0], askAgainTool+" "+key) { + t.Fatalf("ask again: %v, %v; asked %s (was %s), recorded %v", out, err, a3.Ask.ID, a2.Ask.ID, recorded) + } + answersKV.values[key], _ = mdc.EncodeAnswer(mdc.Answer{Option: "keep", Warrant: a3.Ask.ID + "@2026-10-25T03:01:00Z", Words: "the operator"}) + if _, err := askAgainByHand(st, record, key, "again", "lead@shanks", now); err == nil || !strings.Contains(err.Error(), "keep") { + t.Errorf("an answered asking is asked again: %v", err) + } + for _, c := range []struct{ key, by, why string }{{key, stdio.Unproven, "a caller"}, {"hold.none", "lead@shanks", "no ask"}} { + if _, err := askAgainByHand(st, record, c.key, "w", c.by, now); err == nil { + t.Errorf("%s: asked again", c.why) + } + } } // 1906/H4: an answer counts only for the asking it answered: an answer to an earlier hold of the same fingerprint, or to an ask diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go index 877b6c1e..d81e9acf 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go @@ -404,6 +404,7 @@ func serveTools() error { tools = append(tools, viewTools(os.Getenv, keptSource, time.Now)...) // What each node's engine reads, as the assigner uses it (ADR 0618 §3): its own state engine-reads, asked again through it. tools = append(tools, enginereads.Tool(engineReadsTool, "the assigner", enginereads.KeptState(stdio.State(engineReadsState)), false, time.Now)) + tools = append(tools, askAgainTools(runtimeStore{}, stdio.Ask, time.Now)...) return stdio.Serve("", tools) } diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index 7c0a697c..c1ceeaea 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -96,8 +96,8 @@ func TestManifest(t *testing.T) { // which lists and asks again what each node's engine reads. if !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts", "assignments", "seat-holders", "wants", "assigner_read_assignments", "assigner_read_seat_holders", "assigner_read_wants", "assigner_read_findings", "assigner_read_asks", "assigner_read_differences", "assigner_read_copies", - "assigner_first_views", "assigner_first_view_again", "assigner_engine_reads", "assigner_seat_list", "assigner_seat_list_again"}) { - t.Errorf("the assigner serves tools %v, want its three put tools, three first views, six read tools, assigner_engine_reads and the seat list's two (ADR 0636 §7)", m.Tools) + "assigner_first_views", "assigner_first_view_again", "assigner_engine_reads", "assigner_seat_list", "assigner_seat_list_again", "assigner_ask_again"}) { + t.Errorf("the assigner serves tools %v, want its three put tools, three first views, six read tools, assigner_engine_reads, the seat list's two (ADR 0636 §7) and assigner_ask_again (ADR 0668 §5)", m.Tools) } if m.RunsAs != m.Module || m.SecretsOwner != m.RunsAs { t.Errorf("runs-as = %q: the bundle runs on the assigner's own account (ADR 0621 §1)", m.RunsAs) diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index bb459b24..e594b25c 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -89,7 +89,8 @@ "assigner_first_view_again", "assigner_engine_reads", "assigner_seat_list", - "assigner_seat_list_again" + "assigner_seat_list_again", + "assigner_ask_again" ], "state": [ { "name": "assignments", "history": 64 }, From 3076aabbe0fa6faa5a15e46429797581a73deb2f Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 10 Nov 2026 03:17:50 +0100 Subject: [PATCH 06/22] =?UTF-8?q?D3:=20the=20teller=20keeps=20an=20assigne?= =?UTF-8?q?r=20ask=20whose=20publish=20failed=20waiting,=20and=20asks=20it?= =?UTF-8?q?=20at=20the=20next=20end=20and=20at=20its=20start=20(issue=2019?= =?UTF-8?q?66,=20ADR=200668=20=C2=A71)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A failed publish deleted the kept record for "the next writing of the key", which nothing made. The ask now joins the teller's wait for room, kept on the bus, which asks again at the next end and at the start. The gate's asks are left as they are (ADR 0666 §6 replaces them). Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-teller/assignerasks.go | 9 ++++---- .../mesh-delivery-teller/assignerasks_test.go | 23 +++++++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/assignerasks.go b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/assignerasks.go index 7e5dd9d4..e5f7ab28 100644 --- a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/assignerasks.go +++ b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/assignerasks.go @@ -191,10 +191,11 @@ func (t *Teller) raiseAssignerAsk(key string) { return } if err := t.Asker.Ask(a.Ask); err != nil { - t.logf("%s: the ask %s was not asked: %v", key, id, err) - if kept, _, _ := t.keptOf(id); open(kept) { - _ = t.Store.Delete(keptKey(id)) // kept but not published: forgotten, so the next writing of the key asks it - } + // Kept but not published (a transport fault, the router restarting): nothing writes the key again, so the ask is not + // forgotten but waits, kept on the bus, to be asked again at the next end and at the teller's start (issue 1966, + // ADR 0668 §1). Before, the record was deleted for "the next writing of the key", which never came. + t.logf("%s: the ask %s was not asked: %v; it waits to be asked again at the next end and at the start", key, id, err) + t.unheard(id) return } t.assignerAsked(id) diff --git a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/assignerasks_test.go b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/assignerasks_test.go index 8420dcbc..1604f9df 100644 --- a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/assignerasks_test.go +++ b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/assignerasks_test.go @@ -1,6 +1,7 @@ package main import ( + "errors" "slices" "encoding/json" @@ -34,6 +35,28 @@ func TestAnAssignerAskIsRaisedOnTheOperatorChannelOnce(t *testing.T) { if got.ID != ask.ID || occ.Digest(got) != occ.Digest(ask) { t.Errorf("the router's ask %+v differs from the assigner's", got) } + // Issue 1966, ADR 0668 §1: a publish that fails (the router restarting) is not forgotten: the ask waits, kept, and is + // asked at the teller's next start (or the next end), though nothing writes the assigner's key again. + b = newBed(t) + assigner, _ = withView(b) + assigner[key] = value + publish, failed := b.client.Publish, false + b.client.Publish = func(subject string, body any, id string) error { + if subject == oc.AcceptSubjectAsk(tellerModule) && !failed { + failed = true + return errors.New("nats: no responders available for request") + } + return publish(subject, body, id) + } + b.t.Publish = b.client.Publish + b.t.OnAssignerAsk(key) + if asked := b.askedOf(oc.AcceptSubjectAsk(tellerModule)); !failed || len(asked) != 0 { + t.Fatalf("published %d asks through a failing publish", len(asked)) + } + b.t.OnCurrent() + if asked := b.askedOf(oc.AcceptSubjectAsk(tellerModule)); len(asked) != 1 { + t.Errorf("after a failed publish and a start, published %d asks, want one; log:\n%s", len(asked), b.logs.String()) + } } // 1906/H1: without the view's bucket the assigner's asks are still raised. From 0d16381b280cb8ca3d0c5c7fccdcdf8dd0470075 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 10 Nov 2026 03:30:37 +0100 Subject: [PATCH 07/22] =?UTF-8?q?D7:=20the=20builder's=20ask=20put=20is=20?= =?UTF-8?q?a=20kept=20call,=20a=20put=20or=20note=20a=20transport=20fault?= =?UTF-8?q?=20kept=20from=20the=20key=20fails=20it=20in=20that=20fault's?= =?UTF-8?q?=20words,=20and=20builder=5Fbuild=5Fagain=20takes=20a=20key=20l?= =?UTF-8?q?eft=20building=20(issue=201966,=201748=20note=2027,=20ADR=20066?= =?UTF-8?q?8=20=C2=A71,=20=C2=A72,=20=C2=A75)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The put carries the ask's id as its message id, so it is made again through a deadline and no responders within the bound; a put or a note that still meets one fails the key as a transport fault, which the builder does not retake by itself (ADR 0664 §3) and the hand tool does. A key left building with no look holding it (its end never written: "claimed", no ask) is taken as it stands by builder_build_again, from another builder too. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../mesh-delivery-builder/buildrun/asks.go | 31 ++++++++-- .../buildrun/asks_test.go | 61 +++++++++++++++++++ .../cmd/mesh-delivery-builder-tools/main.go | 9 +-- .../internal/build/buildagain.go | 15 ++++- .../internal/build/claim_test.go | 7 +++ .../internal/build/run.go | 13 +++- .../internal/build/run_test.go | 37 ++++++++++- 7 files changed, 157 insertions(+), 16 deletions(-) diff --git a/modules/mesh-delivery-builder/buildrun/asks.go b/modules/mesh-delivery-builder/buildrun/asks.go index 5e88371c..d9a0c35b 100644 --- a/modules/mesh-delivery-builder/buildrun/asks.go +++ b/modules/mesh-delivery-builder/buildrun/asks.go @@ -14,6 +14,7 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-delivery/sdk/buildstep" + "git.novox.be/novox/mesh-sdk/go/busretry" ) // **An asker puts an ask on its build seat's stream and waits for its step** (novox/hq ADR 0525 §1, §6; issue 1887 A4, @@ -98,7 +99,14 @@ func (k Asker) dropped(a buildstep.StepAsk, why string) { } } -// Put puts a on its stream: the seat's accept subject of its language and kind. +// PutTry is how long one publish of an ask waits for its stream's answer before it is made again (Put). +var PutTry = 10 * time.Second + +// Put puts a on its stream: the seat's accept subject of its language and kind. The put is a kept call (novox/hq ADR 0668 +// §1, §2; issue 1966, issue 1748 note 27): it carries the ask's id as its message id, so the stream takes it once, and it +// is made again through every transport fault (a deadline, no responders) within busretry.Bound, each try bounded by +// PutTry, until ctx ends. A put that still meets one is an error opening with buildstep.TransportFaultWords, so the key it +// fails is one the builder does not take again by itself (ADR 0664 §3) and builder_build_again does. func (k Asker) Put(ctx context.Context, a buildstep.StepAsk) error { raw, err := buildstep.EncodeStepAsk(a) if err != nil { @@ -108,10 +116,25 @@ func (k Asker) Put(ctx context.Context, a buildstep.StepAsk) error { if err != nil { return err } - if err := k.Publisher.PublishAsk(ctx, subject, a.ID, raw); err != nil { - return fmt.Errorf("putting the %s ask %s: %w", a.Kind, a.ID, err) + _, err = busretry.DoRepeatable("putting the "+a.Kind+" ask "+a.ID, func() (struct{}, error) { + if err := ctx.Err(); err != nil { + return struct{}{}, fmt.Errorf("the put was given up: %v", err) // not a fault to wait out: the build ended + } + try, cancel := context.WithTimeout(ctx, PutTry) + defer cancel() + err := k.Publisher.PublishAsk(try, subject, a.ID, raw) + if err != nil && ctx.Err() != nil { + return struct{}{}, fmt.Errorf("the put was given up: %v", err) + } + return struct{}{}, err + }, nil) + switch { + case err == nil: + return nil + case busretry.TimedOut(err) || busretry.Undelivered(err): + return fmt.Errorf("%sthe put of the %s ask %s: %w", buildstep.TransportFaultWords, a.Kind, a.ID, err) } - return nil + return fmt.Errorf("putting the %s ask %s: %w", a.Kind, a.ID, err) } // Wait waits for the step of an ask already put (a builder that started again takes over its own asks with it, never diff --git a/modules/mesh-delivery-builder/buildrun/asks_test.go b/modules/mesh-delivery-builder/buildrun/asks_test.go index 29406198..a460f924 100644 --- a/modules/mesh-delivery-builder/buildrun/asks_test.go +++ b/modules/mesh-delivery-builder/buildrun/asks_test.go @@ -5,12 +5,16 @@ import ( "encoding/json" "errors" "fmt" + "slices" "strings" "sync" "testing" "time" + "github.com/nats-io/nats.go" + "git.novox.be/novox/mesh-delivery/sdk/buildstep" + "git.novox.be/novox/mesh-sdk/go/busretry" "git.novox.be/novox/mesh-sdk/go/health" ) @@ -41,6 +45,39 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) { if _, err := k.Ask(context.Background(), goAsk("mvl0frv18d1958")); err == nil || !strings.Contains(err.Error(), "shanks refused: go 1.27 installed") { t.Fatalf("a refused step: %v", err) } + // Issue 1966 (ADR 0668 §1, §2; 1748 note 27): the put is a kept call under the ask's id, made again through a deadline and + // no responders until the stream takes it; a refusal of the stream is not made again; a put that meets a transport fault + // past the bound says it in buildstep.TransportFaultWords, so the key it fails is the hand tool's to build again. + t.Cleanup(busretry.Waits(time.Second, time.Millisecond, time.Millisecond)) + for name, c := range map[string]struct { + faults []error + tries int + transport bool + }{ + "a deadline, then no responders, then taken": {faults: []error{context.DeadlineExceeded, nats.ErrNoResponders}, tries: 3}, + "refused by the stream": {faults: []error{errors.New("maximum messages exceeded")}, tries: 1}, + "a deadline past the bound": {faults: slices.Repeat([]error{nats.ErrTimeout}, 10000), transport: true}, + } { + p := &faultyPuts{faults: c.faults, inner: &FakeAsks{}} + restore := func() {} + if c.transport { + restore = busretry.Waits(20*time.Millisecond, time.Millisecond, time.Millisecond) // a bound this row passes + } + err := (Asker{Publisher: p, Waiter: p.inner}).Put(context.Background(), goAsk("mvl0frv18d1960")) + restore() + switch { + case c.transport: + if err == nil || !strings.HasPrefix(err.Error(), buildstep.TransportFaultWords) { + t.Errorf("%s: %v, want it opens with the transport fault's words", name, err) + } + case c.tries == 1: + if err == nil || p.tries != 1 || strings.Contains(err.Error(), buildstep.TransportFaultWords) { + t.Errorf("%s: %v after %d tries, want one try and no transport fault", name, err, p.tries) + } + case err != nil || p.tries != c.tries || len(p.inner.Asks()) != 1 || len(p.ids) != c.tries || p.ids[0] != p.ids[len(p.ids)-1]: + t.Errorf("%s: %v after %d tries under %v, %d asks taken, want %d tries under one id, one ask", name, err, p.tries, p.ids, len(p.inner.Asks()), c.tries) + } + } // Issue 1748 (ADR 0664 §3): a run that ended on a transport fault is no end; the next run's end is the answer, in either // arrival order; three runs then failed say a transport fault on 3 runs with each node; a bus that never gives the ask // again ends the wait saying so. @@ -320,3 +357,27 @@ type failingDrops struct{} func (failingDrops) DropAsk(context.Context, buildstep.StepAsk, string) error { return errors.New("no holder answered") } + +// faultyPuts fails its first puts with faults, then puts through inner (issue 1966). +type faultyPuts struct { + mu sync.Mutex + faults []error + tries int + ids []string + inner *FakeAsks +} + +func (p *faultyPuts) PublishAsk(ctx context.Context, subject, id string, data []byte) error { + p.mu.Lock() + p.tries++ + p.ids = append(p.ids, id) + var err error + if len(p.faults) > 0 { + err, p.faults = p.faults[0], p.faults[1:] + } + p.mu.Unlock() + if err != nil { + return err + } + return p.inner.PublishAsk(ctx, subject, id, data) +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go index cc24d6ec..002ecd3d 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go @@ -160,10 +160,11 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q {Name: build.BuildAgainTool, Description: "Build a failed key again (rebuild a module, build a failed key again): the builder writes the key of module and " + "fingerprint building as a new revision of the same key and asks the build seats, as it does by itself when it looks at a module " + - "again (ADR 0429 clause 18). Answers the new revision and the ask id of each build module (go, node, image). Refuses a key that " + - "is not failed, and says why: done (nothing to do), or waiting (building or queued: " + build.AskAgainTool + " asks its waiting asks " + - "again); a key failed for a reason the builder never retries on the same fingerprint (a pin missing, a version taken, digests that " + - "differ) and a fingerprint its module no longer has are refused too. Use it for a key builder_stuck does not show, which failed.", + "again (ADR 0429 clause 18). Answers the new revision and the ask id of each build module (go, node, image). Also takes a key left " + + "building that no look of the builder holds (its ask never put or its end never written, ADR 0668 §5): an ask the key names and " + + "its stream holds is waited for, one never put is put under its id. Refuses a key done (nothing to do) or queued (" + + build.AskAgainTool + " asks its waiting asks again), and one the builder is looking at now; a key failed for a reason the builder never retries on the same fingerprint (a pin missing, a version taken, digests that " + + "differ) and a fingerprint its module no longer has are refused too. Use it for a failed key, or one builder_stuck shows building with no ask or no step.", Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{ "module": map[string]any{"type": "string", "description": "the module's name"}, "fp": map[string]any{"type": "string", "description": "its new-form fingerprint, src2:"}, diff --git a/modules/mesh-delivery-builder/internal/build/buildagain.go b/modules/mesh-delivery-builder/internal/build/buildagain.go index 6f39187e..2c135769 100644 --- a/modules/mesh-delivery-builder/internal/build/buildagain.go +++ b/modules/mesh-delivery-builder/internal/build/buildagain.go @@ -84,7 +84,8 @@ func (b *Builder) handEnded(key string) { // BuildAgain builds the failed key of module at fp again, as the builder does by itself: the key is written building as a // new revision and the build module asks are put on the build seats. It answers once the asks are put, or at bound. A key -// that is not failed is refused with why: done (nothing to do) or waiting (builder_ask_again asks its waiting asks again). +// left building that no look of this builder holds is taken as it stands and its asks put if they never were (issue 1966). +// A key done or queued is refused with why: done (nothing to do) or waiting (builder_ask_again asks its waiting asks again). // ctx is the builder's own context: the build it starts lives on it. func (b *Builder) BuildAgain(ctx context.Context, module, fp, why, by string, bound time.Duration) BuildAgainResult { b.init() @@ -106,8 +107,12 @@ func (b *Builder) BuildAgain(ctx context.Context, module, fp, why, by string, bo switch { case cur.State == builds.BuildDone: return BuildAgainResult{Refused: fmt.Sprintf("%s is done: nothing to do", key)} - case cur.State == builds.BuildBuilding || cur.State == builds.BuildQueued: + case cur.State == builds.BuildQueued: return BuildAgainResult{Refused: fmt.Sprintf("%s is waiting (%s: %s): %s asks its waiting asks again", key, cur.State, cur.Step, AskAgainTool)} + case cur.State == builds.BuildBuilding: + // A key left building that no look of this builder holds (refused below when one does): its put or its end was never + // written (issue 1966, 1748 note 27: "building, claimed", no ask). It is taken as it stands, an ask it names and the + // stream holds is waited for, one never put is put under its id, one never named is asked (ADR 0668 §5). case cur.State != builds.BuildFailed: return BuildAgainResult{Refused: fmt.Sprintf("%s is %q, not failed", key, cur.State)} case builds.FailedUntilChanged(cur.Error) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix): @@ -139,7 +144,11 @@ func (b *Builder) BuildAgain(ctx context.Context, module, fp, why, by string, bo } b.mu.Unlock() }() - b.logf("%s: failed (%s); built again by hand, asked by %s: %s", key, cur.Error, by, why) + if cur.State == builds.BuildBuilding { + b.logf("%s: left building (%s) with no look of this builder; built again by hand, asked by %s: %s", key, cur.Step, by, why) + } else { + b.logf("%s: failed (%s); built again by hand, asked by %s: %s", key, cur.Error, by, why) + } b.startOnce(ctx, key, module, m) wait, stop := context.WithTimeout(ctx, bound) // a context's bound for a person's answer (ADR 0437 §8) defer stop() diff --git a/modules/mesh-delivery-builder/internal/build/claim_test.go b/modules/mesh-delivery-builder/internal/build/claim_test.go index 04b68925..eeda081f 100644 --- a/modules/mesh-delivery-builder/internal/build/claim_test.go +++ b/modules/mesh-delivery-builder/internal/build/claim_test.go @@ -18,6 +18,8 @@ type memStore struct { writes int // before runs once ahead of the next write, so a test changes the key under the writer. before func() + // faults fail the next updates, one each, writing nothing (a deadline met on the bus). + faults []error } func newMem() *memStore { return &memStore{vals: map[string][]byte{}, revs: map[string]uint64{}} } @@ -67,6 +69,11 @@ func (m *memStore) Update(_ context.Context, key string, value []byte, rev uint6 m.mu.Lock() defer m.mu.Unlock() m.hook() + if len(m.faults) > 0 { + err := m.faults[0] + m.faults = m.faults[1:] + return 0, err + } if _, ok := m.vals[key]; !ok || m.revs[key] != rev { return 0, ErrConflict } diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 8b74d448..36a9b468 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -21,6 +21,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds/forms" sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" repotrunk "git.novox.be/novox/mesh-delivery/sdk/trunk" + "git.novox.be/novox/mesh-sdk/go/busretry" "git.novox.be/novox/mesh-sdk/go/catalogue" "mesh-delivery-builder/internal/versions" @@ -477,7 +478,9 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB return } } - cl, err := b.Claimer.Claim(ctx, module, m.Fp2, m.Manifest, ClaimOptions{TakeOver: first, ByHand: byHand, ImageBuilder: imageSeat, Trunk: trunk, Published: b.isPublished, + // A key built again by hand is taken over as it stands too: one left building by a look that ended without its write + // (issue 1966) is the hand tool's to finish. + cl, err := b.Claimer.Claim(ctx, module, m.Fp2, m.Manifest, ClaimOptions{TakeOver: first || byHand, ByHand: byHand, ImageBuilder: imageSeat, Trunk: trunk, Published: b.isPublished, Stored: b.storedOnce(key)}) if err != nil { b.logf("%s: %v", module, err) @@ -924,7 +927,13 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes } } else { if err := asked.note(ctx, language, ask.ID, "asked "+l.module); err != nil { - l.err = fmt.Errorf("the ask of %s could not be written to the key: %w", l.module, err) + // The note and the put are one act (novox/hq ADR 0668 §1; issue 1748 note 27): a note a transport fault kept from + // the key fails the key with that fault's words, which builder_build_again takes (ADR 0664 §3). + fault := "" + if busretry.TimedOut(err) || busretry.Undelivered(err) { + fault = sdkstep.TransportFaultWords + "the note of the ask: " + } + l.err = fmt.Errorf("%sthe ask of %s could not be written to the key: %w", fault, l.module, err) return } if err := b.Asks.Put(ctx, ask); err != nil { diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index b4620e5b..a1429211 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -981,6 +981,8 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { failed string // during is a look of the builder's own at the module while the by-hand build runs: it still runs afterwards. during bool + // standing is a key taken as it stands (left building): its claim is the key's own revision, not a new one. + standing bool }{ {name: "build again: a look the builder queued during the by-hand build still runs afterwards", again: true, during: true, key: &builds.BuilderBuild{State: builds.BuildFailed, Error: "ace: no tun"}}, @@ -1002,8 +1004,16 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { key: &builds.BuilderBuild{State: builds.BuildFailed, Error: "ace: the step failed: no tun"}}, {name: "build again: a done key is refused, nothing to do", again: true, refuse: "is done: nothing to do", key: &builds.BuilderBuild{State: builds.BuildDone, Step: "built"}}, - {name: "build again: a building key is refused, builder_ask_again is for it", again: true, refuse: "is waiting", - key: &builds.BuilderBuild{State: builds.BuildBuilding, By: "novox.builder", Step: "asked mesh-build-go", Asks: map[string]string{buildstep.AskGo: "mvl0frv18d1960"}}}, + {name: "build again: a queued key is refused, builder_ask_again is for it", again: true, refuse: "is waiting", + key: &builds.BuilderBuild{State: builds.BuildQueued, By: "novox.builder", Step: "waits for the package x", Waits: "x@v1.0.0"}}, + // Issue 1966 (ADR 0668 §5; 1748 note 27): a key left building with no ask, no look of the builder holding it, is taken as + // it stands, by this builder or from another, asked and done; one a look holds is refused. + {name: "build again: a key left building claimed and unasked is asked and done", again: true, standing: true, + key: &builds.BuilderBuild{State: builds.BuildBuilding, By: "novox.builder", Step: "claimed"}}, + {name: "build again: a key another builder left building unasked is taken over", again: true, standing: true, + key: &builds.BuilderBuild{State: builds.BuildBuilding, By: "ace.builder", Step: "claimed"}}, + {name: "build again: a key left building that a look holds is refused", again: true, refuse: "is being looked at", busy: 1, + key: &builds.BuilderBuild{State: builds.BuildBuilding, By: "novox.builder", Step: "claimed"}}, {name: "build again: a key failed for a pin missing is refused, it waits for another fingerprint", again: true, refuse: "does not retry", key: &builds.BuilderBuild{State: builds.BuildFailed, Error: builds.NotPinned + "web"}}, {name: "build again: a key with no build is refused", again: true, refuse: "has no build key"}, @@ -1047,6 +1057,9 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { r.b.hands[r.key] = &handWatch{} } r.b.known["m"] = m // the builder has read the modules value, and looks at nothing until asked + if c.standing { + r.b.tried[r.key] = true // its look of the key ended before its write: a later look takes over nothing by itself + } log := &logged{} r.b.Logf = log.logf res := r.b.BuildAgain(context.Background(), "m", fp1, "no tun on ace", "shanks.lead", 2*time.Second) @@ -1075,7 +1088,7 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { return } asks := r.asks.Asks() - if res.Refused != "" || res.Pending != "" || res.Revision <= rev || len(asks) != 1 || res.Asks[buildstep.AskGo] != asks[0].ID { + if res.Refused != "" || res.Pending != "" || res.Revision < rev || (res.Revision == rev && !c.standing) || len(asks) != 1 || res.Asks[buildstep.AskGo] != asks[0].ID { t.Fatalf("answer %+v after %d asks; want the new revision (was %d) and the ask id of the one ask", res, len(asks), rev) } if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone || after <= res.Revision { @@ -1350,6 +1363,24 @@ func TestAnAskTheKeyNamesAndNobodyPutIsPutAtStart(t *testing.T) { t.Errorf("held %v: the key = %+v, want done", held, v) } } + // Issue 1966 (ADR 0668 §1, §5; 1748 note 27): the note of the ask and its put are one act. A note a deadline kept from the + // key fails the key in the transport fault's words: the builder does not take it again by itself (ADR 0664 §3), and + // builder_build_again builds it. + r := newRig(t, goBundle) + r.st.faults = []error{fmt.Errorf("the key: %w", context.DeadlineExceeded)} + if v := r.consider(t, "modules/m"); v.State != builds.BuildFailed || !builds.FailedOnTransportFault(v.Error) || len(r.asks.Asks()) != 0 { + t.Fatalf("a note the bus did not answer: the key = %+v after %d asks, want failed on a transport fault, nothing put", v, len(r.asks.Asks())) + } + if v := r.consider(t, "modules/m"); v.State != builds.BuildFailed || len(r.asks.Asks()) != 0 { + t.Fatalf("looked at again by itself: the key = %+v after %d asks, want it left failed", v, len(r.asks.Asks())) + } + if res := r.b.BuildAgain(context.Background(), "m", fp1, "the bus was back", "shanks.lead", 2*time.Second); res.Refused != "" || len(r.asks.Asks()) != 1 { + t.Fatalf("built again by hand: %+v after %d asks", res, len(r.asks.Asks())) + } + r.b.Wait() + if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone { + t.Errorf("built again by hand: the key = %+v, want done", v) + } } // unreadableHeld cannot tell whether an ask is held. From e4dabdbc7a93cb34d1211129826f2babd3aa1533 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 10 Nov 2026 03:34:57 +0100 Subject: [PATCH 08/22] =?UTF-8?q?D7:=20a=20build=20key=20a=20look=20left?= =?UTF-8?q?=20building=20because=20its=20end=20could=20not=20be=20written?= =?UTF-8?q?=20is=20call-unanswered=20after=20five=20minutes=20(issue=20196?= =?UTF-8?q?6,=201748=20note=2027,=20ADR=200668=20=C2=A73,=20=C2=A75)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The look arms a key of the state bounds with its own time to live; the bus's marker of its end raises call-unanswered about the build key while it is still building and no look holds it, naming builder_build_again; the key's next written end deletes the bound and clears the finding. A bound that cannot be written raises at once. No timer. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-builder/findings.go | 10 ++- .../cmd/mesh-delivery-builder/main.go | 10 +++ .../cmd/mesh-delivery-builder/refused_test.go | 49 +++++++++++ .../cmd/mesh-delivery-builder/unput.go | 84 +++++++++++++++++++ .../internal/build/run.go | 19 +++++ .../internal/build/run_test.go | 17 ++++ .../internal/versions/publish.go | 16 +++- 7 files changed, 199 insertions(+), 6 deletions(-) create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/unput.go diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/findings.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/findings.go index 10b6bac1..b730238d 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/findings.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/findings.go @@ -82,12 +82,16 @@ func (f *findings) buildEnded(ctx context.Context, e builds.BuildLogEntry) { // raise writes fd; it is written again only when its words changed, so the keeper's condition moves only then. func (f *findings) raise(ctx context.Context, fd finding) { - o := meshconditionsclient.Observation{Scope: fd.scope, ID: fd.id, Kind: fd.kind, Severity: fd.severity, + f.raiseObservation(ctx, meshconditionsclient.Observation{Scope: fd.scope, ID: fd.id, Kind: fd.kind, Severity: fd.severity, Summary: fd.summary, Headline: fd.headline, Explanation: fd.explanation, Resolved: fd.resolved, Needs: fd.needs, - Source: "builder." + fd.kind} + Source: "builder." + fd.kind}) +} + +// raiseObservation writes o, as raise does. +func (f *findings) raiseObservation(ctx context.Context, o meshconditionsclient.Observation) { kv := f.bucket() if kv == nil { - f.logf("finding %s (not written, %s is not on the bus yet): %s", o.Key(), findingsBucket, fd.summary) + f.logf("finding %s (not written, %s is not on the bus yet): %s", o.Key(), findingsBucket, o.Summary) return } if _, err := meshconditionsclient.Raise(findingBucket{ctx, kv}, builds.BuilderModule, "", o, f.now()); err != nil { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 24f16218..71dd3e3d 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -19,6 +19,7 @@ import ( "syscall" "time" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" sdkfindings "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient/findings" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" @@ -186,6 +187,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Stored: storedVia(calls.Call), Node: cred.Node, } + // A key a look left building because its end could not be written is call-unanswered past five minutes (issue 1966, + // ADR 0668 §3): its bound is a key of the state bounds. + unput := &unputs{bounds: pubBoundsKV.KV, store: b.Claimer.Store, looking: b.Looking, findings: found, + bound: meshconditionsclient.CallUnansweredAfter, now: time.Now, logf: logger.Printf} + b.Left, b.Ended = unput.left, unput.ended tagAsksKV := process.AwaitBucket(ctx, js, conn, versions.TagAsksBucket, "builder_tag and builder_untag are not answered", logger.Printf) // A check's build of a head a kept tag ask names numbers its packages tagged (design 51 *Tagged versions*), and the ask // is marked numbered with its versions. @@ -396,6 +402,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro }) pubBoundsKV.Ready(func(kv jetstream.KeyValue) { each := func(e jetstream.KeyValueEntry) { + if _, ok := versions.UnputBoundOf(e.Key()); ok { + unput.expired(ctx, e) + return + } if versions.IsOldFormBound(e.Key()) { if e.Operation() == jetstream.KeyValuePurge { reads.stepWindow() // a node's seven days ended: the step raises old-form-held now diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go index 0994e9e9..9d1b8050 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go @@ -391,6 +391,48 @@ func aFailedTrunkBuildIsAFinding(t *testing.T) { if !h.heard(context.Background(), published) || st.vals[refusedKey] != nil || st.vals[unansweredKey] != nil || st.vals[unreadKey] != nil { t.Fatalf("a later published leaves the findings: %v", st.vals) } + // Issue 1966 (ADR 0668 §3, §5; 1748 note 27): a key a look left building because its end could not be written arms its + // bound; the bound's end while the key is still building and no look holds it is call-unanswered about the key; the key's + // next written end deletes the bound and clears it; a key a look holds, or one no longer building, raises nothing; a + // bound that cannot be written raises at once. + buildKey := "m.1111111111111111111111111111111111111111111111111111111111111111" + building, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildBuilding, By: "novox.builder", Step: "claimed"}) + done, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, By: "novox.builder", Step: "built"}) + keys := memBuilds{memVersions{buildKey: building}} + looking := false + unputMarks := &memBounds{keys: map[string]bool{}} + u := &unputs{bounds: func() jetstream.KeyValue { return unputMarks }, store: keys, looking: func(string) bool { return looking }, + findings: f, bound: meshconditionsclient.CallUnansweredAfter, now: func() time.Time { return time.Unix(600, 0) }, logf: t.Logf} + callKey := meshconditionsclient.Key(meshconditionsclient.ScopeCall, buildKey, meshconditionsclient.KindCallUnanswered) + u.left(context.Background(), buildKey, "nats: timeout") + if !unputMarks.keys[versions.UnputBoundKey(buildKey)] || st.vals[callKey] != nil { + t.Fatalf("a key left building: bound %v, finding %v; want the bound armed, no finding yet", unputMarks.keys, st.vals[callKey] != nil) + } + for name, c := range map[string]struct { + value []byte + look bool + }{"a look holds it": {building, true}, "no longer building": {done, false}} { + keys.memVersions[buildKey], looking = c.value, c.look + u.expired(context.Background(), purged{key: versions.UnputBoundKey(buildKey)}) + if st.vals[callKey] != nil { + t.Errorf("%s: the bound's end raised call-unanswered", name) + } + } + keys.memVersions[buildKey], looking = building, false + u.expired(context.Background(), purged{key: versions.UnputBoundKey(buildKey)}) + if got, err := meshconditionsclient.DecodeFinding(st.vals[callKey]); err != nil || got.Kind != meshconditionsclient.KindCallUnanswered || !strings.Contains(got.Summary, buildKey) || !strings.Contains(got.Needs, build.BuildAgainTool) { + t.Fatalf("the bound's end of a key left building: the finding = %+v, %v", got, err) + } + u.ended(context.Background(), buildKey) + if unputMarks.keys[versions.UnputBoundKey(buildKey)] || st.vals[callKey] != nil { + t.Errorf("the key's end written: bound %v, finding left %v", unputMarks.keys, st.vals[callKey] != nil) + } + u.bounds = func() jetstream.KeyValue { return nil } + u.left(context.Background(), buildKey, "nats: timeout") + if got, err := meshconditionsclient.DecodeFinding(st.vals[callKey]); err != nil || !strings.Contains(got.Explanation, "nor its bound") { + t.Fatalf("a bound not written: the finding = %+v, %v", got, err) + } + delete(st.vals, callKey) // ADR 0615 §5 (review round 1): a consumer not there yet is family-not-heard naming its stream, cleared when it binds. notHeard := f.notHeard(context.Background()) unheardKey := heard.Key(builds.BuilderModule, "FORGE") @@ -490,6 +532,13 @@ func (m memVersions) KeysOf(context.Context, string) ([]string, error) { return slices.Collect(maps.Keys(m)), nil } +// memBuilds is the builder's builds in memory. +type memBuilds struct{ memVersions } + +func (m memBuilds) Keys(context.Context) ([]string, error) { + return slices.Collect(maps.Keys(m.memVersions)), nil +} + // memBounds is the state bounds in memory: which keys stand. type memBounds struct { jetstream.KeyValue diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/unput.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/unput.go new file mode 100644 index 00000000..55c9ccab --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/unput.go @@ -0,0 +1,84 @@ +package main + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + + "mesh-delivery-builder/internal/build" + "mesh-delivery-builder/internal/versions" +) + +// **A key left building unput is a finding after five minutes** (issue 1966, 1748 note 27; novox/hq ADR 0668 §3, §5). A +// look whose end could not be written leaves its key building with no look holding it: its ask put or not, nothing ends +// it but builder_build_again or the builder's next start. The look arms a key of the state bounds (UnputBoundKey) with +// its own time to live; the bus's marker of its end, heard on the watch, raises call-unanswered about the build key while +// the key is still building and no look holds it. The key's next written end deletes the bound and clears the finding. +// No timer. +type unputs struct { + bounds func() jetstream.KeyValue + store build.Store + looking func(key string) bool + findings *findings + bound time.Duration + now func() time.Time + logf func(string, ...any) +} + +// unputCallee is what the finding names as the call left unanswered: the build seats' put of the key's asks and the +// builder's write of its end. +const unputCallee = "the put of its asks on the build seats and the write of its end" + +// left arms key's bound; one that cannot be written would never end, so the finding is raised at once. +func (u *unputs) left(ctx context.Context, key, why string) { + var err error + if kv := u.bounds(); kv == nil { + err = fmt.Errorf("%s is not on the bus yet", versions.BoundsBucket) + } else if _, err = kv.Create(ctx, versions.UnputBoundKey(key), versions.EncodeBound("the build key "+key+" left building unput: "+why, u.now().Add(u.bound)), jetstream.KeyTTL(u.bound)); errors.Is(err, jetstream.ErrKeyExists) { + err = nil + } + if err == nil { + return + } + u.logf("the unput bound of %s is not written, so it is raised now: %v", key, err) + u.raise(ctx, key, fmt.Sprintf("its end could not be written (%s), nor its bound (%v)", why, err), u.now()) +} + +// ended deletes key's bound and clears its finding: its end was written. +func (u *unputs) ended(ctx context.Context, key string) { + if kv := u.bounds(); kv != nil { + if err := kv.Delete(ctx, versions.UnputBoundKey(key)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + u.logf("the unput bound of %s is not deleted: %v", key, err) + } + } + u.findings.clear(ctx, meshconditionsclient.KindCallUnanswered, meshconditionsclient.ScopeCall, key) +} + +// expired is told each change of the state bounds: the end of an unput bound raises call-unanswered while its key is +// still building and no look holds it. +func (u *unputs) expired(ctx context.Context, e jetstream.KeyValueEntry) { + key, ok := versions.UnputBoundOf(e.Key()) + if !ok || e.Operation() != jetstream.KeyValuePurge || u.looking(key) { + return + } + raw, _, found, err := u.store.Get(ctx, key) + if err != nil || !found { + return + } + if v, err := builds.DecodeBuilderBuild(raw); err != nil || v.State != builds.BuildBuilding { + return + } + u.raise(ctx, key, fmt.Sprintf("its look ended without writing its end, and nothing ended it within %s", u.bound), u.now().Add(-u.bound)) +} + +func (u *unputs) raise(ctx context.Context, key, fault string, since time.Time) { + o := meshconditionsclient.CallUnanswered(key, unputCallee, "", fault, "builder."+meshconditionsclient.KindCallUnanswered, since) + o.Needs = build.BuildAgainTool + " builds the key again: an ask its stream holds is waited for, one never put is put; " + build.StuckTool + " shows it." + u.findings.raiseObservation(ctx, o) +} diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 36a9b468..a7711f09 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -53,6 +53,11 @@ type Builder struct { } Toolchains ToolchainEntries Manifests Manifests + // Left, when set, is told a key a look claimed and left building because its end could not be written (issue 1966, + // novox/hq ADR 0668 §3; 1748 note 27): no look holds it then, so nothing ends it but builder_build_again or the + // builder's next start. Ended is told each key whose end was written. + Left func(ctx context.Context, key, why string) + Ended func(ctx context.Context, key string) // Limit is how many builds the builder has asked at once; their asks wait in the build seats' streams, which are the // backlog (ADR 0525 §9), not here. 256 when 0. Limit int @@ -132,6 +137,14 @@ type planned struct { err error } +// Looking says a look of this builder holds key now, by itself or asked by hand. +func (b *Builder) Looking(key string) bool { + b.init() + b.mu.Lock() + defer b.mu.Unlock() + return b.inflight[key] || b.hands[key] != nil +} + func (b *Builder) init() { b.once.Do(func() { limit := b.Limit @@ -551,8 +564,14 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB } if err := b.Claimer.Finish(ctx, cl, result); err != nil { b.logf("%s: the build could not be written: %v", cl.Key, err) + if b.Left != nil && !errors.Is(err, ErrConflict) && ctx.Err() == nil { + b.Left(ctx, cl.Key, err.Error()) // left building with no look holding it (issue 1966) + } return } + if b.Ended != nil { + b.Ended(ctx, cl.Key) + } ended := LogEnded(module, m, b.refOf(module, m.Fp2), b.Node, b.now(), result) ended.OutOfStep = b.takeOutOfStep(key) // the generated files the build module's compare found out of step (design 26) b.logBuild(ended) diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index a1429211..cfc3ad9b 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -1381,6 +1381,23 @@ func TestAnAskTheKeyNamesAndNobodyPutIsPutAtStart(t *testing.T) { if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone { t.Errorf("built again by hand: the key = %+v, want done", v) } + // Its end not written either (1748 note 27's key: building, claimed, no ask): the look tells Left, and no look holds the + // key; built again by hand it is asked and done, and Ended is told. + r = newRig(t, goBundle) + r.st.faults = []error{context.DeadlineExceeded, context.DeadlineExceeded} + var left, ended []string + r.b.Left = func(_ context.Context, key, why string) { left = append(left, key+": "+why) } + r.b.Ended = func(_ context.Context, key string) { ended = append(ended, key) } + if v := r.consider(t, "modules/m"); v.State != builds.BuildBuilding || v.Step != "claimed" || len(v.Asks) != 0 || len(left) != 1 || !strings.HasPrefix(left[0], r.key) || len(ended) != 0 || r.b.Looking(r.key) { + t.Fatalf("an end not written: the key = %+v, left %q, ended %q, looking %v", v, left, ended, r.b.Looking(r.key)) + } + if res := r.b.BuildAgain(context.Background(), "m", fp1, "left building", "shanks.lead", 2*time.Second); res.Refused != "" { + t.Fatalf("built again by hand: %+v", res) + } + r.b.Wait() + if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone || len(r.asks.Asks()) != 1 || len(ended) != 1 { + t.Errorf("built again by hand: the key = %+v after %d asks, ended %q", v, len(r.asks.Asks()), ended) + } } // unreadableHeld cannot tell whether an ask is held. diff --git a/modules/mesh-delivery-builder/internal/versions/publish.go b/modules/mesh-delivery-builder/internal/versions/publish.go index d185f6bf..7b401b8a 100644 --- a/modules/mesh-delivery-builder/internal/versions/publish.go +++ b/modules/mesh-delivery-builder/internal/versions/publish.go @@ -19,8 +19,8 @@ import ( // heard on the watch, is what raises publish-unanswered while the version is still numbered. No timer. const ( // BoundsState is the builder's state of its expiring keys, each with its own time to live: a publish's bound - // (PublishBoundKey) and a node's seven days holding the old form's window (OldFormBoundKey). Its end, the bus's - // marker heard on the watch, is what acts. + // (PublishBoundKey), a node's seven days holding the old form's window (OldFormBoundKey) and a build key left + // building unput (UnputBoundKey). Its end, the bus's marker heard on the watch, is what acts. BoundsState = "bounds" BoundsBucket = "mesh-delivery-builder_bounds" // PublishBound is the build module's own wait for the holder's word (design 51 *How a publish ends*: at most @@ -36,8 +36,17 @@ const ( const ( publishBoundPrefix = "publish." oldFormBoundPrefix = "old-form." + unputBoundPrefix = "unput." ) +// UnputBoundKey is the bound of a build key a look left building because its end could not be written (issue 1966, +// novox/hq ADR 0668 §3): `unput.`. Its end while the key still stands so, with no look holding it, is the +// finding call-unanswered about the key. +func UnputBoundKey(buildKey string) string { return unputBoundPrefix + buildKey } + +// UnputBoundOf is the build key an unput bound names, and whether key is one. +func UnputBoundOf(key string) (string, bool) { return strings.CutPrefix(key, unputBoundPrefix) } + // PublishBoundKey is the bound of name's version: `publish.`. func PublishBoundKey(name, version string) string { return publishBoundPrefix + Key(name, version) } @@ -58,7 +67,8 @@ const BoundsTool = "builder_bounds" // BoundsDescription is builder_bounds' description. const BoundsDescription = "The builder's expiring keys, its state " + BoundsState + " (design 51 *Health and conditions*): each key with what it " + "bounds (a version's publish, whose end unanswered is the finding " + PublishUnanswered + "; or a node's seven days holding the old " + - "form's window, whose end is the finding " + OldFormHeld + "), when it ends, and how long is left. The bus ends each key by its own time to live." + "form's window, whose end is the finding " + OldFormHeld + "; or a build key left building because its end could not be written, " + + "whose end is the finding call-unanswered), when it ends, and how long is left. The bus ends each key by its own time to live." // Bound is a key's value in BoundsState: what it bounds, in plain words, and when it ends. type Bound struct { From d3f68c5372f0d62650a50985d74d9951a4683434 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 10 Nov 2026 03:34:57 +0100 Subject: [PATCH 09/22] =?UTF-8?q?D5:=20indexer=5Fdrop=5Fpending=20counts?= =?UTF-8?q?=20a=20later=20pending=20merge=20as=20covering=20only=20when=20?= =?UTF-8?q?its=20commit=20has=20the=20pending=20merge's=20as=20an=20ancest?= =?UTF-8?q?or=20(issue=201966,=20ADR=200668=20=C2=A76,=20ADR=200633=20?= =?UTF-8?q?=C2=A75)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../internal/index/pending.go | 20 +++++++++++++++++-- .../internal/index/pending_test.go | 11 ++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index 9999039f..08128919 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -1307,7 +1307,23 @@ func (ix *Indexer) DropPending(ctx context.Context, key, by, why string, abandon } e := entries[at] if !abandon { - covered := slices.ContainsFunc(entries[at+1:], func(o PendingEntry) bool { return o.Forge.Repository == e.Forge.Repository }) + // Covered: a later merge of the same repository, pending or taken in, has this merge's commit as an ancestor (ADR + // 0668 §6), asked of the forge (older); a later merge whose history does not hold it covers nothing. + covered := false + for _, o := range entries[at+1:] { + if o.Forge.Repository != e.Forge.Repository { + continue + } + in, err := ix.older(ctx, e.Forge.Repository, e.Forge.Merge, o.Forge.Merge) + if err != nil { + r.Answer = fmt.Sprintf("%s is not dropped: whether the later pending merge %s covers it is not read: %v", key, o.Key, err) + return r + } + if in { + covered = true + break + } + } if !covered { taken, err := ix.takenAlready(ctx, forgemsg.GroupMerge{Repository: e.Forge.Repository, Number: e.Forge.Number, Merge: e.Forge.Merge}) if err != nil { @@ -1317,7 +1333,7 @@ func (ix *Indexer) DropPending(ctx context.Context, key, by, why string, abandon covered = taken } if !covered { - r.Answer = fmt.Sprintf("%s is not dropped: no later merge of %s is pending and its taken merge does not cover %s, so its "+ + r.Answer = fmt.Sprintf("%s is not dropped: no later merge of %s, pending or taken in, has %s as an ancestor, so its "+ "changes would never be taken in (ADR 0633 §5); %s reads its message again, or abandon: true drops it all the same", key, e.Forge.Repository, e.Forge.Merge, TakeInPendingTool) return r diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index c0bd87e8..e41d3c03 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -711,6 +711,17 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) answer.Waits[0].Said != takein.TakeInPendingEvent || len(answer.Waits[0].Keys) != 2 { t.Fatalf("indexer_pending = %+v", answer) } + // ADR 0668 §6: a later merge covers a pending one only when its commit has the pending merge's as an ancestor. + history := "cd" + b.ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { + i, j := strings.IndexByte(history, commit[0]), strings.IndexByte(history, in[0]) + return i >= 0 && j >= 0 && i <= j, nil + } + history = "dc" // m.3's commit is not in m.2's later history (a history rewritten) + if r := b.ix.DropPending(ctx, takein.PendingKey(2), "lead@shanks", "replaced by m.3", false); r.Stored || !strings.Contains(r.Answer, "as an ancestor") { + t.Fatalf("a drop a later merge not holding it would cover: %+v", r) + } + history = "cd" b.said = nil if r := b.ix.DropPending(ctx, takein.PendingKey(2), "lead@shanks", "replaced by m.3", false); !r.Stored { t.Fatalf("drop: %+v", r) From 9296db55389959aa0a413ecead0613eb1e7831d4 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 10 Nov 2026 03:38:07 +0100 Subject: [PATCH 10/22] =?UTF-8?q?D6:=20a=20take-in=20waits=20for=20the=20v?= =?UTF-8?q?erdicts=20state,=20so=20every=20manifest=20it=20writes=20is=20j?= =?UTF-8?q?udged=20in=20it=20before=20take-in.taken;=20its=20bind=20evalua?= =?UTF-8?q?tes=20pending=20again=20(issue=201966=20note=20154,=20ADR=20066?= =?UTF-8?q?8=20=C2=A74)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A take-in whose judge pass fails is judged again at the next message heard, as a failed start pass is. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-indexer/main.go | 7 ++++++ .../internal/index/candidates.go | 4 +++- .../internal/index/pending.go | 23 +++++++++++++++++-- .../internal/index/pending_test.go | 17 +++++++++++++- 4 files changed, 47 insertions(+), 4 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index acd2104c..d6bce20d 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -591,6 +591,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // Every manifest judged at the start, so a judge that changed with this program judges every one again (ADR 0578 // §1), once the verdicts state is there; each merge judges them again (index.ApplyMessage). verdicts.Ready(func(jetstream.KeyValue) { + // A take-in held because the verdicts state was not on the bus yet is evaluated again once the start's pass ended, + // whatever it answered (issue 1966 note 154). + defer func() { + if err := ix.PendingAgain(ctx); err != nil { + logger.Printf("take-in: pending is not evaluated again at the verdicts state's bind: %v", err) + } + }() // A pass that fails (a read the bus did not answer) is judged again at the next message or source revision the // indexer hears (index.kickJudge): no timer. j, err := ix.JudgeAtStart(ctx) diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 0849ba47..191eca68 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -245,7 +245,9 @@ func (ix *Indexer) ApplyMessage(ctx context.Context, f forgemsg.Forge) (Applied, // Every manifest judged again: a rule between manifests can turn on the one this merge moved (ADR 0578 §1). // What this take-in newly wrote is fresh: judged in full (ADR 0579 §1). if _, err := ix.JudgeAll(ctx, w.fresh); err != nil { - ix.logf("judge: the manifests are not judged after %s#%d: %v", f.Repository, f.Number, err) + // Judged again at the next message or source revision heard (kickJudge), as a failed start pass is. + ix.judgePending.Store(true) + ix.logf("judge: the manifests are not judged after %s#%d: %v; judged again at the next message heard", f.Repository, f.Number, err) } } ix.kickRetry(ctx) // a registered module the forge could not read is read again on a merge, in the background (issue 1945) diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index 08128919..46b42f5c 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -355,6 +355,15 @@ func revisionWait(kind string, ready []PendingEntry, waitsFor string) takein.Tak // pendingOnBus says whether the pending state is on the bus: a state this delivery brings is bound when it appears // (issue 1790), and until then every merge is taken in at once, so the build this program runs is not needed. A read // that fails otherwise counts as on the bus: the step that follows answers the error. +// errVerdictsNotYet is a take-in held because the verdicts state is not on the bus yet (takeInWholeBy). +var errVerdictsNotYet = errors.New("the verdicts state is not on the bus yet: nothing is taken in until it is") + +// verdictsOnBus says the verdicts state is on the bus (a take-in judges what it writes there). +func (ix *Indexer) verdictsOnBus(ctx context.Context) bool { + _, _, _, err := ix.Verdicts.Get(ctx, "on-the-bus") + return !errors.Is(err, process.ErrNotYet) +} + func (ix *Indexer) pendingOnBus(ctx context.Context) bool { _, _, _, err := ix.Pending.Get(ctx, takein.PendingKey(0)) return !errors.Is(err, process.ErrNotYet) @@ -696,7 +705,7 @@ func (ix *Indexer) UnreadAgain(ctx context.Context) error { } // PendingAgain evaluates pending again after something a fingerprint is computed from changed, a toolchain report -// (ADR 0633 §1, §3): a revision that now names the running build is taken in, and one that names another build is held +// (ADR 0633 §1, §3), or the verdicts state bound (issue 1966): a revision that now names the running build is taken in, and one that names another build is held // for that one. It does nothing while the pending state is not on the bus or the build this program runs is not known. func (ix *Indexer) PendingAgain(ctx context.Context) error { if ix.Pending == nil || !ix.pendingOnBus(ctx) { @@ -713,7 +722,7 @@ func (ix *Indexer) PendingAgain(ctx context.Context) error { return nil } if err == nil && words != "" { - ix.logf("take-in: evaluated again after a toolchain report: %s", words) + ix.logf("take-in: evaluated again: %s", words) } return err } @@ -760,6 +769,9 @@ func (ix *Indexer) evaluate(ctx context.Context, own string, start bool) (Applie if errors.Is(err, errClaimLost) { return a, "claimed by another take-in", nil } + if errors.Is(err, errVerdictsNotYet) { + return a, "the verdicts state is not on the bus yet: the revision waits for it, taken in when it binds", nil + } return a, ix.waitingLine(w), err } return Applied{}, ix.waitingLine(w), ix.holdFor(ctx, ready, names, own, dry) @@ -1002,6 +1014,13 @@ func (ix *Indexer) takeInWhole(ctx context.Context, entries []PendingEntry, own, // takeInWholeBy is takeInWhole; who and why, of a hand act, are carried on the take-in.taken events. func (ix *Indexer) takeInWholeBy(ctx context.Context, entries []PendingEntry, own, by, who, why string) (Applied, error) { + if ix.Verdicts != nil && !ix.verdictsOnBus(ctx) && len(entries) > 0 { + // Every manifest a take-in writes is judged in it and its verdict said before take-in.taken (issue 1966 note 154, + // ADR 0668 §4): with the verdicts state not on the bus yet, nothing is taken in; its bind evaluates pending again. + ix.logf("take-in: %d merges wait, oldest %s: the state %s is not on the bus yet, so nothing is taken in until it is, and no manifest is written without its verdict", + len(entries), entries[0].Key, VerdictsBucket) + return Applied{}, errVerdictsNotYet + } claimed, ok := ix.claim(ctx, entries, own, by) if !ok { return Applied{}, errClaimLost diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index e41d3c03..91d401c3 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -206,8 +206,23 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) t.Run("T7 and T3 a merge alone, an indexer change, a merge behind it and a revert", func(t *testing.T) { b := newTakeInBed(t) own, avahi0 := b.fp(t, Module), b.fp(t, "avahi") - // A merge with no change to the indexer and nothing pending is taken in alone, as before. + // Issue 1966 note 154 (ADR 0668 §4): with the verdicts state not on the bus yet, nothing is taken in and no manifest + // is written without its verdict; its bind evaluates pending again, which takes the merge in with its verdicts. + verdictsBound := b.ix.Verdicts + b.ix.Verdicts = &memStore{unbound: process.ErrNotYet} b.arrive(t, 1, b.merge(20, sha('b'), "tree-avahi-2", "")) + if b.fp(t, "avahi") != avahi0 || !reflect.DeepEqual(b.keys(), []string{takein.PendingKey(1)}) { + t.Fatalf("verdicts not on the bus: avahi %s (was %s), pending %v: want the merge waiting, nothing written", b.fp(t, "avahi"), avahi0, b.keys()) + } + b.ix.Verdicts = verdictsBound + judgedBefore := len(verdictsBound.(*memStore).vals) + if err := b.ix.PendingAgain(ctx); err != nil { + t.Fatal(err) + } + if n := len(verdictsBound.(*memStore).vals); n <= judgedBefore { + t.Fatalf("taken in at the bind: %d verdicts (were %d), want the new manifest's", n, judgedBefore) + } + // A merge with no change to the indexer and nothing pending is taken in alone, as before. if avahi1 := b.fp(t, "avahi"); avahi1 == avahi0 || len(b.keys()) != 0 || len(b.said) != 0 { t.Fatalf("alone: avahi %s (was %s), pending %v, said %v: want taken in", avahi1, avahi0, b.keys(), b.events()) } From b48e9ca6ce981df439ac5ca5e18b60fef98cd2d5 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 10 Nov 2026 03:46:09 +0100 Subject: [PATCH 11/22] =?UTF-8?q?D6k:=20keep=20a=20take-in's=20new=20manif?= =?UTF-8?q?ests=20unjudged=20on=20the=20bus=20until=20a=20judge=20pass=20j?= =?UTF-8?q?udges=20them=20(issue=201966,=20ADR=200668=20=C2=A74)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A judge pass that failed lost the set of manifests the take-in newly wrote, which lived only in its memory: the next pass found no verdict, took each as accepted before any verdict (ADR 0579 §1), and wrote a refusal as a warning. The take-in now keeps each new digest in the indexer's own state unjudged before it writes the module naming it; every judge pass judges those as fresh and ends a key once its verdict is written or no module holds it. indexer_read_unjudged reads the state; indexer_judge judges it again by hand. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../mesh-delivery-indexer/integration_test.go | 2 +- .../cmd/mesh-delivery-indexer/main.go | 5 +- .../internal/index/candidates.go | 3 + .../internal/index/pending.go | 2 + .../internal/index/pending_test.go | 2 +- .../internal/index/toolchains.go | 4 + .../internal/index/unjudged.go | 111 ++++++++++++++++++ .../internal/index/verdicts.go | 8 ++ .../internal/index/verdicts_test.go | 32 +++++ .../internal/index/views.go | 2 + .../mesh-delivery-indexer/manifest_test.go | 4 +- modules/mesh-delivery-indexer/module.json | 4 +- 12 files changed, 173 insertions(+), 6 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/unjudged.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index a9e3c7c8..09d21fa0 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -87,7 +87,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 // The states and the stream the controller makes from the indexer's manifest (old sections, ADR 0595 §11). for bucket, history := range map[string]uint8{index.PendingBucket: 1, index.VerdictsBucket: index.VerdictsHistory, index.GraphBucket: 1, index.TakenBucket: 1, index.PackagesBucket: 1, - index.SeatDefinitionsBucket: 1, index.RequirementsBucket: 1, findingsBucket: 1, index.HoldsBucket: 1} { + index.SeatDefinitionsBucket: 1, index.RequirementsBucket: 1, findingsBucket: 1, index.HoldsBucket: 1, index.UnjudgedBucket: 1} { if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucket, History: history}); err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index d6bce20d..23d17e0e 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -415,6 +415,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The module graph (ADR 0603 §2): a state this delivery brings, bound when it appears; until then no graph is written. graph := process.AwaitBucket(ctx, js, conn, index.GraphBucket, "no module graph is written", logger.Printf) ownBuildState := process.AwaitBucket(ctx, js, conn, index.OwnBuildBucket, "which build this indexer runs is not kept", logger.Printf) + // The manifests a take-in wrote and no judge pass judged yet (issue 1966): a state this delivery brings, bound when it + // appears; until then a take-in's new manifests are kept for their judging in its memory alone. + unjudged := process.AwaitBucket(ctx, js, conn, index.UnjudgedBucket, "a take-in's new manifests are kept for their judging in its memory alone", logger.Printf) tryRecovery := process.AwaitBucket(ctx, js, conn, index.TryRecoveryBucket, "how the recovery of the open tries went is not kept", logger.Printf) findings := process.AwaitBucket(ctx, js, conn, findingsBucket, "no finding of the indexer is written", logger.Printf) // The seat definitions (design 26, DLV-I17): a state this delivery brings, bound when it appears; until then none is kept. @@ -423,7 +426,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro seatBus := &seatBus{defs: awaitedSource{ctx, seatDefs}, marks: awaitedSource{ctx, marks}} seatReader, saySeats := followSeatList(seatBus, saySeatDefinition(js, cred), logger.Printf) readWhenBound(seatReader, seatDefs, marks) - ix := &index.Indexer{Findings: awaitedStore{b: findings}, OldSections: awaitedStore{b: oldSections}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known, + ix := &index.Indexer{Findings: awaitedStore{b: findings}, OldSections: awaitedStore{b: oldSections}, Unjudged: awaitedStore{b: unjudged}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known, Verdicts: awaitedStore{b: verdicts}, SayRefused: sayRefused(conn, cred, logger.Printf), SayAccepted: sayAccepted(conn, cred), Predefined: awaitedStore{b: predefined}, ConfigCandidates: awaitedStore{b: configCandidates}, Logf: logger.Printf, Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call), diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 191eca68..54463a6b 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -68,6 +68,9 @@ type Indexer struct { // OldSections is the state of how many accepted modules use each old section (oldsections.go, issue 1966 D4); nil keeps no // count and the judge refuses no use of an old section. OldSections Store + // Unjudged is the state of the manifest digests a take-in wrote and no judge pass judged yet (unjudged.go, issue 1966); + // nil keeps them only in the take-in's memory. + Unjudged Store // Requirements is the requirements state (requirements.go, DLV-I7): what each consumer's go.sum requires of each mesh // Go module, kept at each merge; nil keeps none. Requirements Store diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index 46b42f5c..962b8ff2 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -877,6 +877,8 @@ func (ix *Indexer) dry() *dryRun { x.Predefined, d.predefined = over(ix.Predefined) x.ConfigCandidates, d.configs = over(ix.ConfigCandidates) x.OldSections, d.oldSections = over(ix.OldSections) + // What the trunk holds unjudged is read through, as fresh; what the dry run would keep stays in its overlay (issue 1966). + x.Unjudged, _ = over(ix.Unjudged) ix.mu.Lock() x.source = sourceNow{rev: ix.source.rev, heads: maps.Clone(ix.source.heads)} x.previews = maps.Clone(ix.previews) diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index 91d401c3..672aa599 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -901,7 +901,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) t.Run("the dry run names every field of the indexer", func(t *testing.T) { // dry (pending.go) builds an indexer field by field: a field added to Indexer is named there, or this fails. named := map[string]bool{} - for _, f := range []string{"Store", "Modules", "Manifests", "Predefined", "ConfigCandidates", "Taken", "Verdicts", "OldSections", + for _, f := range []string{"Store", "Modules", "Manifests", "Predefined", "ConfigCandidates", "Taken", "Verdicts", "OldSections", "Unjudged", "JudgeID", "SayRefused", "SayAccepted", "Packages", "Graph", "Files", "GoSum", "GoMod", "Now", "Builds", "Named", "Trunk", "Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn", "Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "Holds", "Counted", "AskEngines", "HoldBounds", "OpenTries", "TryRecovery", "OwnBuildState"} { diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index 71044311..2ffb0617 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -415,6 +415,10 @@ func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string] } if next.Manifest != "" && (!held || cur.Manifest != next.Manifest) { w.taken(next.Manifest) // a new module, or a changed manifest: fresh, judged in full (ADR 0579 §1) + // Kept on the bus before the module names it, so a judge pass that fails does not lose it (issue 1966). + if err := ix.keepUnjudged(ctx, next.Manifest, key, w); err != nil { + return nil, false, err + } } data, err := meshcatalogclient.EncodeModule(next) wrote = err == nil && !unchanged(data, false, value, found) diff --git a/modules/mesh-delivery-indexer/internal/index/unjudged.go b/modules/mesh-delivery-indexer/internal/index/unjudged.go new file mode 100644 index 00000000..772551b8 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/unjudged.go @@ -0,0 +1,111 @@ +package index + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "slices" + + "git.novox.be/novox/mesh-sdk/go/process" +) + +// What a take-in newly wrote is kept until judged (issue 1966, ADR 0668 §4; ADR 0579 §1). A manifest a take-in writes new +// is fresh: judged in full, never taken as accepted before. A judge pass that fails (the verdicts state not on the bus, a +// restart between the write and the pass) used to lose that set, which lived only in the take-in's memory: the next pass +// found no verdict, took the manifest as held before any verdict, and wrote a refusal as a warning. So the take-in writes +// each digest into the indexer's own state unjudged before it writes the module naming it, every judge pass judges those +// digests as fresh, and a pass deletes a digest's key once its verdict is written or no module holds it any more. The +// tool indexer_read_unjudged reads the state; indexer_judge judges again by hand, which ends what it holds. +const ( + // UnjudgedState is the indexer's state of the digests a take-in wrote and no judge pass judged yet: the bucket + // mesh-delivery-indexer_unjudged, one key per manifest digest. + UnjudgedState = "unjudged" + UnjudgedBucket = Module + "_" + UnjudgedState +) + +// UnjudgedValue is one key of the unjudged state: the module whose new manifest the digest is, and the merge that wrote it. +type UnjudgedValue struct { + V int `json:"v"` + Module string `json:"module"` + Repository string `json:"repository,omitempty"` + Merge string `json:"merge,omitempty"` +} + +// keepUnjudged writes the digest into the unjudged state before the module naming it is written. A state not on the bus +// yet (an indexer whose delivery brings it) is said and the take-in goes on, judged from its memory alone as before; +// any other failure fails the write, so the module is not written with its digest unkept. +func (ix *Indexer) keepUnjudged(ctx context.Context, digest, module string, w *mergeWrite) error { + if ix.Unjudged == nil { + return nil + } + v := UnjudgedValue{V: 1, Module: module} + if w != nil { + v.Repository, v.Merge = w.repository, w.merge + } + data, err := json.Marshal(v) + if err != nil { + return err + } + err = write(ctx, ix.Unjudged, digest, func(value []byte, found bool) ([]byte, bool, error) { + if found { + return nil, false, nil // kept already: the first take-in that wrote it stands + } + return data, false, nil + }) + if errors.Is(err, process.ErrNotYet) { + ix.logf("judge: the state %s is not on the bus yet, so the new manifest %s of %s is kept for its judging only in this take-in", UnjudgedBucket, digest, module) + return nil + } + if err != nil { + return fmt.Errorf("%s.%s: %w", UnjudgedState, digest, err) + } + return nil +} + +// withUnjudged is fresh with every digest the unjudged state holds; a state that cannot be listed is said and fresh +// alone is judged. +func (ix *Indexer) withUnjudged(ctx context.Context, fresh map[string]bool) map[string]bool { + if ix.Unjudged == nil { + return fresh + } + keys, err := ix.Unjudged.Keys(ctx) + if err != nil { + if !errors.Is(err, process.ErrNotYet) { + ix.logf("judge: the state %s cannot be listed (%v), so only this take-in's new manifests are judged as new", UnjudgedBucket, err) + } + return fresh + } + if len(keys) == 0 { + return fresh + } + out := maps.Clone(fresh) + if out == nil { + out = map[string]bool{} + } + for _, k := range keys { + out[k] = true + } + return out +} + +// endUnjudged deletes each key of the unjudged state whose digest the pass judged with its verdict written, or which no +// module holds any more (a later merge replaced it): what is left waits for the next pass. +func (ix *Indexer) endUnjudged(ctx context.Context, judged, held map[string]bool) { + if ix.Unjudged == nil { + return + } + keys, err := ix.Unjudged.Keys(ctx) + if err != nil { + return + } + for _, k := range slices.Sorted(slices.Values(keys)) { + if held[k] && !judged[k] { + continue + } + if err := write(ctx, ix.Unjudged, k, func(_ []byte, found bool) ([]byte, bool, error) { return nil, found, nil }); err != nil { + ix.logf("judge: %s.%s is not ended: %v; judged again as new by the next pass", UnjudgedState, k, err) + } + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/verdicts.go b/modules/mesh-delivery-indexer/internal/index/verdicts.go index 75bdf287..0746bb07 100644 --- a/modules/mesh-delivery-indexer/internal/index/verdicts.go +++ b/modules/mesh-delivery-indexer/internal/index/verdicts.go @@ -84,11 +84,17 @@ func (ix *Indexer) judgeOverHeld(ctx context.Context, fresh map[string]bool, ove } if over == nil { ix.SayUnsaid(ctx) // a manifest-accepted a pass before failed to publish is said again first (DLV-I9b) + // What a take-in wrote and no pass judged yet is fresh too, whichever pass judges it (unjudged.go, issue 1966). + fresh = ix.withUnjudged(ctx, fresh) } held, err := ix.heldModules(ctx) if err != nil { return out, err } + heldDigests, judgedDigests := map[string]bool{}, map[string]bool{} + for _, m := range held { + heldDigests[m.Manifest] = m.Manifest != "" + } trunkOf := map[string]string{} // a judged change's module to the trunk's manifest digest it changes (DLV-I15) // The known packages a manifest's `packages` may name (DLV-I14). The digest judged the trunk does not hold (a pull // request's candidate, one the trunk no longer names) is not judged on them: the checker judges a change's names, @@ -212,6 +218,7 @@ func (ix *Indexer) judgeOverHeld(ctx context.Context, fresh map[string]bool, ove ix.logf("judge: the verdict on %s (digest %s) is not written: %v", name, digests[name], err) continue } + judgedDigests[digests[name]] = true if wrote && over != nil { out.Written++ ix.logf("judge: %s judged again by %s: %s", digests[name], id, v.State) @@ -243,6 +250,7 @@ func (ix *Indexer) judgeOverHeld(ctx context.Context, fresh map[string]bool, ove if over != nil { return out, nil } + ix.endUnjudged(ctx, judgedDigests, heldDigests) if err := ix.keepOldSections(ctx, judge.CountOldSections(accepted), judge.CountOwnerReads(accepted)); err != nil { ix.logf("judge: the count of the old sections is not kept: %v", err) } diff --git a/modules/mesh-delivery-indexer/internal/index/verdicts_test.go b/modules/mesh-delivery-indexer/internal/index/verdicts_test.go index 681b4cfd..204c20b5 100644 --- a/modules/mesh-delivery-indexer/internal/index/verdicts_test.go +++ b/modules/mesh-delivery-indexer/internal/index/verdicts_test.go @@ -13,6 +13,7 @@ import ( "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-indexer/judge" ) @@ -137,6 +138,37 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) { if err := ix.writeOverlay(ctx, map[string]meshcatalog.ModulesByModuleValue{"old": same}, nil, w); err != nil || !w.fresh[other] { t.Fatalf("a changed digest is not fresh: %v %v", w.fresh, err) } + // 1966 (ADR 0668 §4, ADR 0579 §1): what a take-in newly wrote is kept on the bus until judged. A take-in writes a + // manifest a judge refuses in full, its judge pass fails on the verdicts state, and the next pass, which knows nothing + // of that take-in (a start, a pass kicked by a message), still refuses it in full, never as a warning, and ends the key. + ix.Unjudged = newMem() + keptDigest, err := ix.keepManifest(ctx, "kept", `{"module":"kept","version":"1","requires":["amqp"]}`) + if err != nil { + t.Fatal(err) + } + kw := &mergeWrite{repository: "novox/mesh-catalog", merge: "m-kept"} + keptValue := meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("e", 64), Manifest: keptDigest, Repository: "novox/mesh-catalog", Commit: "c3"} + if err := ix.writeOverlay(ctx, map[string]meshcatalog.ModulesByModuleValue{"kept": keptValue}, nil, kw); err != nil || !kw.fresh[keptDigest] { + t.Fatalf("the take-in's write: %v, fresh %v", err, kw.fresh) + } + if value, _, found, _ := ix.Unjudged.Get(ctx, keptDigest); !found || !strings.Contains(string(value), `"module":"kept"`) || !strings.Contains(string(value), "m-kept") { + t.Fatalf("the new manifest is not kept as unjudged: %s", value) + } + ix.Verdicts.(*memStore).failWrite = func(string) error { return process.ErrNotYet } + if _, err := ix.JudgeAll(ctx, kw.fresh); err == nil { + t.Fatal("a pass whose verdicts state is not on the bus did not fail") + } + ix.Verdicts.(*memStore).failWrite = nil + if j, err := ix.JudgeAll(ctx, nil); err != nil || slices.Contains(j.Outgrown, "kept") { + t.Fatalf("the next pass: %+v %v", j, err) + } + if v := verdict(keptDigest); !meshcatalogclient.VerdictRefuses(v) || len(v.Warnings) != 0 { + t.Fatalf("a refusal of a manifest whose judge pass failed became %+v", v) + } + if keys, err := ix.Unjudged.Keys(ctx); err != nil || len(keys) != 0 { + t.Fatalf("the unjudged state still holds %v (%v) after its verdict is written", keys, err) + } + ix.Unjudged = nil // 1966/AC10: a newly accepted manifest asking for bus state is said once as manifest-accepted, carrying both entries // whole; a verdict already held says nothing. var accepted []meshcatalog.ManifestAcceptedEvent diff --git a/modules/mesh-delivery-indexer/internal/index/views.go b/modules/mesh-delivery-indexer/internal/index/views.go index 3bb6ac6f..7bcffe38 100644 --- a/modules/mesh-delivery-indexer/internal/index/views.go +++ b/modules/mesh-delivery-indexer/internal/index/views.go @@ -32,6 +32,7 @@ var ReadTools = []ViewedTool{ {Tool: "indexer_read_taken", State: "taken", Words: "each repository's merge the indexer took in last."}, {Tool: "indexer_read_findings", State: mcc.FindingsState, Words: "the indexer's findings, which the condition keeper raises and clears."}, {Tool: "indexer_read_seat_definitions", State: SeatDefinitionsState, Words: "the seat definitions the trunk's and each open pull request's contracts modules hold, resolved, or why one does not parse."}, + {Tool: "indexer_read_unjudged", State: UnjudgedState, Words: "the manifests a take-in newly wrote that no judge pass judged yet, by digest, each with its module and merge; indexer_judge judges them."}, } // ViewedTool is one tool over one state of the indexer's. @@ -53,4 +54,5 @@ var ViewedStates = map[string]string{ "taken": TakenBucket, mcc.FindingsState: mcc.FindingsBucket(Module), SeatDefinitionsState: SeatDefinitionsBucket, + UnjudgedState: UnjudgedBucket, } diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 1519ab5f..c5fa5e79 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -116,7 +116,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: "try-recovery", History: 1}, {Name: "own-build", History: 1}, {Name: index.PackagesState, History: index.PackagesHistory}, {Name: graph.State, History: graph.History}, {Name: views.MarksState, History: 1}, {Name: mcc.FindingsState, History: 5}, {Name: views.StatusState, History: 1}, {Name: index.RequirementsState, History: 5}, {Name: index.SeatDefinitionsState, History: 5}, - {Name: index.HoldsState, History: 5}, {Name: index.EngineReadsState, History: 1}, {Name: index.HoldBoundsState, History: 1, KeyTTL: true}, {Name: index.PendingState, History: 1}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17, ADR 0632 §2 + {Name: index.HoldsState, History: 5}, {Name: index.EngineReadsState, History: 1}, {Name: index.HoldBoundsState, History: 1, KeyTTL: true}, {Name: index.PendingState, History: 1}, {Name: index.UnjudgedState, History: 1}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17, ADR 0632 §2 t.Errorf("state = %+v, want %+v", m.State, want) } // D-11 (ADR 0634 §1): the holders and buckets the indexer names (internal/index/names.go) are its manifest's: its own @@ -176,7 +176,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build index.SeatDefinitionTool, index.SeatDefinitionsAgainTool, index.OutOfStepTool, // DLV-I19 index.HoldsTool, index.EngineReadsTool, // ADR 0632 §2: the mesh-wide holds and the engines' reads they weigh index.PendingTool, index.TakeInPendingTool, index.DropPendingTool, index.TryCandidatesTool, index.DropTryCandidateTool, index.RecoverTriesTool, index.TakeInMergeTool, index.FingerprintsAgainTool) // issue 1966; 1585/T12, T13 (ADR 0633 §5) - if !reflect.DeepEqual(m.Tools, want) || len(want) != 39 { + if !reflect.DeepEqual(m.Tools, want) || len(want) != 40 { t.Errorf("tools = %v, want %v", m.Tools, want) } // Every read tool of the indexer a reader of this repository invokes is one it serves (the checker's seventh first diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 67f8e7a5..41198549 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -36,6 +36,7 @@ "indexer_read_taken", "indexer_read_findings", "indexer_read_seat_definitions", + "indexer_read_unjudged", "indexer_first_views", "indexer_first_view_again", "indexer_config_readers", @@ -94,7 +95,8 @@ { "name": "holds", "history": 5 }, { "name": "engine-reads", "history": 1 }, { "name": "hold-bounds", "history": 1, "key-ttl": true }, - { "name": "pending", "history": 1 } + { "name": "pending", "history": 1 }, + { "name": "unjudged", "history": 1 } ], "emits": [ "take-in.pending", From 7b3f213ae8d2ab66dd32a9586506df8bf5609dc1 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 10 Nov 2026 03:48:21 +0100 Subject: [PATCH 12/22] =?UTF-8?q?D6i:=20a=20merge=20taken=20in=20before=20?= =?UTF-8?q?the=20verdicts=20state=20binds=20has=20each=20manifest=20judged?= =?UTF-8?q?=20as=20new=20and=20said=20at=20the=20bind,=20in=20the=20indexe?= =?UTF-8?q?r's=20whole-module=20test=20(issue=201966=20note=20154,=20ADR?= =?UTF-8?q?=200668=20=C2=A74)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without the unjudged state the bind's pass accepted the refused manifest with a warning and the waiting one, and said manifest-accepted for both; the row settles Q3: the indexer says what grants follows, so the grants part is unchanged (no D8). Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../mesh-delivery-indexer/integration_test.go | 37 +++++++++++++++++-- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index 09d21fa0..ce7132fd 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -662,8 +662,9 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4, 15 js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds buildLog) { builds.end(t, "novox/mesh-catalog", "avahi", "modules/avahi") }, func(ctx context.Context, js jetstream.JetStream) { - // The verdicts state and the stream of the indexer's events, as the controller makes them from its manifest. - if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.VerdictsBucket, History: index.VerdictsHistory}); err != nil { + // The indexer's state unjudged, as the controller makes it from its manifest; the verdicts state is made only once + // the merge is taken in (issue 1966 note 154: a take-in at the start, before the verdicts state's bind). + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.UnjudgedBucket, History: 1}); err != nil { t.Fatal(err) } }) @@ -707,7 +708,20 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4, 15 t.Errorf("manifest = %+v err %v, want avahi and the file's bytes", got, err) } - verdicts, err := js.KeyValue(ctx, index.VerdictsBucket) + // 1966 (ADR 0668 §4, note 154): the take-in wrote its four manifests with no verdicts state on the bus, so it keeps + // them in its state unjudged; when the verdicts state binds, each is judged as new (bad refused in full, never as a + // warning), every accepted one is said as manifest-accepted, and the state is left empty. + unjudged, err := js.KeyValue(ctx, index.UnjudgedBucket) + if err != nil { + t.Fatal(err) + } + kept, _ := unjudged.Keys(ctx) + for _, raw := range []string{string(manifest), judged["good"], judged["bad"], judged["later"]} { + if !slices.Contains(kept, meshcatalogclient.ManifestDigest([]byte(raw))) { + t.Errorf("the take-in's new manifest %s is not kept unjudged: the state holds %v", meshcatalogclient.ManifestDigest([]byte(raw)), kept) + } + } + verdicts, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.VerdictsBucket, History: index.VerdictsHistory}) if err != nil { t.Fatal(err) } @@ -732,12 +746,20 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4, 15 if !strings.Contains(strings.Join(got["bad"].Problems, " "), `provides "amqp"`) || strings.Join(got["later"].Unknown, " ") != "a-field-from-later" { t.Errorf("the refusal names no problem or the wait no field: %+v", got) } + // The pass ends the state's keys after it wrote and said every verdict. + left, _ := unjudged.Keys(ctx) + for deadline := time.Now().Add(10 * time.Second); len(left) != 0 && time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { + left, _ = unjudged.Keys(ctx) + } + if len(left) != 0 { + t.Errorf("the state unjudged still holds %v within 10 s, once every manifest has its verdict", left) + } // manifest-refused, said once for each refusal (a wait is no refusal). events, err := js.Stream(ctx, "EVENTS") if err != nil { t.Fatal(err) } - said := map[string]bool{} + said, acceptedSaid := map[string]bool{}, map[string]bool{} for seq := uint64(1); ; seq++ { msg, err := events.GetMsg(ctx, seq) if err != nil { @@ -747,10 +769,17 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4, 15 if msg.Subject == "mesh.seat.mesh-catalog.event."+meshcatalog.EventManifestRefused && json.Unmarshal(msg.Data, &r) == nil { said[r.Module] = true } + var a meshcatalog.ManifestAcceptedEvent + if msg.Subject == "mesh.seat.mesh-catalog.event."+meshcatalog.EventManifestAccepted && json.Unmarshal(msg.Data, &a) == nil { + acceptedSaid[a.Module] = true + } } if !said["bad"] || said["later"] || said["good"] { t.Errorf("manifest-refused said for %v, want bad only", said) } + if !acceptedSaid["good"] || acceptedSaid["bad"] || acceptedSaid["later"] { + t.Errorf("manifest-accepted said for %v, want good and no refused or waiting one", acceptedSaid) + } // How a composer holds a refused module by these verdicts is the node-engine's to test (ADR 0593 §7: the composer // and its bus reader are its own). } From 17dd41696cdb2a958477f1eb86d929573055828b Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 10 Nov 2026 03:52:04 +0100 Subject: [PATCH 13/22] =?UTF-8?q?D4f:=20a=20merged=20message=20still=20unr?= =?UTF-8?q?ead=20five=20minutes=20on,=20the=20forge=20heard,=20is=20the=20?= =?UTF-8?q?indexer's=20finding=20call-unanswered=20(issue=201966=20note=20?= =?UTF-8?q?153,=20ADR=200668=20=C2=A73)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A message heard on the forge is the forge bridge heard, the callee of the call: after each one a merge unread since CallUnansweredAfter or longer writes the finding naming the bridge's call, and a merge no longer unread clears it. While nothing is heard on the forge (the bridge's machine away) no finding is written; the merge is read again when the forge is heard. No timer. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../internal/index/pending.go | 27 ++++++-- .../internal/index/pending_test.go | 38 ++++++++++ .../internal/index/unreadcall.go | 69 +++++++++++++++++++ 3 files changed, 129 insertions(+), 5 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/unreadcall.go diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index 962b8ff2..238fa5e1 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -92,6 +92,9 @@ type Pending struct { // ADR 0664's classes): not lost, it is read again on the next message heard on the forge, at the start and by // indexer_take_in_pending (issue 1966 note 153, ADR 0668 §1). Unread string `json:"unread,omitempty"` + // UnreadSince is when the message was first found unread: past meshconditionsclient.CallUnansweredAfter, with the forge + // heard since, it is the finding call-unanswered (unreadcall.go, ADR 0668 §3). + UnreadSince time.Time `json:"unread-since,omitzero"` } // PendingEntry is one key of the pending state as read: its key, revision, value and the merged message decoded. For a @@ -698,10 +701,17 @@ func (ix *Indexer) UnreadAgain(ctx context.Context) error { if err != nil { return err } - if !slices.ContainsFunc(entries, func(e PendingEntry) bool { return e.Value.Unread != "" }) { - return nil + if slices.ContainsFunc(entries, func(e PendingEntry) bool { return e.Value.Unread != "" }) { + if err := ix.PendingAgain(ctx); err != nil { + return err + } + if entries, err = ReadPending(ctx, ix.Pending, ix.logf); err != nil { + return err + } } - return ix.PendingAgain(ctx) + // The forge was heard, so the bridge that says it is: a read still failing past the bound is a finding (ADR 0668 §3). + ix.unreadFindings(ctx, entries) + return nil } // PendingAgain evaluates pending again after something a fingerprint is computed from changed, a toolchain report @@ -817,9 +827,16 @@ func (ix *Indexer) holdLost(ctx context.Context, ready, lost []PendingEntry, rea } for _, g := range lost { reason, isUnread := reasons[g.Key], unread[g.Key] + now := ix.now().UTC() ix.markSaid(ctx, []PendingEntry{g}, &t, func(p *Pending) { - if p.Lost, p.Unread = reason, ""; isUnread { - p.Lost, p.Unread = "", reason + p.Lost, p.Unread = reason, "" + if !isUnread { + p.UnreadSince = time.Time{} + return + } + p.Lost, p.Unread = "", reason + if p.UnreadSince.IsZero() { + p.UnreadSince = now } }) } diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index 672aa599..066f9450 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -11,6 +11,8 @@ import ( "testing" "time" + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" @@ -584,6 +586,8 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) {"the bridge's answer", &busretry.Answered{Words: "no merge at that commit"}, false}, } { b = held() + clock := time.Date(2026, 11, 10, 1, 0, 0, 0, time.UTC) + b.ix.Now, b.ix.Findings = func() time.Time { return clock }, newMem() answers := b.ix.Message b.ix.held = nil b.ix.Message = func(context.Context, string, string, int) ([]byte, error) { return nil, c.fault } @@ -594,6 +598,37 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) if len(waits) != 1 || (len(waits[0].Unread) == 1) != c.unread || (len(waits[0].Lost) == 1) == c.unread { t.Fatalf("%s: indexer_pending %+v: want unread %v, lost %v", c.name, waits, c.unread, !c.unread) } + // ADR 0668 §3: unread within five minutes is no finding; past them, with nothing heard on the forge (the bridge's + // machine away, only a toolchain report heard), none either; the next message heard on the forge, the read still + // failing, writes call-unanswered naming the bridge's call; the merge read and taken in clears it. + finding := func() (string, bool) { + keys, _ := b.ix.Findings.Keys(ctx) + for _, k := range keys { + if strings.HasSuffix(k, "."+mcc.KindCallUnanswered) { + v, _, _, _ := b.ix.Findings.Get(ctx, k) + return string(v), true + } + } + return "", false + } + if c.unread { + for _, step := range []struct { + after time.Duration + heard bool + finding bool + }{{time.Minute, true, false}, {5 * time.Minute, false, false}, {0, true, true}} { + clock = clock.Add(step.after) + var err error + if step.heard { + err = b.ix.UnreadAgain(ctx) + } else { + err = b.ix.PendingAgain(ctx) + } + if v, found := finding(); err != nil || found != step.finding || found && !strings.Contains(v, UnreadCallee) { + t.Fatalf("%s, %s later, forge heard %v: finding %v %s (%v), want %v", c.name, step.after, step.heard, found, v, err, step.finding) + } + } + } b.ix.Message = answers if err := b.ix.UnreadAgain(ctx); err != nil { t.Fatal(err) @@ -601,6 +636,9 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) if taken := len(b.keys()) == 0 && b.commit(t, "avahi") == sha('d'); taken != c.unread { t.Errorf("%s, a message heard after: pending %v, avahi at %s: want taken in %v", c.name, b.keys(), b.commit(t, "avahi"), c.unread) } + if v, found := finding(); found { + t.Errorf("%s: the finding stands after the merge was taken in: %s", c.name, v) + } } }) t.Run("T5 two take-ins of one pending revision: the one that loses the claim writes nothing", func(t *testing.T) { diff --git a/modules/mesh-delivery-indexer/internal/index/unreadcall.go b/modules/mesh-delivery-indexer/internal/index/unreadcall.go new file mode 100644 index 00000000..e8d2f1da --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/unreadcall.go @@ -0,0 +1,69 @@ +package index + +import ( + "context" + "strings" + + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" +) + +// **A merged message unread past five minutes, the forge heard, is a finding** (issue 1966 note 153; novox/hq ADR 0668 §3). +// The indexer's call for a merged message is a kept call: a transport fault leaves the merge unread (Pending.Unread) and +// it is read again after every message heard on the forge (UnreadAgain). A message heard on the forge is the forge bridge +// heard, the callee of that call: so after each one, a merge still unread since meshconditionsclient.CallUnansweredAfter +// or longer is the finding call-unanswered in the indexer's findings, which the condition keeper raises, and a merge no +// longer unread (read, taken in, covered or dropped) clears its finding. While nothing is heard on the forge (the bridge's +// machine away) no finding is written: the merge stays unread and is read again when the forge is heard. No timer. + +// UnreadCallee is the call the finding names: the forge bridge's tool for one merged message. +const UnreadCallee = ForgeBridgeModule + ".bridge_merged_message" + +// unreadCallID is the finding's id for the pending key: the merge it waits on. +func unreadCallID(key string) string { return "merged-message." + key } + +// unreadFindings raises the finding of every merge of entries unread past the bound and clears the finding of every other. +func (ix *Indexer) unreadFindings(ctx context.Context, entries []PendingEntry) { + if ix.Findings == nil { + return + } + unread := map[string]bool{} + for _, e := range entries { + if e.Value.Unread == "" { + continue + } + unread[mcc.Key(mcc.ScopeCall, unreadCallID(e.Key), mcc.KindCallUnanswered)] = true + since := e.Value.UnreadSince + if since.IsZero() || ix.now().Sub(since) < mcc.CallUnansweredAfter { + continue + } + o := mcc.CallUnanswered(unreadCallID(e.Key), UnreadCallee, "", e.Value.Unread, "indexer."+mcc.KindCallUnanswered, since) + o.Needs = TakeInPendingTool + " reads the merge again now; " + PendingTool + " shows it unread." + key := o.Key() + if err := write(ctx, ix.Findings, key, func(value []byte, found bool) ([]byte, bool, error) { + if found { + if f, err := mcc.DecodeFinding(value); err == nil && f.Summary == o.Summary { + return nil, false, nil + } + } + data, err := mcc.EncodeFinding(o.Finding(Module, "", ix.now().UTC())) + return data, false, err + }); err != nil { + ix.logf("take-in: the finding %s is not written: %v", key, err) + } else { + ix.logf("take-in: %s (%s#%d) is unread since %s and the forge is heard: the finding %s", e.Key, e.Forge.Repository, e.Forge.Number, since.Format("2006-01-02T15:04:05Z07:00"), key) + } + } + keys, err := ix.Findings.Keys(ctx) + if err != nil { + return + } + prefix, suffix := mcc.ScopeCall+"."+unreadCallID(""), "."+mcc.KindCallUnanswered + for _, k := range keys { + if !strings.HasPrefix(k, prefix) || !strings.HasSuffix(k, suffix) || unread[k] { + continue + } + if err := write(ctx, ix.Findings, k, func(_ []byte, found bool) ([]byte, bool, error) { return nil, found, nil }); err != nil { + ix.logf("take-in: the finding %s is not cleared: %v", k, err) + } + } +} From bd39747c22f57957d8067b144e9df6bdfab5a52d Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 10 Nov 2026 03:53:05 +0100 Subject: [PATCH 14/22] =?UTF-8?q?D1i:=20a=20row=20for=20the=20indexer's=20?= =?UTF-8?q?bind:=20every=20counted=20node's=20manifest-reads=20is=20called?= =?UTF-8?q?=20once=20(issue=201966=20note=20155,=20ADR=200668=20=C2=A74)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-indexer/main_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go index 40e32bfe..1bb57db3 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go @@ -12,6 +12,7 @@ import ( "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" @@ -79,6 +80,20 @@ func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part) if em := viewsEmitter(context.Background(), nil, process.Account{}, nil, &process.AwaitedBucket{}, nil, t.Logf); em.Findings == nil { t.Error("the indexer's views emitter writes no view-unsaid finding") } + t.Run("the consumer bound: every counted node's manifest-reads called once", func(t *testing.T) { // 1966 note 155, ADR 0668 §4 + // A manifest-reads-changed said before the indexer's consumer existed is never delivered: the hub's OnBound + // (reads.atBind) calls every node the assignments counts, so a stale statement ends at the bind. + var called []string + reads := &engineReads{known: true, nodes: []string{"ace", "g14"}, keeper: &enginereads.Keeper{Sync: true, + Call: func(node, _ string, _ any) (json.RawMessage, error) { + called = append(called, node) + return nil, errors.New("no responders") + }}} + reads.atBind(time.Unix(0, 0)) + if strings.Join(called, ",") != "ace,g14" { + t.Errorf("at the bind called %v, want ace and g14 once each", called) + } + }) t.Run("not a forge message: terminated and logged", func(t *testing.T) { // 1238 (never redelivered) var logs bytes.Buffer st := &memStore{vals: map[string][]byte{}} From cddaf14cd0fa5e3919cd0752917a3fac44407b98 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 10 Nov 2026 04:11:07 +0100 Subject: [PATCH 15/22] =?UTF-8?q?D7a:=20the=20assigner,=20the=20indexer=20?= =?UTF-8?q?and=20the=20builder=20ask=20a=20node's=20manifest-reads=20once?= =?UTF-8?q?=20before=20the=20kept=20call,=20so=20a=20node=20that=20answers?= =?UTF-8?q?=20nothing=20reads=20as=20its=20fallback=20at=20once=20(issue?= =?UTF-8?q?=201966,=20ADR=200668=20=C2=A73;=20review=20round=201,=20findin?= =?UTF-8?q?g=201)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go | 1 + modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go | 1 + modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go | 1 + 3 files changed, 3 insertions(+) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 1f51d4bc..9fb35753 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -317,6 +317,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, reads := &enginereads.Keeper{Fallback: w.recordReads, Changed: wake, Logf: logger.Printf} if bus != nil { reads.Call = bus.CallKeptOn // a kept call, called again through a transport fault (ADR 0668 §1) + reads.Once = bus.CallOn // its fault keeps the fallback at once while the kept call asks on (ADR 0668 §3) } if kv := kvs[engineReadsBucket]; kv != nil { reads.Put = func(node string, v []byte) error { _, err := kv.Put(ctx, node, v); return err } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 71dd3e3d..71e15570 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -270,6 +270,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // What each counted node's engine reads (ADR 0618 §2, ADR 0624 §1): asked through the node-engine seat's verb as the // builder's own account, for every node the assignments names, and kept in the state engine-reads once it is bound. reads := newEngineReads(calls.CallKeptOn, gate, logger.Printf) + reads.keeper.Once = calls.CallOn // its fault keeps the last statement at once while the kept call asks on (ADR 0668 §3) // The old reference form's window over them (DLV-B9), kept in the state old-form once it is bound; builder_old_form // reads it and drops a retired node by hand. reads.ctx = context.WithoutCancel(ctx) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 23d17e0e..d973cc3f 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -493,6 +493,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // §2): asked through the node-engine seat's verb as the indexer's own account, for every node the assignments names, // kept in the state engine-reads once it is bound; each change weighs the holds again. reads := newEngineReads(calls.CallKeptOn, gate, func() { ix.KickHolds(ctx) }, logger.Printf) + reads.keeper.Once = calls.CallOn // its fault keeps the last statement at once while the kept call asks on (ADR 0668 §3) gate.OnPlaced = reads.placed ix.Counted, ix.AskEngines = reads.counted, reads.ask // The holds, new with ADR 0632 (issue 1790's kind), bound when they appear: until then a merge that would hold a From 8f50f22622cc09fdd2831aa66f950320a785ad95 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 10 Nov 2026 04:13:02 +0100 Subject: [PATCH 16/22] =?UTF-8?q?D8:=20a=20trunk's=20held=20manifest=20jud?= =?UTF-8?q?ged=20by=20its=20digest=20is=20said=20as=20manifest-accepted=20?= =?UTF-8?q?or=20manifest-refused,=20as=20the=20full=20pass=20says=20it=20(?= =?UTF-8?q?issue=201966,=20ADR=200668=20=C2=A74;=20review=20round=201,=20f?= =?UTF-8?q?inding=203)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit indexer_judge with a digest and the start pass's judgeStale wrote the verdict and said nothing, so a part waiting on manifest-accepted never heard it. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../internal/index/verdicts.go | 7 ++++++- .../internal/index/verdicts_test.go | 20 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/modules/mesh-delivery-indexer/internal/index/verdicts.go b/modules/mesh-delivery-indexer/internal/index/verdicts.go index 0746bb07..24ce8159 100644 --- a/modules/mesh-delivery-indexer/internal/index/verdicts.go +++ b/modules/mesh-delivery-indexer/internal/index/verdicts.go @@ -101,10 +101,15 @@ func (ix *Indexer) judgeOverHeld(ctx context.Context, fresh map[string]bool, ove // knowing its delivery group, whose new packages this state does not know yet (the lead's ruling, issue 1966). The // trunk's own manifests are, and a merge's are again in full (namesAtMerge). known := ix.knownPackages(ctx) + // A digest judged over the trunk that is the trunk's own held one is said as the full pass says it (manifest-accepted, + // manifest-refused): indexer_judge with it, or the start pass's judgeStale, is that manifest's only judgement (review + // round 1 of issue 1966; ADR 0668 §4). A candidate's verdict is said nowhere. + onTrunk := map[string]bool{} if over != nil { fresh = map[string]bool{} for module, digest := range over { m := held[module] + onTrunk[module] = m.Manifest == digest if m.Manifest != "" && m.Manifest != digest { trunkOf[module] = m.Manifest } @@ -219,7 +224,7 @@ func (ix *Indexer) judgeOverHeld(ctx context.Context, fresh map[string]bool, ove continue } judgedDigests[digests[name]] = true - if wrote && over != nil { + if wrote && over != nil && !onTrunk[name] { out.Written++ ix.logf("judge: %s judged again by %s: %s", digests[name], id, v.State) } else if wrote { diff --git a/modules/mesh-delivery-indexer/internal/index/verdicts_test.go b/modules/mesh-delivery-indexer/internal/index/verdicts_test.go index 204c20b5..09d6cf52 100644 --- a/modules/mesh-delivery-indexer/internal/index/verdicts_test.go +++ b/modules/mesh-delivery-indexer/internal/index/verdicts_test.go @@ -215,6 +215,26 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) { if last := accepted[len(accepted)-1]; last.Module != "plain" || last.Digest != plain || last.BusState == nil || len(last.BusState) != 0 { t.Fatalf("a manifest asking for no bus state was said as %+v", last) } + // Review round 1 of issue 1966: a trunk's held manifest judged by its digest (indexer_judge, judgeStale) is said as the + // full pass says it, accepted or refused; a candidate's digest judged over the trunk is said nowhere. + byDigest := hold("bydigest", `{"module":"bydigest","version":"1"}`) + refusedBy := hold("refusedby", `{"module":"refusedby","version":"1","requires":["amqp"]}`) + saidBefore := len(said) + for module, digest := range map[string]string{"bydigest": byDigest, "refusedby": refusedBy} { + if _, err := ix.judgeOver(ctx, nil, module, digest); err != nil { + t.Fatal(err) + } + } + if last := accepted[len(accepted)-1]; last.Digest != byDigest || len(said) != saidBefore+1 || said[len(said)-1].Digest != refusedBy { + t.Fatalf("judged by its digest, the trunk's manifest was said as %+v, refusals %d (were %d)", last, len(said), saidBefore) + } + candidate, err := ix.keepManifest(ctx, "bydigest", `{"module":"bydigest","version":"2"}`) + if err != nil { + t.Fatal(err) + } + if _, err := ix.judgeOver(ctx, nil, "bydigest", candidate); err != nil || accepted[len(accepted)-1].Digest != byDigest { + t.Fatalf("a candidate's verdict was said: %v %+v", err, accepted[len(accepted)-1]) + } // DLV-I9b: a failed publish is published again; one failing twice writes the finding naming the module, which the next // judging pass says again and clears. ix.Findings = newMem() From 23ddaa8bed9e7202b2cae35a8c07ea344d9750b0 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 10 Nov 2026 04:17:45 +0100 Subject: [PATCH 17/22] =?UTF-8?q?D9:=20the=20assigner=20keeps=20the=20wait?= =?UTF-8?q?=20of=20its=20bind's=20asks=20read=20and=20of=20a=20heard=20nod?= =?UTF-8?q?e's=20manifest-reads,=20and=20says=20each=20unanswered=20past?= =?UTF-8?q?=20five=20minutes=20as=20call-unanswered=20(issue=201966,=20ADR?= =?UTF-8?q?=200668=20=C2=A73;=20review=20round=201,=20finding=204)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bind's read of the operator channel's asks is made again after every transport fault until answered; a node whose engine answers nothing while its machine is heard is asked again at each pass once its ask ended. Past CallUnansweredAfter with the callee's machine heard each writes the finding in the assigner's findings, cleared once answered; none while the machine is away. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-assigner/askended.go | 65 +++++++++++++++-- .../cmd/mesh-delivery-assigner/main.go | 2 +- .../cmd/mesh-delivery-assigner/pass.go | 69 +++++++++++++++++++ .../cmd/mesh-delivery-assigner/pass_test.go | 68 ++++++++++++++++++ .../cmd/mesh-delivery-assigner/world.go | 19 +++++ 5 files changed, 217 insertions(+), 6 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/askended.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/askended.go index 893232ee..7b041c7f 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/askended.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/askended.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "errors" "time" @@ -9,11 +10,15 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents" + "git.novox.be/novox/mesh-sdk/go/busretry" "git.novox.be/novox/mesh-sdk/go/process" + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" "mesh-delivery-assigner/internal/assign" + plc "mesh-delivery-assigner/internal/assignment" "mesh-delivery-assigner/internal/buckets" "mesh-delivery-assigner/internal/puts" ) @@ -50,21 +55,71 @@ const channelAsksVerb = "seat:" + oc.Seat + "." + oc.VerbAsks // assigner's consumer: an ask-ended the teller said before the consumer existed is not delivered to it (issue 1966 note 155, // ADR 0668 §4), so each held ask whose asking ended there expired or cancelled is written again as that event writes it // (askAgain, with every rule and its compare-and-set). The read is a kept call (CallKept), made off the hub's start. -func readEndedAtBind(ctx context.Context, js jetstream.JetStream, hub *process.EventHub, asksKV jetstream.KeyValue, bus caller, logf func(string, ...any)) { +func readEndedAtBind(ctx context.Context, js jetstream.JetStream, hub *process.EventHub, asksKV jetstream.KeyValue, bus caller, wait callWait, logf func(string, ...any)) { if asksKV == nil || bus == nil { return } hub.OnBound(func(created time.Time) { go func() { - if err := endedAtBind(ctx, js, asksKV, bus, created, time.Now(), logf); err != nil { + if err := endedAtBind(ctx, js, asksKV, bus, wait, created, time.Now(), logf); err != nil { logf("the operator channel's ended asks are not read at the bind of %s (%v): an ask that ended there before is asked again when the teller says it ended", created.Format(time.RFC3339), err) } }() }) } -// endedAtBind is readEndedAtBind's one read: the held asks, the seat's verb, and each ended asking written again. -func endedAtBind(ctx context.Context, js jetstream.JetStream, asksKV jetstream.KeyValue, bus caller, created, now time.Time, logf func(string, ...any)) error { +// callWait is how a kept call the assigner waits on is said (novox/hq ADR 0668 §3): heard says the callee's machine is +// heard, now is the clock, and findings the assigner's findings, where the call unanswered past +// meshconditionsclient.CallUnansweredAfter is the finding call-unanswered while heard, cleared when it is answered. +type callWait struct { + heard func() bool + now func() time.Time + findings assign.Findings +} + +// asksCallID is the finding's id for the bind's read of the operator channel's asks. +const asksCallID = "operator-channel-asks" + +// keep makes call again after every transport fault (busretry: a request that reached no one, or one whose deadline +// passed) until it is answered or ctx ends, each try itself a kept call; past the bound with the callee heard it writes +// the finding call-unanswered, and clears it when answered. No timer: each try is the wait. +func (cw callWait) keep(ctx context.Context, id, callee string, call func() (json.RawMessage, error), logf func(string, ...any)) (json.RawMessage, error) { + clock := cw.now + if clock == nil { + clock = time.Now + } + since, said := clock(), false + key := mcc.Key(mcc.ScopeCall, id, mcc.KindCallUnanswered) + for { + answer, err := call() + if err == nil || !(busretry.Undelivered(err) || busretry.TimedOut(err)) || ctx.Err() != nil { + if said && cw.findings.Delete != nil { + if derr := cw.findings.Delete(key); derr != nil { + logf("the finding %s is not cleared: %v", key, derr) + } + } + return answer, err + } + if said || clock().Sub(since) < mcc.CallUnansweredAfter || cw.heard == nil || !cw.heard() || cw.findings.Put == nil { + continue + } + o := mcc.CallUnanswered(id, callee, "", err.Error(), "assigner."+mcc.KindCallUnanswered, since) + value, eerr := mcc.EncodeFinding(o.Finding(plc.Module, "", clock().UTC())) + if eerr == nil { + eerr = cw.findings.Put(key, value) + } + if eerr != nil { + logf("the finding %s is not written: %v", key, eerr) + continue + } + said = true + logf("%s is unanswered since %s and its callee's machine is heard: the finding %s", callee, since.UTC().Format(time.RFC3339), key) + } +} + +// endedAtBind is readEndedAtBind's one read: the held asks, the seat's verb (a kept call whose wait is kept, wait), and +// each ended asking written again. +func endedAtBind(ctx context.Context, js jetstream.JetStream, asksKV jetstream.KeyValue, bus caller, wait callWait, created, now time.Time, logf func(string, ...any)) error { keys, err := asksKV.Keys(ctx) if errors.Is(err, jetstream.ErrNoKeysFound) { return nil @@ -82,7 +137,7 @@ func endedAtBind(ctx context.Context, js jetstream.JetStream, asksKV jetstream.K if !ok { return nil } - answer, err := bus.CallKept(channelAsksVerb, query) + answer, err := wait.keep(ctx, asksCallID, channelAsksVerb, func() (json.RawMessage, error) { return bus.CallKept(channelAsksVerb, query) }, logf) if err != nil { return err } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 9fb35753..a18d9632 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -484,7 +484,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, if err := hearAskEnded(ctx, js, hub, kvs[buckets.Asks], logger.Printf); err != nil { logger.Printf("%s is not heard, so an ask that ends at the operator channel without an answer is not asked again until the assigner starts again: %v", askEndedSubject, err) } - readEndedAtBind(ctx, js, hub, kvs[buckets.Asks], bus, logger.Printf) + readEndedAtBind(ctx, js, hub, kvs[buckets.Asks], bus, callWait{heard: func() bool { return w.seatHeard(oc.Seat) }, findings: findings}, logger.Printf) hearReadsChanged(ctx, js, hub, reads, findings, gate.eventsLive, logger.Printf) } if bus != nil && kvs[buckets.Puts] != nil { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index 4dfc33e8..6f067139 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -24,6 +24,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads" + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "mesh-delivery-assigner/internal/assign" plc "mesh-delivery-assigner/internal/assignment" @@ -42,6 +43,8 @@ type passer struct { said map[string]bool // findings is the assigner's findings bucket (its said conditions, ADR 0564 §2). findings assign.Findings + // unansweredSaid are the call-unanswered findings of manifest-reads this process wrote (ADR 0668 §3). + unansweredSaid map[string]bool // failedProvisions are the failed provisions whose finding this process wrote (ADR 0570 §4). failedProvisions map[string]bool // missing are the required provisions with no key this process pulsed, and missingSaid those whose finding it wrote. @@ -116,6 +119,72 @@ func (p *passer) engineReads(n assign.Nodes) { n.Reads[node] = reads } } + p.unansweredReads(known, n.Online) +} + +// readsCallID is the finding's id for a node's manifest-reads: the node the kept call waits on. +func readsCallID(node string) string { return "manifest-reads." + node } + +// unansweredReads keeps the wait of every node whose engine has not answered manifest-reads while its machine is heard +// (online): it is asked again once its last ask ended, so the kept call goes on until answered; past +// meshconditionsclient.CallUnansweredAfter it is the finding call-unanswered in the assigner's findings, cleared once +// answered or while its machine is away, when none is written (novox/hq ADR 0668 §3). Each pass is the event: no timer. +func (p *passer) unansweredReads(known []string, online map[string]bool) { + if p.reads.Call == nil { + return + } + heard := func(node string) bool { return online[node] } + for _, node := range known { + if s, ok := p.reads.Statements()[node]; ok && !s.Answered && heard(node) && !p.reads.Asking(node) { + p.reads.AskNode(node, "unanswered since "+s.UnansweredSince.UTC().Format(time.RFC3339)+", its machine heard") + } + } + if p.findings.Put == nil || p.findings.Delete == nil { + return + } + now := p.clock() + statements := p.reads.Statements() + raised := map[string]bool{} + for node, since := range p.reads.Unanswered(known, heard, mcc.CallUnansweredAfter, now) { + o := mcc.CallUnanswered(readsCallID(node), enginereads.Invoke, node, statements[node].Words, "assigner."+mcc.KindCallUnanswered, since) + o.Needs = "assigner_engine_reads with ask_again asks the node again now and shows its statement." + key := o.Key() + raised[key] = true + if p.unansweredSaid[key] { + continue + } + value, err := mcc.EncodeFinding(o.Finding(plc.Module, "", now.UTC())) + if err == nil { + err = p.findings.Put(key, value) + } + if err != nil { + p.logf("the finding %s is not written: %v", key, err) + continue + } + p.logf("%s on %s is unanswered since %s and its machine is heard: the finding %s", enginereads.Verb, node, since.UTC().Format(time.RFC3339), key) + if p.unansweredSaid == nil { + p.unansweredSaid = map[string]bool{} + } + p.unansweredSaid[key] = true + } + // What a pass before or another process wrote: every such finding of the assigner's no longer raised is cleared. + prefix, suffix := mcc.ScopeCall+"."+readsCallID(""), "."+mcc.KindCallUnanswered + keys := slices.Collect(maps.Keys(p.unansweredSaid)) + if p.findings.Keys != nil { + if held, err := p.findings.Keys(); err == nil { + keys = append(keys, held...) + } + } + for _, key := range keys { + if raised[key] || !strings.HasPrefix(key, prefix) || !strings.HasSuffix(key, suffix) { + continue + } + if err := p.findings.Delete(key); err != nil { + p.logf("the finding %s is not cleared: %v", key, err) + continue + } + delete(p.unansweredSaid, key) + } } func (p *passer) clock() time.Time { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index 64b17792..50cba6c4 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" stdio "git.novox.be/novox/mesh-sdk/go" + "maps" "mesh-delivery-assigner/internal/buckets" "mesh-delivery-assigner/internal/puts" "slices" @@ -36,6 +37,7 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-delivery/sdk/graph" "mesh-delivery-assigner/internal/assign" @@ -153,6 +155,33 @@ func TestAHoldAskedAgainAfterALiftAndARestartHasANewID(t *testing.T) { t.Errorf("%s: asked again", c.why) } } + // ADR 0668 §3 (review round 1): the bind's read of the operator channel's asks keeps its wait through transport faults; + // past five minutes with the channel's machine heard it is the finding call-unanswered, cleared once answered; with + // the machine away none is written. A refusal is an answer and ends the wait. + for _, heard := range []bool{false, true} { + clock, findings := now, map[string][]byte{} + cw := callWait{heard: func() bool { return heard }, now: func() time.Time { return clock }, + findings: assign.Findings{Put: func(k string, v []byte) error { findings[k] = v; return nil }, + Delete: func(k string) error { delete(findings, k); return nil }}} + tries, during := 0, 0 + answer, err := cw.keep(context.Background(), asksCallID, channelAsksVerb, func() (json.RawMessage, error) { + tries++ + clock = clock.Add(2 * time.Minute) + if tries == 5 { + during = len(findings) + return json.RawMessage(`{"asks":[]}`), nil + } + return nil, errors.New("nats: no responders available for request") + }, t.Logf) + if want := map[bool]int{false: 0, true: 1}[heard]; err != nil || string(answer) != `{"asks":[]}` || tries != 5 || during != want || len(findings) != 0 { + t.Errorf("heard %v: %s %v after %d tries; findings during %d, after %d", heard, answer, err, tries, during, len(findings)) + } + } + if _, err := (callWait{}).keep(context.Background(), asksCallID, channelAsksVerb, func() (json.RawMessage, error) { + return nil, &busretry.Answered{Words: "refused"} + }, t.Logf); err == nil { + t.Error("a refusal was asked again") + } } // 1906/H4: an answer counts only for the asking it answered: an answer to an earlier hold of the same fingerprint, or to an ask @@ -240,6 +269,45 @@ func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) { if asked["ace"] != 2 { t.Errorf("ace, back online, was asked %d times, want 2", asked["ace"]) } + // ADR 0668 §3 (review round 1): a node whose engine answers nothing while its machine is heard is asked again at each + // pass once its ask ended, and past five minutes is the finding call-unanswered, cleared once it answers; none is + // written while its machine is away. + clock := time.Date(2026, 11, 10, 3, 0, 0, 0, time.UTC) + held := map[string][]byte{} + answers := false + silent := func(node, _ string, _ any) (json.RawMessage, error) { + asked[node]++ + if answers { + return json.Marshal(nodeengine.ManifestReadsAnswer{V: 1, Node: node, ManifestReads: []string{"form:new"}, StatedAt: clock.Format(time.RFC3339Nano)}) + } + return nil, context.DeadlineExceeded + } + q := &passer{w: w, logf: t.Logf, now: func() time.Time { return clock }, findings: assign.Findings{ + Put: func(k string, v []byte) error { held[k] = v; return nil }, + Delete: func(k string) error { delete(held, k); return nil }, + Keys: func() ([]string, error) { return slices.Collect(maps.Keys(held)), nil }}, + reads: &enginereads.Keeper{Call: silent, Sync: true, Now: func() time.Time { return clock }}} + key := mcc.Key(mcc.ScopeCall, readsCallID("ace"), mcc.KindCallUnanswered) + before := asked["ace"] + q.engineReads(w.nodes(nil, nil)) + clock = clock.Add(6 * time.Minute) + off, _ := presenceValue(false) + w.put(buckets.Presence, "ace", off) + q.engineReads(w.nodes(nil, nil)) + if len(held) != 0 { + t.Errorf("a finding was written while ace is away: %v", slices.Collect(maps.Keys(held))) + } + w.put(buckets.Presence, "ace", on) + q.engineReads(w.nodes(nil, nil)) + if _, ok := held[key]; !ok || asked["ace"]-before < 3 { + t.Errorf("ace unanswered six minutes, heard: findings %v, asked %d times", slices.Collect(maps.Keys(held)), asked["ace"]-before) + } + answers = true + q.engineReads(w.nodes(nil, nil)) + q.engineReads(w.nodes(nil, nil)) + if _, ok := held[key]; ok { + t.Errorf("the finding stays after ace answered") + } // The contract carries no checks, so its reader judges what the engine says (mesh-contracts seats/nodeengine): an // answer or an event that does not hold is no statement, and an event's node is its subject's last token. now := time.Now().Format(time.RFC3339Nano) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go index 7c60c464..cf2b67f4 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "errors" "maps" "mesh-delivery-assigner/internal/buckets" @@ -339,6 +340,24 @@ func presenceOf(key string, value []byte) (node string, online, ok bool) { return node, p.Online, true } +// seatHeard says a machine holding seat (a seat held once) is online as the watchman holds it: the callee of a call to +// the seat's verb is heard (novox/hq ADR 0668 §3). +func (w *world) seatHeard(seat string) bool { + online := w.online() + for _, value := range w.read(plc.SeatHoldersBucket) { + var sh mma.SeatHolders + if json.Unmarshal(value, &sh) != nil || sh.Seat != seat { + continue + } + for _, h := range sh.Holders { + if online[h.Node] { + return true + } + } + } + return false +} + // online is, by node, whether the watchman holds the node online. func (w *world) online() map[string]bool { out := map[string]bool{} From 00e84bb977f9052f669aadd304e74f8983ccceb7 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 10 Nov 2026 04:20:04 +0100 Subject: [PATCH 18/22] =?UTF-8?q?D10:=20the=20assigner=20calls=20the=20ope?= =?UTF-8?q?rator-channel=20seat's=20asks=20with=20its=20declared=20input?= =?UTF-8?q?=20and=20reads=20its=20declared=20answer=20(issue=201966,=20ADR?= =?UTF-8?q?=200668=20=C2=A74;=20review=20round=201,=20finding=205)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../internal/assign/asks_test.go | 2 +- .../internal/assign/ended.go | 18 ++++-------------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/modules/mesh-delivery-assigner/internal/assign/asks_test.go b/modules/mesh-delivery-assigner/internal/assign/asks_test.go index d550f1fa..5264609f 100644 --- a/modules/mesh-delivery-assigner/internal/assign/asks_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/asks_test.go @@ -147,7 +147,7 @@ func anAskEndedAtTheChannelIsReadAtBind(t *testing.T) { t.Error("with no ask held the channel is asked") } q, ok := ChannelAsksQuery(held) - if since, _ := q["since"].(string); !ok || q["state"] != "ended" || since != asked.Add(-time.Hour-time.Minute).Format(time.RFC3339) { + if !ok || q.State != "ended" || q.Bound != ChannelAsksBound || q.Since != asked.Add(-time.Hour-time.Minute).Format(time.RFC3339) { t.Errorf("the query is %v, want the ended asks since %s", q, asked.Add(-time.Hour-time.Minute).Format(time.RFC3339)) } at := func(key string, a asks.Ask, state string) string { diff --git a/modules/mesh-delivery-assigner/internal/assign/ended.go b/modules/mesh-delivery-assigner/internal/assign/ended.go index 8b5e4e7f..a59e1bbf 100644 --- a/modules/mesh-delivery-assigner/internal/assign/ended.go +++ b/modules/mesh-delivery-assigner/internal/assign/ended.go @@ -84,7 +84,7 @@ const ChannelAsksBound = 1 << 20 // every asking of its held asks (key to value, as the asks state holds them) that ended at the channel: the ended asks // opened since the earliest of them could have been opened, less a minute (issue 1966 note 155, ADR 0668 §4). ok is false // when no held ask reads, and then nothing is asked. -func ChannelAsksQuery(held map[string][]byte) (query map[string]any, ok bool) { +func ChannelAsksQuery(held map[string][]byte) (query oc.AsksInput, ok bool) { var since time.Time for key, value := range held { a, ask, err := DecodeAsk(key, value) @@ -102,19 +102,9 @@ func ChannelAsksQuery(held map[string][]byte) (query map[string]any, ok bool) { } } if !ok { - return nil, false + return oc.AsksInput{}, false } - return map[string]any{"state": "ended", "since": since.Add(-time.Minute).UTC().Format(time.RFC3339), "bound": ChannelAsksBound}, true -} - -// channelAsks is what the assigner reads of the verb asks' answer: each ask's key (".", the seat's state asks) -// and the state it ended with. -type channelAsks struct { - Asks []struct { - Key string `json:"key"` - State string `json:"state"` - } `json:"asks"` - Cut string `json:"cut"` + return oc.AsksInput{State: "ended", Since: since.Add(-time.Minute).UTC().Format(time.RFC3339), Bound: ChannelAsksBound}, true } // EndedAtChannel is, of the operator-channel seat's answer of its verb asks, each held ask whose asking ended there @@ -123,7 +113,7 @@ type channelAsks struct { // to it (issue 1966 note 155, ADR 0668 §4). cut is the answer's own cut, said by the caller; an answer that does not read // is the error. func EndedAtChannel(held map[string][]byte, answer []byte) (ended []tellerevents.Ended, cut string, err error) { - var got channelAsks + var got oc.AsksAnswer // the seat's declared answer: each ask's key and its state, and the cut if err := json.Unmarshal(answer, &got); err != nil { return nil, "", fmt.Errorf("assign: the operator channel's asks do not read: %w", err) } From 0f52bb80ad6ebf82409f970318102d87327891b3 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 10 Nov 2026 04:38:38 +0100 Subject: [PATCH 19/22] =?UTF-8?q?D11:=20the=20builder=20and=20the=20indexe?= =?UTF-8?q?r=20say=20a=20manifest-reads=20call=20unanswered=20past=20five?= =?UTF-8?q?=20minutes,=20its=20machine=20heard,=20as=20call-unanswered=20i?= =?UTF-8?q?n=20their=20own=20findings=20(issue=201966,=20ADR=200668=20?= =?UTF-8?q?=C2=A73;=20review=20round=202)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both write it through buildrun.ReadsCalls over enginereads Keeper.Unanswered as the assigner does, read the watchman's presence for a machine heard, arm a bound key that expires at the five minutes (no timer) and clear the finding when the node answers or its machine is away. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/mesh-delivery-builder/buildrun/go.mod | 1 + .../buildrun/readscalls.go | 135 ++++++++++++++++++ .../cmd/mesh-delivery-builder/main.go | 11 ++ .../cmd/mesh-delivery-builder/reads.go | 21 ++- .../cmd/mesh-delivery-builder/readscall.go | 63 ++++++++ .../cmd/mesh-delivery-builder/refused_test.go | 62 ++++++++ .../mesh-delivery-builder/manifest_test.go | 7 +- modules/mesh-delivery-builder/module.json | 3 +- .../cmd/mesh-delivery-indexer/main.go | 11 ++ .../cmd/mesh-delivery-indexer/main_test.go | 82 +++++++++++ .../cmd/mesh-delivery-indexer/reads.go | 21 ++- .../cmd/mesh-delivery-indexer/readscall.go | 76 ++++++++++ .../mesh-delivery-indexer/manifest_test.go | 3 +- modules/mesh-delivery-indexer/module.json | 3 +- 14 files changed, 491 insertions(+), 8 deletions(-) create mode 100644 modules/mesh-delivery-builder/buildrun/readscalls.go create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/readscall.go create mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/readscall.go diff --git a/modules/mesh-delivery-builder/buildrun/go.mod b/modules/mesh-delivery-builder/buildrun/go.mod index 712cc087..f344fa0a 100644 --- a/modules/mesh-delivery-builder/buildrun/go.mod +++ b/modules/mesh-delivery-builder/buildrun/go.mod @@ -6,6 +6,7 @@ require ( git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 diff --git a/modules/mesh-delivery-builder/buildrun/readscalls.go b/modules/mesh-delivery-builder/buildrun/readscalls.go new file mode 100644 index 00000000..9af1275b --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/readscalls.go @@ -0,0 +1,135 @@ +package buildrun + +import ( + "context" + "maps" + "slices" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads" + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" +) + +// ReadsCalls says a node's manifest-reads left unanswered by its engine while its machine is heard (online), the way the +// assigner does (novox/hq ADR 0668 §3), for a part that keeps the call through enginereads.Keeper: past +// meshconditionsclient.CallUnansweredAfter it is the finding call-unanswered in the part's own findings, cleared once the +// node answers or its machine is away, when none is written. Every step is an event (a statement changed, a node's +// presence changed, the counted nodes changed, a bound's expiry); the bound is a key of the part's own state that expires +// at its end (Arm), never a timer. +type ReadsCalls struct { + Keeper *enginereads.Keeper + Presence *PresenceSet + // Nodes are the counted nodes. + Nodes func() []string + // Source is the finding's source, ".call-unanswered"; Needs says the part's tool that steers the call. + Source, Needs string + // Raise writes the finding o in the part's findings; Clear removes the call-unanswered finding of the call id. + Raise func(ctx context.Context, o mcc.Observation) error + Clear func(ctx context.Context, id string) error + // Arm keeps a bound for node ending at ends, whose expiry steps again; nil arms none. + Arm func(ctx context.Context, node string, ends time.Time) error + Now func() time.Time + Logf func(string, ...any) + + mu sync.Mutex + said map[string]time.Time // node -> the since its finding was written for +} + +// PresenceBucket is the watchman's presence state, which the part reads to know a machine is heard. +const PresenceBucket = "mesh-delivery-watchman_presence" + +// ReadsCallBoundPrefix starts the key of a node's bound in the part's bounds state. +const ReadsCallBoundPrefix = "reads-call." + +// ReadsCallOf is the node a bound key is of. +func ReadsCallOf(key string) (string, bool) { return strings.CutPrefix(key, ReadsCallBoundPrefix) } + +func (c *ReadsCalls) now() time.Time { + if c.Now != nil { + return c.Now() + } + return time.Now() +} + +func (c *ReadsCalls) logf(f string, a ...any) { + if c.Logf != nil { + c.Logf(f, a...) + } +} + +func readsCallID(node string) string { return "manifest-reads." + node } + +// Step weighs every counted node: a node whose engine has not answered while its machine is heard is asked again once +// its last ask ended; past the bound it is the finding, before it a bound is armed; one answered or away has none. +func (c *ReadsCalls) Step(ctx context.Context) { + if c == nil || c.Keeper == nil || c.Presence == nil || c.Raise == nil || c.Clear == nil { + return + } + nodes := c.Nodes() + statements := c.Keeper.Statements() + for _, node := range nodes { + if s, ok := statements[node]; ok && !s.Answered && c.Presence.Online(node) && !c.Keeper.Asking(node) { + c.Keeper.AskNode(node, "unanswered since "+s.UnansweredSince.UTC().Format(time.RFC3339)+", its machine heard") + } + } + now := c.now() + raised := map[string]bool{} + for node, since := range c.Keeper.Unanswered(nodes, c.Presence.Online, mcc.CallUnansweredAfter, now) { + o := mcc.CallUnanswered(readsCallID(node), enginereads.Invoke, node, statements[node].Words, c.Source, since) + o.Needs = c.Needs + key := o.Key() + raised[key] = true + c.mu.Lock() + was, said := c.said[node] + c.mu.Unlock() + if said && was.Equal(since) { + continue + } + if err := c.Raise(ctx, o); err != nil { + c.logf("the finding %s is not written: %v", key, err) + continue + } + c.logf("%s on %s is unanswered since %s and its machine is heard: the finding %s", enginereads.Verb, node, since.UTC().Format(time.RFC3339), key) + c.mu.Lock() + if c.said == nil { + c.said = map[string]time.Time{} + } + c.said[node] = since + c.mu.Unlock() + } + // A node unanswered, heard, and not yet past the bound: its bound's expiry is the next step. + if c.Arm != nil { + for _, node := range nodes { + s, ok := statements[node] + if ok && !s.Answered && !s.UnansweredSince.IsZero() && c.Presence.Online(node) && !raised[mcc.Key(mcc.ScopeCall, readsCallID(node), mcc.KindCallUnanswered)] { + if err := c.Arm(ctx, node, s.UnansweredSince.Add(mcc.CallUnansweredAfter)); err != nil { + c.logf("the bound of %s's manifest-reads is not armed: %v", node, err) + } + } + } + } + // Every counted node's finding no longer raised, and every one written before, is cleared. + nodeSet := map[string]bool{} + for _, node := range nodes { + nodeSet[node] = true + } + c.mu.Lock() + for node := range c.said { + nodeSet[node] = true + } + c.mu.Unlock() + for _, node := range slices.Sorted(maps.Keys(nodeSet)) { + if raised[mcc.Key(mcc.ScopeCall, readsCallID(node), mcc.KindCallUnanswered)] { + continue + } + if err := c.Clear(ctx, readsCallID(node)); err != nil { + c.logf("the finding of %s's manifest-reads is not cleared: %v", node, err) + continue + } + c.mu.Lock() + delete(c.said, node) + c.mu.Unlock() + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 71e15570..4b27a7a2 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -294,6 +294,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro reads.stepWindow() }) gate.OnPlaced = reads.placed + // A kept manifest-reads call unanswered past five minutes, its node's machine heard (the watchman's presence), is + // call-unanswered in the builder's findings (issue 1966, ADR 0668 §3). + presence := &buildrun.PresenceSet{} + reads.callsOf(found, pubBoundsKV.KV, presence, logger.Printf) + watchPresence(ctx, js, presence, reads.stepCalls, errs, logger.Printf) b.Forms = reads.built // Until the assignments's first view is answered the gate passes every report on, as before the assignments is settled. assignmentsErrs := make(chan error, 1) @@ -407,6 +412,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro unput.expired(ctx, e) return } + if _, ok := buildrun.ReadsCallOf(e.Key()); ok { + if e.Operation() == jetstream.KeyValuePurge { // its five minutes ended + reads.stepCalls() + } + return + } if versions.IsOldFormBound(e.Key()) { if e.Operation() == jetstream.KeyValuePurge { reads.stepWindow() // a node's seven days ended: the step raises old-form-held now diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reads.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reads.go index b014f154..f9770a2f 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reads.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reads.go @@ -34,12 +34,17 @@ type engineReads struct { // window is the old reference form's window over these statements (DLV-B9); nil keeps none. window *oldForm ctx context.Context + // calls is the wait of the kept calls (callsOf); nil weighs none. + calls *buildrun.ReadsCalls } // newEngineReads asks through call (process.Conn.CallKeptOn, a kept call; ADR 0668) and keeps each answer in the state once it is bound. func newEngineReads(call enginereads.Caller, gate *buildrun.ToolchainGate, logf func(string, ...any)) *engineReads { r := &engineReads{gate: gate, keeper: &enginereads.Keeper{Call: call, Logf: logf}} - r.keeper.Changed = r.stepWindow + r.keeper.Changed = func() { + r.stepWindow() + r.stepCalls() + } return r } @@ -57,6 +62,19 @@ func (r *engineReads) stepWindow() { r.window.step(ctx, r.named(), counted) } +// stepCalls weighs the kept manifest-reads calls (buildrun.ReadsCalls): after a statement changed, a node's presence, the +// assignments's change and the end of a call's bound. +func (r *engineReads) stepCalls() { + if r.calls == nil { + return + } + ctx := r.ctx + if ctx == nil { + ctx = context.Background() + } + r.calls.Step(ctx) +} + // named are the nodes the assignments names, none dropped: what the window's step judges a read by. func (r *engineReads) named() []string { r.mu.Lock() @@ -106,6 +124,7 @@ func (r *engineReads) placed() { } } r.stepWindow() + r.stepCalls() } // atBind asks every counted node again when the builder's consumer is bound (the hub's OnBound): a manifest-reads-changed diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/readscall.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/readscall.go new file mode 100644 index 00000000..399b152c --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/readscall.go @@ -0,0 +1,63 @@ +package main + +import ( + "context" + "errors" + "time" + + "github.com/nats-io/nats.go/jetstream" + + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + "git.novox.be/novox/mesh-sdk/go/process" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" + "mesh-delivery-builder/internal/versions" +) + +// callsOf is the wait of the builder's kept manifest-reads calls (buildrun.ReadsCalls): a node whose engine has not +// answered while its machine is heard is, past five minutes, the finding call-unanswered in the builder's findings +// (issue 1966, ADR 0668 §3); its bound is a key of the state bounds with its own time to live. +func (r *engineReads) callsOf(f *findings, bounds func() jetstream.KeyValue, presence *buildrun.PresenceSet, logf func(string, ...any)) *buildrun.ReadsCalls { + c := &buildrun.ReadsCalls{Keeper: r.keeper, Presence: presence, Nodes: r.named, Logf: logf, + Source: "builder." + mcc.KindCallUnanswered, Needs: "builder_engine_reads with ask_again asks the node again now and shows its statement.", + Raise: func(ctx context.Context, o mcc.Observation) error { f.raiseObservation(ctx, o); return nil }, + Clear: func(ctx context.Context, id string) error { + f.clear(ctx, mcc.KindCallUnanswered, mcc.ScopeCall, id) + return nil + }, + Arm: func(ctx context.Context, node string, ends time.Time) error { + kv := bounds() + if kv == nil { + return errors.New(versions.BoundsBucket + " is not on the bus yet") + } + left := max(time.Until(ends), time.Second) + _, err := kv.Create(ctx, buildrun.ReadsCallBoundPrefix+node, versions.EncodeBound("the manifest-reads call to "+node, ends), jetstream.KeyTTL(left)) + if errors.Is(err, jetstream.ErrKeyExists) { + err = nil + } + return err + }} + r.calls = c + return c +} + +// watchPresence follows the watchman's presence, which says a machine is heard: a node coming online or going weighs the +// calls again. +func watchPresence(ctx context.Context, js jetstream.JetStream, presence *buildrun.PresenceSet, step func(), errs chan<- error, logf func(string, ...any)) { + const without = "no node counts as heard, so a manifest-reads call left unanswered is not raised until the builder starts again" + kv := process.OptionalBucket(ctx, js, buildrun.PresenceBucket, without, logf) + if kv == nil { + return + } + each := func(e jetstream.KeyValueEntry) { + if presence.Put(e.Key(), e.Value(), !opOf(e)) { + step() + } + } + w, err := settle(ctx, kv, each) + if err != nil { + process.NotWatched(logf, buildrun.PresenceBucket, err, without) + return + } + go func() { errs <- follow(ctx, w, each) }() +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go index 9d1b8050..c76687e2 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go @@ -24,6 +24,7 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads" "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs" sweep "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshartifactsweep" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" @@ -137,6 +138,67 @@ func theCountedNodesEngineReads(t *testing.T) { if _, err := tool.Run(map[string]any{"node": "n1", "drop": "true", "why": "a test"}); err != nil || st.v["n1"] != nil { t.Errorf("drop: %v, the key %s", err, st.v["n1"]) } + t.Run("ADR 0668 §3: a manifest-reads call unanswered past five minutes, its machine heard, is call-unanswered", func(t *testing.T) { + // n2 answers nothing: while its machine is away no finding; heard and still unanswered five minutes on, the finding + // call-unanswered in the builder's findings and a bound armed before; answered, or away again, it is cleared. + place("n1", "n2") + now := time.Unix(1000, 0) + clock := func() time.Time { return now } + down := true + r2 := newEngineReads(func(node, _ string, _ any) (json.RawMessage, error) { + if down { + return nil, nats.ErrNoResponders + } + return json.Marshal(nodeengine.ManifestReadsAnswer{V: 1, Node: node, ManifestReads: []string{"form:new"}, StatedAt: now.Format(time.RFC3339Nano)}) + }, gate, t.Logf) + r2.keeper.Sync, r2.keeper.Now = true, clock + st := &memFindings{vals: map[string][]byte{}} + armed := map[string]time.Time{} + presence := &buildrun.PresenceSet{} + calls := r2.callsOf(&findings{store: st, logf: t.Logf, now: clock}, func() jetstream.KeyValue { return nil }, presence, t.Logf) + calls.Now = clock + calls.Arm = func(_ context.Context, node string, ends time.Time) error { armed[node] = ends; return nil } + key := meshconditionsclient.Key(meshconditionsclient.ScopeCall, "manifest-reads.n2", meshconditionsclient.KindCallUnanswered) + r2.placed() // n2 asked, unanswered at 1000 + now = now.Add(6 * time.Minute) + r2.stepCalls() + if len(st.vals) != 0 || len(armed) != 0 { + t.Fatalf("n2's machine is away: findings %v, bounds %v", st.vals, armed) + } + presence.Put("n2", presenceOnline(t, true), false) + now = time.Unix(1000, 0).Add(2 * time.Minute) + r2.stepCalls() + if len(st.vals) != 0 || !armed["n2"].Equal(time.Unix(1000, 0).Add(meshconditionsclient.CallUnansweredAfter)) { + t.Fatalf("heard before five minutes: findings %v, bounds %v", st.vals, armed) + } + now = time.Unix(1000, 0).Add(meshconditionsclient.CallUnansweredAfter) + r2.stepCalls() + f, err := meshconditionsclient.DecodeFinding(st.vals[key]) + if err != nil || f.Observer != builds.BuilderModule || f.Kind != meshconditionsclient.KindCallUnanswered { + t.Fatalf("heard five minutes on: finding %+v, %v (keys %v)", f, err, slices.Collect(maps.Keys(st.vals))) + } + presence.Put("n2", presenceOnline(t, false), false) + r2.stepCalls() + if len(st.vals) != 0 { + t.Errorf("its machine away again leaves %v", st.vals) + } + presence.Put("n2", presenceOnline(t, true), false) + r2.stepCalls() + down = false + r2.keeper.AskNode("n2", "answered now") + if len(st.vals) != 0 { + t.Errorf("answered leaves %v", st.vals) + } + }) +} + +func presenceOnline(t *testing.T, on bool) []byte { + t.Helper() + b, err := json.Marshal(meshwatchdogs.PresenceByNodeValue{Online: on}) + if err != nil { + t.Fatal(err) + } + return b } func mustStatement(t *testing.T, s enginereads.Statement) []byte { diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 6e883afc..6af6a26e 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -100,10 +100,11 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "queues", History: 1}, {Name: "reasks", History: 1}, {Name: "tag-ask-bounds", History: 1, KeyTTL: true}, {Name: "tag-asks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory t.Errorf("state = %+v, want %+v", m.State, want) } - // 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and - // holds no capacity; it puts asks on the three build seats and reads their steps (ADR 0525 §1, §4, §5). + // 1887/A4c: the builder chooses no node, so it reads no have, invokes no build module's tool and holds no capacity; it + // puts asks on the three build seats and reads their steps (ADR 0525 §1, §4, §5). It reads the watchman's presence + // only to know a node's machine is heard, for a manifest-reads call left unanswered (issue 1966, ADR 0668 §3). if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains", - "mesh-delivery-assigner.assignments", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) { + "mesh-delivery-assigner.assignments", "mesh-delivery-gate.source", "mesh-delivery-watchman.presence"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } if want := []string{"node-build-go", "node-build-image", "node-build-node"}; !reflect.DeepEqual(sorted(m.Uses), want) { diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index 2f4140d0..a5fc4f0c 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -107,7 +107,8 @@ "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-delivery-assigner.assignments", - "mesh-delivery-gate.source" + "mesh-delivery-gate.source", + "mesh-delivery-watchman.presence" ], "uses": [ "node-build-go", diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index d973cc3f..409492b9 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -506,8 +506,19 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // weighing only, said. holdBounds := process.AwaitBucket(ctx, js, conn, index.HoldBoundsBucket, "a hold standing past its bound is raised at its next weighing only", logger.Printf) ix.HoldBounds = expiringKeys{holdBounds} + // A kept manifest-reads call unanswered past five minutes, its node's machine heard (the watchman's presence), is + // call-unanswered in the indexer's findings (issue 1966, ADR 0668 §3). + presence := &buildrun.PresenceSet{} + reads.callsOf(ctx, findings.KV, ix.HoldBounds, presence, logger.Printf) + watchPresence(ctx, js, presence, reads.stepCalls, errs, logger.Printf) holdBounds.Ready(func(kv jetstream.KeyValue) { each := func(e jetstream.KeyValueEntry) { + if _, ok := buildrun.ReadsCallOf(e.Key()); ok { + if e.Operation() == jetstream.KeyValuePurge { // a manifest-reads call's five minutes ended + reads.stepCalls() + } + return + } if e.Operation() == jetstream.KeyValuePurge { // its time to live ended ix.HoldBoundEnded(ctx, e.Key()) } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go index 1bb57db3..a4b612c0 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go @@ -13,8 +13,12 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads" + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/process" @@ -94,6 +98,61 @@ func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part) t.Errorf("at the bind called %v, want ace and g14 once each", called) } }) + t.Run("a manifest-reads call unanswered past five minutes, its machine heard, is call-unanswered (ADR 0668 §3)", func(t *testing.T) { + // While the node's machine is away no finding is written; heard, a bound is armed and, five minutes on, the finding + // is written in the indexer's findings; away again or answered, it is cleared. + now := time.Unix(1000, 0) + clock := func() time.Time { return now } + down := true + held := &findingsKV{vals: map[string][]byte{}} + reads := &engineReads{known: true, nodes: []string{"ace"}} + reads.keeper = &enginereads.Keeper{Sync: true, Now: clock, Call: func(node, _ string, _ any) (json.RawMessage, error) { + if down { + return nil, errors.New("no responders") + } + return json.Marshal(nodeengine.ManifestReadsAnswer{V: 1, Node: node, ManifestReads: []string{"form:new"}, StatedAt: now.Format(time.RFC3339Nano)}) + }} + reads.keeper.Changed = reads.stepCalls + armed := map[string]time.Time{} + presence := &buildrun.PresenceSet{} + calls := reads.callsOf(context.Background(), func() jetstream.KeyValue { return held }, nil, presence, t.Logf) + calls.Now = clock + calls.Arm = func(_ context.Context, node string, ends time.Time) error { armed[node] = ends; return nil } + online := func(on bool) { + b, _ := json.Marshal(meshwatchdogs.PresenceByNodeValue{Online: on}) + presence.Put("ace", b, false) + } + key := mcc.Key(mcc.ScopeCall, "manifest-reads.ace", mcc.KindCallUnanswered) + reads.keeper.AskNode("ace", "counted") // unanswered at 1000 + now = now.Add(6 * time.Minute) + reads.stepCalls() + if len(held.vals) != 0 || len(armed) != 0 { + t.Fatalf("its machine is away: findings %v, bounds %v", held.vals, armed) + } + online(true) + now = time.Unix(1000, 0).Add(time.Minute) + reads.stepCalls() + if len(held.vals) != 0 || !armed["ace"].Equal(time.Unix(1000, 0).Add(mcc.CallUnansweredAfter)) { + t.Fatalf("heard before five minutes: findings %v, bounds %v", held.vals, armed) + } + now = time.Unix(1000, 0).Add(mcc.CallUnansweredAfter) + reads.stepCalls() + if f, err := mcc.DecodeFinding(held.vals[key]); err != nil || f.Observer != "mesh-delivery-indexer" || f.Kind != mcc.KindCallUnanswered { + t.Fatalf("heard five minutes on: finding %+v, %v", f, err) + } + online(false) + reads.stepCalls() + if len(held.vals) != 0 { + t.Errorf("its machine away again leaves %v", held.vals) + } + online(true) + reads.stepCalls() + down = false + reads.keeper.AskNode("ace", "answered") + if len(held.vals) != 0 || !reads.keeper.Statements()["ace"].Answered { + t.Errorf("answered leaves %v", held.vals) + } + }) t.Run("not a forge message: terminated and logged", func(t *testing.T) { // 1238 (never redelivered) var logs bytes.Buffer st := &memStore{vals: map[string][]byte{}} @@ -211,3 +270,26 @@ func TestTheSourceMainKeyIsTakenAtItsRevision(t *testing.T) { // 1344/AC1 (the w } func ptr[T any](v T) *T { return &v } + +// findingsKV is a findings bucket of the two calls the indexer's call finding makes. +type findingsKV struct { + jetstream.KeyValue + vals map[string][]byte +} + +func (f *findingsKV) Put(_ context.Context, key string, value []byte) (uint64, error) { + f.vals[key] = value + return uint64(len(f.vals)), nil +} + +func (f *findingsKV) Get(_ context.Context, key string) (jetstream.KeyValueEntry, error) { + if _, ok := f.vals[key]; !ok { + return nil, jetstream.ErrKeyNotFound + } + return nil, nil +} + +func (f *findingsKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error { + delete(f.vals, key) + return nil +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/reads.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/reads.go index 890a8f10..89b7c249 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/reads.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/reads.go @@ -28,11 +28,29 @@ type engineReads struct { mu sync.Mutex known bool // the assignments is read nodes []string + // calls is the wait of the kept calls (callsOf); nil weighs none. ctx is the indexer's. + calls *buildrun.ReadsCalls + ctx context.Context } // newEngineReads asks through call (process.Conn.CallKeptOn, a kept call; ADR 0668) and tells changed after each statement changed. func newEngineReads(call enginereads.Caller, gate *buildrun.ToolchainGate, changed func(), logf func(string, ...any)) *engineReads { - return &engineReads{gate: gate, keeper: &enginereads.Keeper{Call: call, Logf: logf, Changed: changed}} + r := &engineReads{gate: gate, ctx: context.Background()} + r.keeper = &enginereads.Keeper{Call: call, Logf: logf, Changed: func() { + if changed != nil { + changed() + } + r.stepCalls() + }} + return r +} + +// stepCalls weighs the kept manifest-reads calls (buildrun.ReadsCalls): after a statement changed, a node's presence, the +// assignments's change and the end of a call's bound. +func (r *engineReads) stepCalls() { + if r.calls != nil { + r.calls.Step(r.ctx) + } } // bind keeps the statements in kv and hears it: every node's last statement at the start, a person's ask again and drop @@ -76,6 +94,7 @@ func (r *engineReads) placed() { if r.keeper.Changed != nil { r.keeper.Changed() } + r.stepCalls() } // atBind asks every counted node again when the indexer's consumer is bound (the hub's OnBound): a manifest-reads-changed diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/readscall.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/readscall.go new file mode 100644 index 00000000..2246fcf6 --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/readscall.go @@ -0,0 +1,76 @@ +package main + +import ( + "context" + "errors" + "slices" + "time" + + "github.com/nats-io/nats.go/jetstream" + + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" + "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-indexer/internal/index" +) + +// callsOf is the wait of the indexer's kept manifest-reads calls (buildrun.ReadsCalls): a node whose engine has not +// answered while its machine is heard is, past five minutes, the finding call-unanswered in the indexer's findings +// (issue 1966, ADR 0668 §3); its bound is a key of the state hold-bounds, which expires at its end. +func (r *engineReads) callsOf(ctx context.Context, findings func() jetstream.KeyValue, bounds index.Bounds, presence *buildrun.PresenceSet, logf func(string, ...any)) *buildrun.ReadsCalls { + c := &buildrun.ReadsCalls{Keeper: r.keeper, Presence: presence, Logf: logf, + Nodes: func() []string { r.mu.Lock(); defer r.mu.Unlock(); return slices.Clone(r.nodes) }, + Source: "indexer." + mcc.KindCallUnanswered, Needs: index.EngineReadsTool + " with ask_again asks the node again now and shows its statement.", + Raise: func(ctx context.Context, o mcc.Observation) error { + kv := findings() + if kv == nil { + return process.ErrNotYet + } + value, err := mcc.EncodeFinding(o.Finding(index.Module, "", time.Now().UTC())) + if err == nil { + _, err = kv.Put(ctx, o.Key(), value) + } + return err + }, + Clear: func(ctx context.Context, id string) error { + kv := findings() + if kv == nil { + return nil + } + key := mcc.Key(mcc.ScopeCall, id, mcc.KindCallUnanswered) + if _, err := kv.Get(ctx, key); err != nil { + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil + } + return err + } + return kv.Delete(ctx, key) + }, + Arm: func(ctx context.Context, node string, ends time.Time) error { + return bounds.Arm(ctx, buildrun.ReadsCallBoundPrefix+node, ends) + }} + r.ctx, r.calls = ctx, c + return c +} + +// watchPresence follows the watchman's presence, which says a machine is heard: a node coming online or going weighs the +// calls again. +func watchPresence(ctx context.Context, js jetstream.JetStream, presence *buildrun.PresenceSet, step func(), errs chan<- error, logf func(string, ...any)) { + const without = "no node counts as heard, so a manifest-reads call left unanswered is not raised until the indexer starts again" + kv := process.OptionalBucket(ctx, js, buildrun.PresenceBucket, without, logf) + if kv == nil { + return + } + each := func(e jetstream.KeyValueEntry) { + if presence.Put(e.Key(), e.Value(), !opOf(e)) { + step() + } + } + w, err := settle(ctx, kv, each) + if err != nil { + process.NotWatched(logf, buildrun.PresenceBucket, err, without) + return + } + go func() { errs <- follow(ctx, w, each) }() +} diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index c5fa5e79..86c933e3 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -137,7 +137,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build } } // DLV-F1r (ADR 0615 §1): the assigner's assignments is read through its first view and views family, never its state. - if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-delivery-builder.build-log", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) { + // The watchman's presence says a node's machine is heard, for a manifest-reads call left unanswered (issue 1966, ADR 0668 §3). + if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-delivery-builder.build-log", "mesh-delivery-checker.previews", "mesh-delivery-gate.source", "mesh-delivery-watchman.presence"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } // "Reads FORGE" is a consumes of the indexer's stream: the broker grants only buckets by reads (mesh-control diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 41198549..9344bd46 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -119,7 +119,8 @@ "mesh-build-node.toolchains", "mesh-delivery-builder.build-log", "mesh-delivery-checker.previews", - "mesh-delivery-gate.source" + "mesh-delivery-gate.source", + "mesh-delivery-watchman.presence" ], "invokes": [ "mesh-delivery-assigner.assignments", From 8e426b236da5abedb45da8e18923d85b0abdad51 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 10 Nov 2026 04:55:31 +0100 Subject: [PATCH 20/22] =?UTF-8?q?D12:=20the=20watchman=20serves=20the=20wa?= =?UTF-8?q?tchdogs=20seat's=20presence=20with=20its=20declared=20machine,?= =?UTF-8?q?=20answering=20that=20machine's=20online=20and=20since=20in=20t?= =?UTF-8?q?he=20seat's=20answer=20(issue=201966,=20ADR=200596=20=C2=A72;?= =?UTF-8?q?=20review=20round=203,=20finding=201)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The builder and the indexer ask it whether a node's machine is heard instead of reading the watchman's state; key and prefix stay for the whole read. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-watchman-tools/main.go | 40 ++++++++++++++++++- .../mesh-delivery-watchman-tools/main_test.go | 33 ++++++++++++++- 2 files changed, 69 insertions(+), 4 deletions(-) diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman-tools/main.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman-tools/main.go index 0ce55f26..fa5a0672 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman-tools/main.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman-tools/main.go @@ -13,6 +13,7 @@ import ( "time" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/views" @@ -105,6 +106,35 @@ var keyArgs = map[string]any{"type": "object", "properties": map[string]any{ "key": map[string]any{"type": "string", "description": "one key only"}, "prefix": map[string]any{"type": "string", "description": "only the keys that begin with it"}}} +// presenceArgs are the seat's input of presence, machine, beside the watchman's own key and prefix. +var presenceArgs = map[string]any{"type": "object", "properties": map[string]any{ + "machine": map[string]any{"type": "string", "description": "one machine: whether the watchman hears it and since when"}, + "key": map[string]any{"type": "string", "description": "one key only"}, + "prefix": map[string]any{"type": "string", "description": "only the keys that begin with it"}}} + +// presenceOf answers the seat's presence of one machine (meshwatchdogs.PresenceAnswer): online and since as the state +// presence holds it under the machine's key; a machine it holds no word of is not online, with no since (novox/hq ADR +// 0596 §2: a part asks this verb, never reads the state). +func presenceOf(st state, machine string) (any, error) { + e, err := st.Get(meshwatchdogs.PresenceByNodeKey(machine)) + if err != nil { + return nil, fmt.Errorf("presence %s cannot be read: %w", machine, err) + } + answer := meshwatchdogs.PresenceAnswer{Machine: machine} + if e == nil { + return answer, nil + } + v, err := meshwatchdogs.DecodePresenceByNodeValue(e.Value) + if err != nil { + return nil, fmt.Errorf("presence %s does not read: %w", machine, err) + } + answer.Online = v.Online + if !e.At.IsZero() { + answer.Since = e.At.UTC().Format(time.RFC3339Nano) + } + return answer, nil +} + // Tools are the seat's verbs and the module's tools, over the states states gives and the program asked through ask. func Tools(states func(string) state, ask func(socket.Ask) (json.RawMessage, error)) []stdio.Tool { read := func(name string) func(map[string]any) (any, error) { @@ -141,9 +171,15 @@ func Tools(states func(string) state, ask func(socket.Ask) (json.RawMessage, err Description: "What stopped moving: every key of the watchman's stuck bucket, . with what it waited for since " + "when, and whether it moved since (never deleted). A key not moved is also the watchman's finding " + "delivery..stuck, a node's presence machine..silent. Give key for one, prefix to narrow. Read-only."}, - {Name: verb(VerbPresence), Input: keyArgs, Run: read("presence"), + {Name: verb(VerbPresence), Input: presenceArgs, Run: func(args map[string]any) (any, error) { + if machine := str(args, "machine"); machine != "" { + return presenceOf(states("presence"), machine) + } + return read("presence")(args) + }, Description: "Who is online on the bus as the watchman hears it: each node's agent () and each delivery part's instance " + - "(part..), online or not since the entry's time. Give key for one, prefix to narrow. Read-only."}, + "(part..), online or not since the entry's time. Give machine for one machine's online and since, " + + "key for one key, prefix to narrow. Read-only."}, {Name: verb(VerbDeadLetters), Run: dead, Description: "What a consumer gave up on, kept in DEAD_LETTERS (novox/hq ADR 0564 §5, ADR 0264, ADR 0376): newest first, " + "for one consumer (its name or .), at most limit; one whole by id (a restricted one's body " + diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman-tools/main_test.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman-tools/main_test.go index 97a92628..d6100784 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman-tools/main_test.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman-tools/main_test.go @@ -4,9 +4,12 @@ import ( "encoding/json" "errors" "os" + "reflect" "slices" "testing" + "time" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs" stdio "git.novox.be/novox/mesh-sdk/go" "mesh-delivery-watchman/internal/socket" @@ -27,7 +30,7 @@ func (f fakeState) Get(key string) (*stdio.StateEntry, error) { if !ok { return nil, nil } - return &stdio.StateEntry{Key: key, Value: json.RawMessage(v), Revision: 3}, nil + return &stdio.StateEntry{Key: key, Value: json.RawMessage(v), Revision: 3, At: time.Date(2026, 11, 10, 3, 0, 0, 0, time.UTC)}, nil } // 1952/WM1, ADR 0564 §12: every state the manifest keeps is read by a tool, every act the program starts by hand is @@ -51,7 +54,8 @@ func TestEveryStateAndActHasATool(t *testing.T) { t.Fatal(err) } asked := []socket.Ask{} - states := map[string]fakeState{"findings": {"machine.ace.silent": `{"v":1}`, "core.doctor.self-check-silent": `{"v":1}`}} + states := map[string]fakeState{"findings": {"machine.ace.silent": `{"v":1}`, "core.doctor.self-check-silent": `{"v":1}`}, + "presence": {"ace": `{"v":1,"online":true}`, "g14": `{"v":1,"online":false}`, "part.mesh-delivery-builder.ace": `{"v":1,"online":true}`}} tools := Tools(func(name string) state { return states[name] }, func(a socket.Ask) (json.RawMessage, error) { asked = append(asked, a) if a.Act != "" { @@ -126,6 +130,31 @@ func TestEveryStateAndActHasATool(t *testing.T) { t.Fatal("a key not held is answered") } }) + // Issue 1966, ADR 0596 §2 (review round 3): presence with the seat's machine answers that machine's online and since in + // the seat's declared answer; one the watchman holds no word of is not online; without it, the keys as before. + t.Run("presence answers one machine as the seat declares", func(t *testing.T) { + for machine, want := range map[string]meshwatchdogs.PresenceAnswer{ + "ace": {Machine: "ace", Online: true, Since: "2026-11-10T03:00:00Z"}, + "g14": {Machine: "g14", Since: "2026-11-10T03:00:00Z"}, + "novox": {Machine: "novox"}, + } { + out, err := served[Seat+"."+VerbPresence].Run(map[string]any{"machine": machine}) + if err != nil { + t.Fatal(err) + } + b, _ := json.Marshal(out) + var got meshwatchdogs.PresenceAnswer + if err := json.Unmarshal(b, &got); err != nil || !reflect.DeepEqual(got, want) { + t.Errorf("presence of %s = %s (%v), want %+v", machine, b, err, want) + } + } + out, _ := served[Seat+"."+VerbPresence].Run(map[string]any{"prefix": "part."}) + b, _ := json.Marshal(out) + var whole meshwatchdogs.PresenceAnswer + if err := json.Unmarshal(b, &whole); err != nil || whole.Holds != 3 || len(whole.Keys) != 1 || whole.State != "presence" { + t.Errorf("presence whole = %s (%v)", b, err) + } + }) t.Run("dead-letters and resync ask the program", func(t *testing.T) { if _, err := served[Seat+"."+VerbDeadLetters].Run(map[string]any{"consumer": "ace_web", "limit": float64(3), "id": float64(7)}); err != nil { t.Fatal(err) From 98ffc1fb80467f8c44ad127d41b8f5e86020f23f Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 10 Nov 2026 05:01:15 +0100 Subject: [PATCH 21/22] =?UTF-8?q?D13:=20the=20builder=20and=20the=20indexe?= =?UTF-8?q?r=20ask=20the=20watchdogs=20seat's=20presence=20when=20a=20mani?= =?UTF-8?q?fest-reads=20call's=20bound=20ends,=20through=20the=20node-engi?= =?UTF-8?q?ne=20seat's=20client's=20ReadsCalls,=20and=20read=20no=20watchm?= =?UTF-8?q?an=20state=20(issue=201966,=20ADR=200596=20=C2=A72,=20ADR=20066?= =?UTF-8?q?8=20=C2=A73,=20ADR=200634=20=C2=A71;=20review=20round=203,=20fi?= =?UTF-8?q?ndings=201=20and=202)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both manifests drop the read mesh-delivery-watchman.presence and invoke seat:mesh-watchdogs.presence; buildrun's ReadsCalls, its presence bucket and both watchPresence copies go. The machine is asked once when the bound key expires; a node that returns is asked again on manifest-reads-changed, and the finding clears on its answer. The indexer's finding is stamped with the wait's clock. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/mesh-delivery-builder/buildrun/go.mod | 1 - .../buildrun/readscalls.go | 135 ------------------ .../cmd/mesh-delivery-builder/main.go | 13 +- .../cmd/mesh-delivery-builder/reads.go | 27 ++-- .../cmd/mesh-delivery-builder/readscall.go | 75 +++++----- .../cmd/mesh-delivery-builder/refused_test.go | 55 +++---- .../mesh-delivery-builder/manifest_test.go | 11 +- modules/mesh-delivery-builder/module.json | 4 +- .../cmd/mesh-delivery-indexer/main.go | 12 +- .../cmd/mesh-delivery-indexer/main_test.go | 51 +++---- .../cmd/mesh-delivery-indexer/reads.go | 13 +- .../cmd/mesh-delivery-indexer/readscall.go | 78 +++++----- modules/mesh-delivery-indexer/go.mod | 2 +- .../mesh-delivery-indexer/manifest_test.go | 5 +- modules/mesh-delivery-indexer/module.json | 6 +- 15 files changed, 179 insertions(+), 309 deletions(-) delete mode 100644 modules/mesh-delivery-builder/buildrun/readscalls.go diff --git a/modules/mesh-delivery-builder/buildrun/go.mod b/modules/mesh-delivery-builder/buildrun/go.mod index f344fa0a..712cc087 100644 --- a/modules/mesh-delivery-builder/buildrun/go.mod +++ b/modules/mesh-delivery-builder/buildrun/go.mod @@ -6,7 +6,6 @@ require ( git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000 - git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 diff --git a/modules/mesh-delivery-builder/buildrun/readscalls.go b/modules/mesh-delivery-builder/buildrun/readscalls.go deleted file mode 100644 index 9af1275b..00000000 --- a/modules/mesh-delivery-builder/buildrun/readscalls.go +++ /dev/null @@ -1,135 +0,0 @@ -package buildrun - -import ( - "context" - "maps" - "slices" - "strings" - "sync" - "time" - - "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads" - mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" -) - -// ReadsCalls says a node's manifest-reads left unanswered by its engine while its machine is heard (online), the way the -// assigner does (novox/hq ADR 0668 §3), for a part that keeps the call through enginereads.Keeper: past -// meshconditionsclient.CallUnansweredAfter it is the finding call-unanswered in the part's own findings, cleared once the -// node answers or its machine is away, when none is written. Every step is an event (a statement changed, a node's -// presence changed, the counted nodes changed, a bound's expiry); the bound is a key of the part's own state that expires -// at its end (Arm), never a timer. -type ReadsCalls struct { - Keeper *enginereads.Keeper - Presence *PresenceSet - // Nodes are the counted nodes. - Nodes func() []string - // Source is the finding's source, ".call-unanswered"; Needs says the part's tool that steers the call. - Source, Needs string - // Raise writes the finding o in the part's findings; Clear removes the call-unanswered finding of the call id. - Raise func(ctx context.Context, o mcc.Observation) error - Clear func(ctx context.Context, id string) error - // Arm keeps a bound for node ending at ends, whose expiry steps again; nil arms none. - Arm func(ctx context.Context, node string, ends time.Time) error - Now func() time.Time - Logf func(string, ...any) - - mu sync.Mutex - said map[string]time.Time // node -> the since its finding was written for -} - -// PresenceBucket is the watchman's presence state, which the part reads to know a machine is heard. -const PresenceBucket = "mesh-delivery-watchman_presence" - -// ReadsCallBoundPrefix starts the key of a node's bound in the part's bounds state. -const ReadsCallBoundPrefix = "reads-call." - -// ReadsCallOf is the node a bound key is of. -func ReadsCallOf(key string) (string, bool) { return strings.CutPrefix(key, ReadsCallBoundPrefix) } - -func (c *ReadsCalls) now() time.Time { - if c.Now != nil { - return c.Now() - } - return time.Now() -} - -func (c *ReadsCalls) logf(f string, a ...any) { - if c.Logf != nil { - c.Logf(f, a...) - } -} - -func readsCallID(node string) string { return "manifest-reads." + node } - -// Step weighs every counted node: a node whose engine has not answered while its machine is heard is asked again once -// its last ask ended; past the bound it is the finding, before it a bound is armed; one answered or away has none. -func (c *ReadsCalls) Step(ctx context.Context) { - if c == nil || c.Keeper == nil || c.Presence == nil || c.Raise == nil || c.Clear == nil { - return - } - nodes := c.Nodes() - statements := c.Keeper.Statements() - for _, node := range nodes { - if s, ok := statements[node]; ok && !s.Answered && c.Presence.Online(node) && !c.Keeper.Asking(node) { - c.Keeper.AskNode(node, "unanswered since "+s.UnansweredSince.UTC().Format(time.RFC3339)+", its machine heard") - } - } - now := c.now() - raised := map[string]bool{} - for node, since := range c.Keeper.Unanswered(nodes, c.Presence.Online, mcc.CallUnansweredAfter, now) { - o := mcc.CallUnanswered(readsCallID(node), enginereads.Invoke, node, statements[node].Words, c.Source, since) - o.Needs = c.Needs - key := o.Key() - raised[key] = true - c.mu.Lock() - was, said := c.said[node] - c.mu.Unlock() - if said && was.Equal(since) { - continue - } - if err := c.Raise(ctx, o); err != nil { - c.logf("the finding %s is not written: %v", key, err) - continue - } - c.logf("%s on %s is unanswered since %s and its machine is heard: the finding %s", enginereads.Verb, node, since.UTC().Format(time.RFC3339), key) - c.mu.Lock() - if c.said == nil { - c.said = map[string]time.Time{} - } - c.said[node] = since - c.mu.Unlock() - } - // A node unanswered, heard, and not yet past the bound: its bound's expiry is the next step. - if c.Arm != nil { - for _, node := range nodes { - s, ok := statements[node] - if ok && !s.Answered && !s.UnansweredSince.IsZero() && c.Presence.Online(node) && !raised[mcc.Key(mcc.ScopeCall, readsCallID(node), mcc.KindCallUnanswered)] { - if err := c.Arm(ctx, node, s.UnansweredSince.Add(mcc.CallUnansweredAfter)); err != nil { - c.logf("the bound of %s's manifest-reads is not armed: %v", node, err) - } - } - } - } - // Every counted node's finding no longer raised, and every one written before, is cleared. - nodeSet := map[string]bool{} - for _, node := range nodes { - nodeSet[node] = true - } - c.mu.Lock() - for node := range c.said { - nodeSet[node] = true - } - c.mu.Unlock() - for _, node := range slices.Sorted(maps.Keys(nodeSet)) { - if raised[mcc.Key(mcc.ScopeCall, readsCallID(node), mcc.KindCallUnanswered)] { - continue - } - if err := c.Clear(ctx, readsCallID(node)); err != nil { - c.logf("the finding of %s's manifest-reads is not cleared: %v", node, err) - continue - } - c.mu.Lock() - delete(c.said, node) - c.mu.Unlock() - } -} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 4b27a7a2..5d40a676 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -22,6 +22,7 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" sdkfindings "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient/findings" + "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds/forms" @@ -294,11 +295,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro reads.stepWindow() }) gate.OnPlaced = reads.placed - // A kept manifest-reads call unanswered past five minutes, its node's machine heard (the watchman's presence), is - // call-unanswered in the builder's findings (issue 1966, ADR 0668 §3). - presence := &buildrun.PresenceSet{} - reads.callsOf(found, pubBoundsKV.KV, presence, logger.Printf) - watchPresence(ctx, js, presence, reads.stepCalls, errs, logger.Printf) + // A kept manifest-reads call unanswered past five minutes, its node's machine heard (the watchdogs seat's presence, + // asked when the bound ends), is call-unanswered in the builder's findings (issue 1966, ADR 0668 §3, ADR 0596 §2). + reads.callsOf(found, pubBoundsKV.KV, heardBy(calls.CallKept), logger.Printf) b.Forms = reads.built // Until the assignments's first view is answered the gate passes every report on, as before the assignments is settled. assignmentsErrs := make(chan error, 1) @@ -412,9 +411,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro unput.expired(ctx, e) return } - if _, ok := buildrun.ReadsCallOf(e.Key()); ok { + if node, ok := enginereads.BoundOf(e.Key()); ok { if e.Operation() == jetstream.KeyValuePurge { // its five minutes ended - reads.stepCalls() + reads.expiredCall(node) } return } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reads.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reads.go index f9770a2f..1cc99503 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reads.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reads.go @@ -35,7 +35,7 @@ type engineReads struct { window *oldForm ctx context.Context // calls is the wait of the kept calls (callsOf); nil weighs none. - calls *buildrun.ReadsCalls + calls *enginereads.ReadsCalls } // newEngineReads asks through call (process.Conn.CallKeptOn, a kept call; ADR 0668) and keeps each answer in the state once it is bound. @@ -62,17 +62,26 @@ func (r *engineReads) stepWindow() { r.window.step(ctx, r.named(), counted) } -// stepCalls weighs the kept manifest-reads calls (buildrun.ReadsCalls): after a statement changed, a node's presence, the -// assignments's change and the end of a call's bound. +// stepCalls weighs the kept manifest-reads calls (enginereads.ReadsCalls): after a statement changed and the assignments's +// change. func (r *engineReads) stepCalls() { - if r.calls == nil { - return + if r.calls != nil { + r.calls.Step(r.context()) } - ctx := r.ctx - if ctx == nil { - ctx = context.Background() +} + +// expiredCall weighs node's kept manifest-reads call when its bound ended (the key's expiry in the state bounds). +func (r *engineReads) expiredCall(node string) { + if r.calls != nil { + r.calls.Expired(r.context(), node) } - r.calls.Step(ctx) +} + +func (r *engineReads) context() context.Context { + if r.ctx == nil { + return context.Background() + } + return r.ctx } // named are the nodes the assignments names, none dropped: what the window's step judges a read by. diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/readscall.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/readscall.go index 399b152c..4217ca6b 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/readscall.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/readscall.go @@ -2,27 +2,54 @@ package main import ( "context" + "encoding/json" "errors" + "fmt" "time" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads" mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" - "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "mesh-delivery-builder/internal/versions" ) -// callsOf is the wait of the builder's kept manifest-reads calls (buildrun.ReadsCalls): a node whose engine has not -// answered while its machine is heard is, past five minutes, the finding call-unanswered in the builder's findings -// (issue 1966, ADR 0668 §3); its bound is a key of the state bounds with its own time to live. -func (r *engineReads) callsOf(f *findings, bounds func() jetstream.KeyValue, presence *buildrun.PresenceSet, logf func(string, ...any)) *buildrun.ReadsCalls { - c := &buildrun.ReadsCalls{Keeper: r.keeper, Presence: presence, Nodes: r.named, Logf: logf, - Source: "builder." + mcc.KindCallUnanswered, Needs: "builder_engine_reads with ask_again asks the node again now and shows its statement.", - Raise: func(ctx context.Context, o mcc.Observation) error { f.raiseObservation(ctx, o); return nil }, - Clear: func(ctx context.Context, id string) error { - f.clear(ctx, mcc.KindCallUnanswered, mcc.ScopeCall, id) +// presenceInvoke is the watchdogs seat's verb presence as the manifest's invokes names it: whether a node's machine is +// heard, asked of the seat, never read from its holder's state (novox/hq ADR 0596 §2). +const presenceInvoke = "seat:" + meshwatchdogs.Seat + "." + meshwatchdogs.VerbPresence + +// heardBy asks the watchdogs seat's presence of one machine through call (process.Conn.CallKept, a read) and reads its +// declared answer: whether the machine is heard now. +func heardBy(call func(key string, body any) (json.RawMessage, error)) func(context.Context, string) (bool, error) { + return func(_ context.Context, machine string) (bool, error) { + raw, err := call(presenceInvoke, meshwatchdogs.PresenceInput{Machine: machine}) + if err != nil { + return false, err + } + var a meshwatchdogs.PresenceAnswer + if err := json.Unmarshal(raw, &a); err != nil { + return false, fmt.Errorf("%s does not read: %w", presenceInvoke, err) + } + return a.Online, nil + } +} + +// callsOf is the wait of the builder's kept manifest-reads calls (enginereads.ReadsCalls): a node whose engine has not +// answered for five minutes is, when its bound ends and the watchdogs seat says its machine is heard, the finding +// call-unanswered in the builder's findings (issue 1966, ADR 0668 §3); its bound is a key of the state bounds with its +// own time to live. +func (r *engineReads) callsOf(f *findings, bounds func() jetstream.KeyValue, heard func(context.Context, string) (bool, error), logf func(string, ...any)) *enginereads.ReadsCalls { + c := &enginereads.ReadsCalls{Keeper: r.keeper, Nodes: r.named, Heard: heard, Bound: mcc.CallUnansweredAfter, Logf: logf, + Raise: func(ctx context.Context, u enginereads.UnansweredCall) error { + o := mcc.CallUnanswered(enginereads.CallID(u.Node), enginereads.Invoke, u.Node, u.Words, "builder."+mcc.KindCallUnanswered, u.Since) + o.Needs = "builder_engine_reads with ask_again asks the node again now and shows its statement." + f.raiseObservation(ctx, o) + return nil + }, + Clear: func(ctx context.Context, node string) error { + f.clear(ctx, mcc.KindCallUnanswered, mcc.ScopeCall, enginereads.CallID(node)) return nil }, Arm: func(ctx context.Context, node string, ends time.Time) error { @@ -31,33 +58,15 @@ func (r *engineReads) callsOf(f *findings, bounds func() jetstream.KeyValue, pre return errors.New(versions.BoundsBucket + " is not on the bus yet") } left := max(time.Until(ends), time.Second) - _, err := kv.Create(ctx, buildrun.ReadsCallBoundPrefix+node, versions.EncodeBound("the manifest-reads call to "+node, ends), jetstream.KeyTTL(left)) + _, err := kv.Create(ctx, enginereads.BoundPrefix+node, versions.EncodeBound("the manifest-reads call to "+node, ends), jetstream.KeyTTL(left)) if errors.Is(err, jetstream.ErrKeyExists) { err = nil } return err }} + if f != nil && f.now != nil { + c.Now = f.now + } r.calls = c return c } - -// watchPresence follows the watchman's presence, which says a machine is heard: a node coming online or going weighs the -// calls again. -func watchPresence(ctx context.Context, js jetstream.JetStream, presence *buildrun.PresenceSet, step func(), errs chan<- error, logf func(string, ...any)) { - const without = "no node counts as heard, so a manifest-reads call left unanswered is not raised until the builder starts again" - kv := process.OptionalBucket(ctx, js, buildrun.PresenceBucket, without, logf) - if kv == nil { - return - } - each := func(e jetstream.KeyValueEntry) { - if presence.Put(e.Key(), e.Value(), !opOf(e)) { - step() - } - } - w, err := settle(ctx, kv, each) - if err != nil { - process.NotWatched(logf, buildrun.PresenceBucket, err, without) - return - } - go func() { errs <- follow(ctx, w, each) }() -} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go index c76687e2..78fbdfb7 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go @@ -139,8 +139,9 @@ func theCountedNodesEngineReads(t *testing.T) { t.Errorf("drop: %v, the key %s", err, st.v["n1"]) } t.Run("ADR 0668 §3: a manifest-reads call unanswered past five minutes, its machine heard, is call-unanswered", func(t *testing.T) { - // n2 answers nothing: while its machine is away no finding; heard and still unanswered five minutes on, the finding - // call-unanswered in the builder's findings and a bound armed before; answered, or away again, it is cleared. + // n2 answers nothing: a bound is kept at five minutes and the watchdogs seat's presence is asked only when it ends + // (ADR 0596 §2), once; away, no finding; heard, the finding call-unanswered in the builder's findings at the + // builder's clock; answered, it is cleared. place("n1", "n2") now := time.Unix(1000, 0) clock := func() time.Time { return now } @@ -154,53 +155,41 @@ func theCountedNodesEngineReads(t *testing.T) { r2.keeper.Sync, r2.keeper.Now = true, clock st := &memFindings{vals: map[string][]byte{}} armed := map[string]time.Time{} - presence := &buildrun.PresenceSet{} - calls := r2.callsOf(&findings{store: st, logf: t.Logf, now: clock}, func() jetstream.KeyValue { return nil }, presence, t.Logf) - calls.Now = clock + online, asked := false, []any{} + heard := heardBy(func(key string, body any) (json.RawMessage, error) { + asked = append(asked, key, body) + return json.Marshal(meshwatchdogs.PresenceAnswer{Machine: "n2", Online: online}) + }) + calls := r2.callsOf(&findings{store: st, logf: t.Logf, now: clock}, func() jetstream.KeyValue { return nil }, heard, t.Logf) calls.Arm = func(_ context.Context, node string, ends time.Time) error { armed[node] = ends; return nil } key := meshconditionsclient.Key(meshconditionsclient.ScopeCall, "manifest-reads.n2", meshconditionsclient.KindCallUnanswered) r2.placed() // n2 asked, unanswered at 1000 - now = now.Add(6 * time.Minute) - r2.stepCalls() - if len(st.vals) != 0 || len(armed) != 0 { - t.Fatalf("n2's machine is away: findings %v, bounds %v", st.vals, armed) + if len(st.vals) != 0 || len(asked) != 0 || !armed["n2"].Equal(time.Unix(1000, 0).Add(meshconditionsclient.CallUnansweredAfter)) { + t.Fatalf("before five minutes: findings %v, presence asked %v, bounds %v", st.vals, asked, armed) } - presence.Put("n2", presenceOnline(t, true), false) - now = time.Unix(1000, 0).Add(2 * time.Minute) - r2.stepCalls() - if len(st.vals) != 0 || !armed["n2"].Equal(time.Unix(1000, 0).Add(meshconditionsclient.CallUnansweredAfter)) { - t.Fatalf("heard before five minutes: findings %v, bounds %v", st.vals, armed) + now = now.Add(meshconditionsclient.CallUnansweredAfter) + r2.expiredCall("n2") + if len(st.vals) != 0 || len(asked) != 2 || asked[0] != presenceInvoke || asked[1] != (meshwatchdogs.PresenceInput{Machine: "n2"}) { + t.Fatalf("its machine away: findings %v, presence asked %v", st.vals, asked) } - now = time.Unix(1000, 0).Add(meshconditionsclient.CallUnansweredAfter) - r2.stepCalls() + online = true + r2.expiredCall("n2") f, err := meshconditionsclient.DecodeFinding(st.vals[key]) - if err != nil || f.Observer != builds.BuilderModule || f.Kind != meshconditionsclient.KindCallUnanswered { - t.Fatalf("heard five minutes on: finding %+v, %v (keys %v)", f, err, slices.Collect(maps.Keys(st.vals))) + if err != nil || f.Observer != builds.BuilderModule || f.Kind != meshconditionsclient.KindCallUnanswered || len(asked) != 4 { + t.Fatalf("heard five minutes on: finding %+v, %v (keys %v), presence asked %v", f, err, slices.Collect(maps.Keys(st.vals)), asked) } - presence.Put("n2", presenceOnline(t, false), false) r2.stepCalls() - if len(st.vals) != 0 { - t.Errorf("its machine away again leaves %v", st.vals) + if len(st.vals) != 1 || len(asked) != 4 { + t.Errorf("a step after it: findings %v, presence asked %v", st.vals, asked) } - presence.Put("n2", presenceOnline(t, true), false) - r2.stepCalls() down = false - r2.keeper.AskNode("n2", "answered now") + r2.keeper.AskNode("n2", "answered now, its manifest-reads-changed heard") if len(st.vals) != 0 { t.Errorf("answered leaves %v", st.vals) } }) } -func presenceOnline(t *testing.T, on bool) []byte { - t.Helper() - b, err := json.Marshal(meshwatchdogs.PresenceByNodeValue{Online: on}) - if err != nil { - t.Fatal(err) - } - return b -} - func mustStatement(t *testing.T, s enginereads.Statement) []byte { t.Helper() raw, err := enginereads.Encode(s) diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 6af6a26e..dae603fa 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -100,11 +100,11 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "queues", History: 1}, {Name: "reasks", History: 1}, {Name: "tag-ask-bounds", History: 1, KeyTTL: true}, {Name: "tag-asks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory t.Errorf("state = %+v, want %+v", m.State, want) } - // 1887/A4c: the builder chooses no node, so it reads no have, invokes no build module's tool and holds no capacity; it - // puts asks on the three build seats and reads their steps (ADR 0525 §1, §4, §5). It reads the watchman's presence - // only to know a node's machine is heard, for a manifest-reads call left unanswered (issue 1966, ADR 0668 §3). + // 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and + // holds no capacity; it puts asks on the three build seats and reads their steps (ADR 0525 §1, §4, §5). Whether a + // node's machine is heard it asks the watchdogs seat's presence (issue 1966, ADR 0596 §2, ADR 0668 §3). if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains", - "mesh-delivery-assigner.assignments", "mesh-delivery-gate.source", "mesh-delivery-watchman.presence"}; !reflect.DeepEqual(sorted(m.Reads), want) { + "mesh-delivery-assigner.assignments", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } if want := []string{"node-build-go", "node-build-image", "node-build-node"}; !reflect.DeepEqual(sorted(m.Uses), want) { @@ -124,9 +124,10 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 } } // Design 51 *The floor*: the floor's three sources, the registry seats' versions, bridge_tags and indexer_requirements. + // The watchdogs seat's presence says a node's machine is heard (issue 1966, ADR 0596 §2, ADR 0668 §3). if want := []string{"mesh-delivery-assigner.assigner_read_assignments", "mesh-delivery-forge-bridge.bridge_tags", "mesh-delivery-indexer.indexer_read_candidates", "mesh-delivery-indexer.indexer_read_manifests", "mesh-delivery-indexer.indexer_read_modules", "mesh-delivery-indexer.indexer_read_packages", "mesh-delivery-indexer.indexer_requirements", "seat:container-image-registry.remove", "seat:container-image-registry.versions", - "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-hand-acts.hand-act", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-engine.manifest-reads", "seat:npm-package-registry.remove", + "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-engine.manifest-reads", "seat:npm-package-registry.remove", "seat:npm-package-registry.versions", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions"}; !reflect.DeepEqual(sorted(m.Invokes), want) || len(m.Config) != 0 { t.Errorf("invokes = %v, config = %v, want %v and no config", m.Invokes, m.Config, want) } diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index a5fc4f0c..9ab32e47 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -67,6 +67,7 @@ "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-hand-acts.hand-act", + "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", @@ -107,8 +108,7 @@ "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-delivery-assigner.assignments", - "mesh-delivery-gate.source", - "mesh-delivery-watchman.presence" + "mesh-delivery-gate.source" ], "uses": [ "node-build-go", diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 409492b9..abd61bd5 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -506,16 +506,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // weighing only, said. holdBounds := process.AwaitBucket(ctx, js, conn, index.HoldBoundsBucket, "a hold standing past its bound is raised at its next weighing only", logger.Printf) ix.HoldBounds = expiringKeys{holdBounds} - // A kept manifest-reads call unanswered past five minutes, its node's machine heard (the watchman's presence), is - // call-unanswered in the indexer's findings (issue 1966, ADR 0668 §3). - presence := &buildrun.PresenceSet{} - reads.callsOf(ctx, findings.KV, ix.HoldBounds, presence, logger.Printf) - watchPresence(ctx, js, presence, reads.stepCalls, errs, logger.Printf) + // A kept manifest-reads call unanswered past five minutes, its node's machine heard (the watchdogs seat's presence, + // asked when the bound ends), is call-unanswered in the indexer's findings (issue 1966, ADR 0668 §3, ADR 0596 §2). + reads.callsOf(ctx, findings.KV, ix.HoldBounds, heardBy(calls.CallKept), logger.Printf) holdBounds.Ready(func(kv jetstream.KeyValue) { each := func(e jetstream.KeyValueEntry) { - if _, ok := buildrun.ReadsCallOf(e.Key()); ok { + if node, ok := enginereads.BoundOf(e.Key()); ok { if e.Operation() == jetstream.KeyValuePurge { // a manifest-reads call's five minutes ended - reads.stepCalls() + reads.expiredCall(node) } return } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go index a4b612c0..e64afeeb 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go @@ -18,7 +18,6 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/process" @@ -99,8 +98,8 @@ func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part) } }) t.Run("a manifest-reads call unanswered past five minutes, its machine heard, is call-unanswered (ADR 0668 §3)", func(t *testing.T) { - // While the node's machine is away no finding is written; heard, a bound is armed and, five minutes on, the finding - // is written in the indexer's findings; away again or answered, it is cleared. + // A bound is kept at five minutes; when it ends the watchdogs seat's presence is asked, once (ADR 0596 §2): away, + // no finding; heard, the finding in the indexer's findings, stamped with the wait's clock; answered, it is cleared. now := time.Unix(1000, 0) clock := func() time.Time { return now } down := true @@ -114,43 +113,33 @@ func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part) }} reads.keeper.Changed = reads.stepCalls armed := map[string]time.Time{} - presence := &buildrun.PresenceSet{} - calls := reads.callsOf(context.Background(), func() jetstream.KeyValue { return held }, nil, presence, t.Logf) + online, asked := false, []any{} + heard := heardBy(func(key string, body any) (json.RawMessage, error) { + asked = append(asked, key, body) + return json.Marshal(meshwatchdogs.PresenceAnswer{Machine: "ace", Online: online}) + }) + calls := reads.callsOf(context.Background(), func() jetstream.KeyValue { return held }, nil, heard, t.Logf) calls.Now = clock calls.Arm = func(_ context.Context, node string, ends time.Time) error { armed[node] = ends; return nil } - online := func(on bool) { - b, _ := json.Marshal(meshwatchdogs.PresenceByNodeValue{Online: on}) - presence.Put("ace", b, false) - } key := mcc.Key(mcc.ScopeCall, "manifest-reads.ace", mcc.KindCallUnanswered) reads.keeper.AskNode("ace", "counted") // unanswered at 1000 - now = now.Add(6 * time.Minute) - reads.stepCalls() - if len(held.vals) != 0 || len(armed) != 0 { - t.Fatalf("its machine is away: findings %v, bounds %v", held.vals, armed) + if len(held.vals) != 0 || len(asked) != 0 || !armed["ace"].Equal(time.Unix(1000, 0).Add(mcc.CallUnansweredAfter)) { + t.Fatalf("before five minutes: findings %v, presence asked %v, bounds %v", held.vals, asked, armed) } - online(true) - now = time.Unix(1000, 0).Add(time.Minute) - reads.stepCalls() - if len(held.vals) != 0 || !armed["ace"].Equal(time.Unix(1000, 0).Add(mcc.CallUnansweredAfter)) { - t.Fatalf("heard before five minutes: findings %v, bounds %v", held.vals, armed) + now = time.Unix(1000, 0).Add(mcc.CallUnansweredAfter + time.Second) + reads.expiredCall("ace") + if len(held.vals) != 0 || len(asked) != 2 || asked[0] != presenceInvoke || asked[1] != (meshwatchdogs.PresenceInput{Machine: "ace"}) { + t.Fatalf("its machine away: findings %v, presence asked %v", held.vals, asked) } - now = time.Unix(1000, 0).Add(mcc.CallUnansweredAfter) - reads.stepCalls() - if f, err := mcc.DecodeFinding(held.vals[key]); err != nil || f.Observer != "mesh-delivery-indexer" || f.Kind != mcc.KindCallUnanswered { - t.Fatalf("heard five minutes on: finding %+v, %v", f, err) + online = true + reads.expiredCall("ace") + if f, err := mcc.DecodeFinding(held.vals[key]); err != nil || f.Observer != "mesh-delivery-indexer" || f.Kind != mcc.KindCallUnanswered || !f.At.Equal(now) { + t.Fatalf("heard five minutes on: finding %+v, %v; want it at the wait's clock %v", f, err, now) } - online(false) - reads.stepCalls() - if len(held.vals) != 0 { - t.Errorf("its machine away again leaves %v", held.vals) - } - online(true) - reads.stepCalls() down = false reads.keeper.AskNode("ace", "answered") - if len(held.vals) != 0 || !reads.keeper.Statements()["ace"].Answered { - t.Errorf("answered leaves %v", held.vals) + if len(held.vals) != 0 || !reads.keeper.Statements()["ace"].Answered || len(asked) != 4 { + t.Errorf("answered leaves %v, presence asked %v", held.vals, asked) } }) t.Run("not a forge message: terminated and logged", func(t *testing.T) { // 1238 (never redelivered) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/reads.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/reads.go index 89b7c249..ede70275 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/reads.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/reads.go @@ -29,7 +29,7 @@ type engineReads struct { known bool // the assignments is read nodes []string // calls is the wait of the kept calls (callsOf); nil weighs none. ctx is the indexer's. - calls *buildrun.ReadsCalls + calls *enginereads.ReadsCalls ctx context.Context } @@ -45,14 +45,21 @@ func newEngineReads(call enginereads.Caller, gate *buildrun.ToolchainGate, chang return r } -// stepCalls weighs the kept manifest-reads calls (buildrun.ReadsCalls): after a statement changed, a node's presence, the -// assignments's change and the end of a call's bound. +// stepCalls weighs the kept manifest-reads calls (enginereads.ReadsCalls): after a statement changed and the assignments's +// change. func (r *engineReads) stepCalls() { if r.calls != nil { r.calls.Step(r.ctx) } } +// expiredCall weighs node's kept manifest-reads call when its bound ended (the key's expiry in the state hold-bounds). +func (r *engineReads) expiredCall(node string) { + if r.calls != nil { + r.calls.Expired(r.ctx, node) + } +} + // bind keeps the statements in kv and hears it: every node's last statement at the start, a person's ask again and drop // later. Every counted node is asked again once bound, so what was asked before is kept in it too. func (r *engineReads) bind(ctx context.Context, kv jetstream.KeyValue, errs chan<- error, logf func(string, ...any)) { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/readscall.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/readscall.go index 2246fcf6..aec566ce 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/readscall.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/readscall.go @@ -2,43 +2,68 @@ package main import ( "context" + "encoding/json" "errors" + "fmt" "slices" "time" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads" mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs" "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-indexer/internal/index" ) -// callsOf is the wait of the indexer's kept manifest-reads calls (buildrun.ReadsCalls): a node whose engine has not -// answered while its machine is heard is, past five minutes, the finding call-unanswered in the indexer's findings -// (issue 1966, ADR 0668 §3); its bound is a key of the state hold-bounds, which expires at its end. -func (r *engineReads) callsOf(ctx context.Context, findings func() jetstream.KeyValue, bounds index.Bounds, presence *buildrun.PresenceSet, logf func(string, ...any)) *buildrun.ReadsCalls { - c := &buildrun.ReadsCalls{Keeper: r.keeper, Presence: presence, Logf: logf, - Nodes: func() []string { r.mu.Lock(); defer r.mu.Unlock(); return slices.Clone(r.nodes) }, - Source: "indexer." + mcc.KindCallUnanswered, Needs: index.EngineReadsTool + " with ask_again asks the node again now and shows its statement.", - Raise: func(ctx context.Context, o mcc.Observation) error { +// presenceInvoke is the watchdogs seat's verb presence as the manifest's invokes names it: whether a node's machine is +// heard, asked of the seat, never read from its holder's state (novox/hq ADR 0596 §2). +const presenceInvoke = "seat:" + meshwatchdogs.Seat + "." + meshwatchdogs.VerbPresence + +// heardBy asks the watchdogs seat's presence of one machine through call (process.Conn.CallKept, a read) and reads its +// declared answer: whether the machine is heard now. +func heardBy(call func(key string, body any) (json.RawMessage, error)) func(context.Context, string) (bool, error) { + return func(_ context.Context, machine string) (bool, error) { + raw, err := call(presenceInvoke, meshwatchdogs.PresenceInput{Machine: machine}) + if err != nil { + return false, err + } + var a meshwatchdogs.PresenceAnswer + if err := json.Unmarshal(raw, &a); err != nil { + return false, fmt.Errorf("%s does not read: %w", presenceInvoke, err) + } + return a.Online, nil + } +} + +// callsOf is the wait of the indexer's kept manifest-reads calls (enginereads.ReadsCalls): a node whose engine has not +// answered for five minutes is, when its bound ends and the watchdogs seat says its machine is heard, the finding +// call-unanswered in the indexer's findings, stamped with the wait's clock (issue 1966, ADR 0668 §3); its bound is a key +// of the state hold-bounds, which expires at its end. +func (r *engineReads) callsOf(ctx context.Context, findings func() jetstream.KeyValue, bounds index.Bounds, heard func(context.Context, string) (bool, error), logf func(string, ...any)) *enginereads.ReadsCalls { + c := &enginereads.ReadsCalls{Keeper: r.keeper, Heard: heard, Bound: mcc.CallUnansweredAfter, Logf: logf, + Nodes: func() []string { r.mu.Lock(); defer r.mu.Unlock(); return slices.Clone(r.nodes) }, + Raise: func(ctx context.Context, u enginereads.UnansweredCall) error { kv := findings() if kv == nil { return process.ErrNotYet } - value, err := mcc.EncodeFinding(o.Finding(index.Module, "", time.Now().UTC())) + o := mcc.CallUnanswered(enginereads.CallID(u.Node), enginereads.Invoke, u.Node, u.Words, "indexer."+mcc.KindCallUnanswered, u.Since) + o.Needs = index.EngineReadsTool + " with ask_again asks the node again now and shows its statement." + value, err := mcc.EncodeFinding(o.Finding(index.Module, "", u.At)) if err == nil { _, err = kv.Put(ctx, o.Key(), value) } return err }, - Clear: func(ctx context.Context, id string) error { + Clear: func(ctx context.Context, node string) error { kv := findings() if kv == nil { return nil } - key := mcc.Key(mcc.ScopeCall, id, mcc.KindCallUnanswered) + key := mcc.Key(mcc.ScopeCall, enginereads.CallID(node), mcc.KindCallUnanswered) if _, err := kv.Get(ctx, key); err != nil { if errors.Is(err, jetstream.ErrKeyNotFound) { return nil @@ -46,31 +71,12 @@ func (r *engineReads) callsOf(ctx context.Context, findings func() jetstream.Key return err } return kv.Delete(ctx, key) - }, - Arm: func(ctx context.Context, node string, ends time.Time) error { - return bounds.Arm(ctx, buildrun.ReadsCallBoundPrefix+node, ends) }} + if bounds != nil { + c.Arm = func(ctx context.Context, node string, ends time.Time) error { + return bounds.Arm(ctx, enginereads.BoundPrefix+node, ends) + } + } r.ctx, r.calls = ctx, c return c } - -// watchPresence follows the watchman's presence, which says a machine is heard: a node coming online or going weighs the -// calls again. -func watchPresence(ctx context.Context, js jetstream.JetStream, presence *buildrun.PresenceSet, step func(), errs chan<- error, logf func(string, ...any)) { - const without = "no node counts as heard, so a manifest-reads call left unanswered is not raised until the indexer starts again" - kv := process.OptionalBucket(ctx, js, buildrun.PresenceBucket, without, logf) - if kv == nil { - return - } - each := func(e jetstream.KeyValueEntry) { - if presence.Put(e.Key(), e.Value(), !opOf(e)) { - step() - } - } - w, err := settle(ctx, kv, each) - if err != nil { - process.NotWatched(logf, buildrun.PresenceBucket, err, without) - return - } - go func() { errs <- follow(ctx, w, each) }() -} diff --git a/modules/mesh-delivery-indexer/go.mod b/modules/mesh-delivery-indexer/go.mod index 84dc549a..3302bf36 100644 --- a/modules/mesh-delivery-indexer/go.mod +++ b/modules/mesh-delivery-indexer/go.mod @@ -68,7 +68,7 @@ require ( require ( git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 // indirect git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts v0.0.0-00010101000000-000000000000 // indirect - git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs v0.0.0-00010101000000-000000000000 // indirect + git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/clients/containerimageregistryclient v0.0.0-00010101000000-000000000000 // indirect git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000 // indirect git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry v0.0.0-00010101000000-000000000000 // indirect diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 86c933e3..09028dc2 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -137,8 +137,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build } } // DLV-F1r (ADR 0615 §1): the assigner's assignments is read through its first view and views family, never its state. - // The watchman's presence says a node's machine is heard, for a manifest-reads call left unanswered (issue 1966, ADR 0668 §3). - if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-delivery-builder.build-log", "mesh-delivery-checker.previews", "mesh-delivery-gate.source", "mesh-delivery-watchman.presence"}; !reflect.DeepEqual(sorted(m.Reads), want) { + if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-delivery-builder.build-log", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } // "Reads FORGE" is a consumes of the indexer's stream: the broker grants only buckets by reads (mesh-control @@ -158,7 +157,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths: // ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a // commit is in another (issue 1900). - if want := []string{"mesh-delivery-assigner.assignments", "gitea.gitea_commit_trees", "gitea.gitea_contains", "gitea.gitea_get_file", "gitea.gitea_open_pulls", "mesh-delivery-checker.checker_tries", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:node-engine.manifest-reads", "seat:mesh-hand-acts.hand-act"}; !reflect.DeepEqual(m.Invokes, want) { + if want := []string{"mesh-delivery-assigner.assignments", "gitea.gitea_commit_trees", "gitea.gitea_contains", "gitea.gitea_get_file", "gitea.gitea_open_pulls", "mesh-delivery-checker.checker_tries", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:node-engine.manifest-reads", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence"}; !reflect.DeepEqual(m.Invokes, want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } // 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none. The same bundle diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 9344bd46..c820abfc 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -119,8 +119,7 @@ "mesh-build-node.toolchains", "mesh-delivery-builder.build-log", "mesh-delivery-checker.previews", - "mesh-delivery-gate.source", - "mesh-delivery-watchman.presence" + "mesh-delivery-gate.source" ], "invokes": [ "mesh-delivery-assigner.assignments", @@ -132,7 +131,8 @@ "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:node-engine.manifest-reads", - "seat:mesh-hand-acts.hand-act" + "seat:mesh-hand-acts.hand-act", + "seat:mesh-watchdogs.presence" ], "own-secrets": { "broker": "${dir:state}/broker" From 262d21c431b3682f91190363689ab8f58cdb5032 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 10 Nov 2026 05:01:15 +0100 Subject: [PATCH 22/22] =?UTF-8?q?D14:=20the=20assigner=20keeps=20its=20man?= =?UTF-8?q?ifest-reads=20wait=20through=20the=20node-engine=20seat's=20cli?= =?UTF-8?q?ent's=20ReadsCalls,=20its=20own=20copy=20gone=20(issue=201966,?= =?UTF-8?q?=20ADR=200668=20=C2=A73,=20ADR=200634=20=C2=A71;=20review=20rou?= =?UTF-8?q?nd=203,=20finding=202)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Its presence says heard at each pass, so it keeps no bound; the findings it holds from an earlier process are weighed through Held. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-assigner/pass.go | 98 +++++++++---------- .../cmd/mesh-delivery-assigner/pass_test.go | 2 +- 2 files changed, 45 insertions(+), 55 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index 6f067139..31cebd7b 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -43,8 +43,8 @@ type passer struct { said map[string]bool // findings is the assigner's findings bucket (its said conditions, ADR 0564 §2). findings assign.Findings - // unansweredSaid are the call-unanswered findings of manifest-reads this process wrote (ADR 0668 §3). - unansweredSaid map[string]bool + // calls is the wait of the kept manifest-reads calls (enginereads.ReadsCalls, ADR 0668 §3), made at the first pass. + calls *enginereads.ReadsCalls // failedProvisions are the failed provisions whose finding this process wrote (ADR 0570 §4). failedProvisions map[string]bool // missing are the required provisions with no key this process pulsed, and missingSaid those whose finding it wrote. @@ -122,69 +122,59 @@ func (p *passer) engineReads(n assign.Nodes) { p.unansweredReads(known, n.Online) } -// readsCallID is the finding's id for a node's manifest-reads: the node the kept call waits on. -func readsCallID(node string) string { return "manifest-reads." + node } - // unansweredReads keeps the wait of every node whose engine has not answered manifest-reads while its machine is heard -// (online): it is asked again once its last ask ended, so the kept call goes on until answered; past +// (online), through the node-engine seat's client (enginereads.ReadsCalls, shared with the builder and the indexer): it +// is asked again once its last ask ended, so the kept call goes on until answered; past // meshconditionsclient.CallUnansweredAfter it is the finding call-unanswered in the assigner's findings, cleared once -// answered or while its machine is away, when none is written (novox/hq ADR 0668 §3). Each pass is the event: no timer. +// answered (novox/hq ADR 0668 §3). Each pass is the event, and the assigner's own presence says heard: no timer, no bound. func (p *passer) unansweredReads(known []string, online map[string]bool) { if p.reads.Call == nil { return } - heard := func(node string) bool { return online[node] } - for _, node := range known { - if s, ok := p.reads.Statements()[node]; ok && !s.Answered && heard(node) && !p.reads.Asking(node) { - p.reads.AskNode(node, "unanswered since "+s.UnansweredSince.UTC().Format(time.RFC3339)+", its machine heard") - } - } if p.findings.Put == nil || p.findings.Delete == nil { + for _, node := range known { + if s, ok := p.reads.Statements()[node]; ok && !s.Answered && online[node] && !p.reads.Asking(node) { + p.reads.AskNode(node, "unanswered since "+s.UnansweredSince.UTC().Format(time.RFC3339)+", its machine heard") + } + } return } - now := p.clock() - statements := p.reads.Statements() - raised := map[string]bool{} - for node, since := range p.reads.Unanswered(known, heard, mcc.CallUnansweredAfter, now) { - o := mcc.CallUnanswered(readsCallID(node), enginereads.Invoke, node, statements[node].Words, "assigner."+mcc.KindCallUnanswered, since) - o.Needs = "assigner_engine_reads with ask_again asks the node again now and shows its statement." - key := o.Key() - raised[key] = true - if p.unansweredSaid[key] { - continue - } - value, err := mcc.EncodeFinding(o.Finding(plc.Module, "", now.UTC())) - if err == nil { - err = p.findings.Put(key, value) - } - if err != nil { - p.logf("the finding %s is not written: %v", key, err) - continue - } - p.logf("%s on %s is unanswered since %s and its machine is heard: the finding %s", enginereads.Verb, node, since.UTC().Format(time.RFC3339), key) - if p.unansweredSaid == nil { - p.unansweredSaid = map[string]bool{} - } - p.unansweredSaid[key] = true - } - // What a pass before or another process wrote: every such finding of the assigner's no longer raised is cleared. - prefix, suffix := mcc.ScopeCall+"."+readsCallID(""), "."+mcc.KindCallUnanswered - keys := slices.Collect(maps.Keys(p.unansweredSaid)) - if p.findings.Keys != nil { - if held, err := p.findings.Keys(); err == nil { - keys = append(keys, held...) + if p.calls == nil || p.calls.Keeper != p.reads { + p.calls = &enginereads.ReadsCalls{Keeper: p.reads, Bound: mcc.CallUnansweredAfter, Now: p.clock, Logf: p.logf, + Raise: func(_ context.Context, u enginereads.UnansweredCall) error { + o := mcc.CallUnanswered(enginereads.CallID(u.Node), enginereads.Invoke, u.Node, u.Words, "assigner."+mcc.KindCallUnanswered, u.Since) + o.Needs = "assigner_engine_reads with ask_again asks the node again now and shows its statement." + value, err := mcc.EncodeFinding(o.Finding(plc.Module, "", u.At)) + if err == nil { + err = p.findings.Put(o.Key(), value) + } + return err + }, + Clear: func(_ context.Context, node string) error { + return p.findings.Delete(mcc.Key(mcc.ScopeCall, enginereads.CallID(node), mcc.KindCallUnanswered)) + }} + if p.findings.Keys != nil { + // What a pass before or another process wrote: every such finding of the assigner's is weighed to be cleared. + prefix, suffix := mcc.ScopeCall+"."+enginereads.CallID(""), "."+mcc.KindCallUnanswered + p.calls.Held = func(context.Context) ([]string, error) { + keys, err := p.findings.Keys() + var nodes []string + for _, key := range keys { + if rest, ok := strings.CutPrefix(key, prefix); ok { + if node, ok := strings.CutSuffix(rest, suffix); ok { + nodes = append(nodes, node) + } + } + } + return nodes, err + } + } else { + p.calls.Held = func(context.Context) ([]string, error) { return nil, nil } } } - for _, key := range keys { - if raised[key] || !strings.HasPrefix(key, prefix) || !strings.HasSuffix(key, suffix) { - continue - } - if err := p.findings.Delete(key); err != nil { - p.logf("the finding %s is not cleared: %v", key, err) - continue - } - delete(p.unansweredSaid, key) - } + p.calls.Nodes = func() []string { return known } + p.calls.Heard = func(_ context.Context, node string) (bool, error) { return online[node], nil } + p.calls.Step(context.Background()) } func (p *passer) clock() time.Time { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index 50cba6c4..b8e5bcbd 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -287,7 +287,7 @@ func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) { Delete: func(k string) error { delete(held, k); return nil }, Keys: func() ([]string, error) { return slices.Collect(maps.Keys(held)), nil }}, reads: &enginereads.Keeper{Call: silent, Sync: true, Now: func() time.Time { return clock }}} - key := mcc.Key(mcc.ScopeCall, readsCallID("ace"), mcc.KindCallUnanswered) + key := mcc.Key(mcc.ScopeCall, enginereads.CallID("ace"), mcc.KindCallUnanswered) before := asked["ace"] q.engineReads(w.nodes(nil, nil)) clock = clock.Add(6 * time.Minute)