A-476 r1: no merge waits for the build the indexer runs; leftovers of the validation and the hold go; a commit's scope is decided inside the recompute

ErrOwnBuildUnknown, its say and the own-build-unknown wait, TakeIn.Unknown and Failed go; the build only names the writer.
indexer_take_in_pending takes only a group in. The node-build-go accept.check and output invokes go. A module of another
repository stays as held inside of(), so a base of this repository's module is not given a value that is never written
(A-476 c, A-476a); one case in the toolchains test proves it.

Builds: design 47 G A-476, A-476a
This commit is contained in:
2026-11-17 20:33:52 +01:00
parent 40b6ced375
commit a79b811113
13 changed files with 103 additions and 180 deletions
@@ -91,7 +91,7 @@ func (ix *Indexer) TakeInBehind(ctx context.Context) []string {
}
switch {
case held[head]:
// A head the pending holds (a member waiting for its group's end or its build) is taken in by that path, whole with
// A head the pending holds (a member waiting for its group's end) is taken in by that path, whole with
// its group: it is not behind (A-463a).
ix.clearUntaken(ctx, key, "the head of "+repository+" waits in the indexer's pending")
case merge == head:
@@ -24,9 +24,8 @@ import (
"mesh-delivery-indexer/judge"
)
// The take-in step (novox/hq ADR 0633): a source revision is taken in only by the indexer it names. A merged message is
// kept in the indexer's own state pending, one key per merge, until the take-in that ends it; a revision naming another
// build of the indexer writes only what building and moving that indexer needs, and the rest stays pending for it.
// The take-in step (novox/hq ADR 0633, A-476): a merged message is kept in the indexer's own state pending, one key per
// merge, until the take-in that ends it; a revision is taken in by the running indexer whatever build of the indexer it names.
const (
// PendingState is the indexer's state of held merges: the bucket mesh-delivery-indexer_pending, keyed
// takein.PendingKey, so the key order is the merge order. An old `state` entry (the indexer is foundation, ADR
@@ -45,10 +44,6 @@ var errClaimLost = errors.New("another take-in claimed the merges first, so this
// runs such a build, so the revision waits until a later merge gives it one or a hand take-in (indexer_take_in_pending).
const NoBuild = "no build: "
// ErrOwnBuildUnknown is what a merged message answers while the indexer cannot learn the build it runs (ADR 0633 §1):
// the message is left unacknowledged, so the bus delivers it again, and nothing is written.
var ErrOwnBuildUnknown = errors.New("the build of the indexer this program runs is not known")
// PendingRef is what a pending value keeps of its merged message instead of a copy (a merge of a repository with many
// modules is about 0.5 MB, over the state's 256 KiB value cap; issue 1585): the small fields the waits are worked out
// from. The message itself is read from the forge bridge's stream by Pending.Seq when a take-in needs it.
@@ -384,26 +379,13 @@ func (ix *Indexer) pendingOnBus(ctx context.Context) bool {
}
// ownBuildNow is the build this program runs, as FixOwnBuild fixed it at the start: empty while unknown, and never asked again.
// It names only the writer of what is written (LastWriter); nothing waits for it (A-476 b).
func (ix *Indexer) ownBuildNow(context.Context) string {
ix.ownMu.Lock()
defer ix.ownMu.Unlock()
return ix.ownFP
}
// sayOwnBuildUnknown says, for a merge this process cannot take in because it does not know its own build (ADR 0633 §1), a
// take-in.pending that names why and since when, so the watchman's take-in-waits bound covers it. The merge is not written to
// pending; the stream delivers it again to the process that knows its build.
func (ix *Indexer) sayOwnBuildUnknown(ctx context.Context, seq uint64, f forgemsg.Forge) {
ix.ownMu.Lock()
since, why := ix.ownSince, ix.ownWhy
ix.ownMu.Unlock()
if since.IsZero() { // never fixed: the start did not run (no engine wired)
since, why = ix.now(), "the build this program runs was never asked"
}
ix.sayTakeIn(ctx, takein.TakeIn{Kind: takein.TakeInPending, Key: takein.PendingKey(seq), What: takein.TakeInRevision,
Merges: []takein.Merge{{Repository: f.Repository, Number: f.Number, Merge: f.Merge}}, Since: since, Unknown: why})
}
// sayTakeIn publishes a take-in event and keeps what it said on the entries, so indexer_pending reads the last event of
// each wait from the state. A publish that fails is said in the log: the next evaluation says it again.
func (ix *Indexer) sayTakeIn(ctx context.Context, t takein.TakeIn) bool {
@@ -451,10 +433,6 @@ func (ix *Indexer) Arrive(ctx context.Context, seq uint64, f forgemsg.Forge) (Ap
return a, "", err
}
own := ix.ownBuildNow(ctx)
if own == "" {
ix.sayOwnBuildUnknown(ctx, seq, f)
return Applied{}, "", ErrOwnBuildUnknown
}
ix.pendingMu.Lock()
defer ix.pendingMu.Unlock()
key := takein.PendingKey(seq)
@@ -497,9 +475,6 @@ func (ix *Indexer) EndGroup(ctx context.Context, end forgemsg.ForgeGroupEnd) (Ap
return Applied{}, "no pending state: the group's merges were taken in one by one", nil
}
own := ix.ownBuildNow(ctx)
if own == "" {
return Applied{}, "", ErrOwnBuildUnknown
}
ix.pendingMu.Lock()
defer ix.pendingMu.Unlock()
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
@@ -672,21 +647,15 @@ func (ix *Indexer) takenAlready(ctx context.Context, m forgemsg.GroupMerge) (boo
}
// TakeInAtStart is the take-in at the start (ADR 0633 §2), before the consumer on the forge bridge's stream starts:
// it evaluates pending once with the build this program runs, taking in the whole revisions that name this build; a
// group still being gathered stays, and a revision naming yet another build is held for that one. A claim left by an
// indexer that ran before is void here. When the build is not known (the node-engine did not answer), it waits for
// nothing: it says so and the consumer starts all the same; a merged message is then left unacknowledged until the
// build is known (clause 1), and its redelivery, or a toolchain report (ToolchainReported), asks again. No timer.
// it evaluates pending once, taking in the whole revisions whatever build of the indexer they name (A-476 b); a group
// still being gathered stays. A claim left by an indexer that ran before is void here. The build this program runs, known
// or not, only names the writer. No timer.
func (ix *Indexer) TakeInAtStart(ctx context.Context) (Applied, string, error) {
if ix.Pending == nil || !ix.pendingOnBus(ctx) {
// Nothing can wait in a state not on the bus: the start asks no build and does not wait for one.
return Applied{}, "the pending state is not on the bus yet: nothing waits", nil
}
own := ix.ownBuildNow(ctx)
if own == "" {
ix.logf("take-in: the build this program runs is not known at its start, so nothing pending is taken in now; the forge consumer starts, and a merged message left unacknowledged asks it again")
return Applied{}, "the build this program runs is not known yet: nothing pending is taken in at the start", nil
}
ix.pendingMu.Lock()
defer ix.pendingMu.Unlock()
a, words, err := ix.evaluate(ctx, own, true)
@@ -694,8 +663,8 @@ func (ix *Indexer) TakeInAtStart(ctx context.Context) (Applied, string, error) {
return a, "the pending state is not on the bus yet: nothing waits", nil
}
if err == nil && len(a.Written) == 0 {
// A start that takes nothing in says so, with the build it learnt (the wait's own build is in the line holdFor says).
ix.logf("take-in: at the start, this program runs build %s and took nothing in: %s", own, words)
// A start that takes nothing in says so.
ix.logf("take-in: at the start, this program (build %q) took nothing in: %s", own, words)
}
return a, words, err
}
@@ -726,16 +695,12 @@ func (ix *Indexer) UnreadAgain(ctx context.Context) error {
}
// PendingAgain evaluates pending again after something a fingerprint is computed from changed, a toolchain report
// (ADR 0633 §1, §3), or the verdicts state bound (issue 1966): a revision that now names the running build is taken in, and one that names another build is held
// for that one. It does nothing while the pending state is not on the bus or the build this program runs is not known.
// (ADR 0633 §3), or the verdicts state bound (issue 1966): what waits is taken in. It does nothing while the pending state is not on the bus.
func (ix *Indexer) PendingAgain(ctx context.Context) error {
if ix.Pending == nil || !ix.pendingOnBus(ctx) {
return nil
}
own := ix.ownBuildNow(ctx)
if own == "" {
return nil
}
ix.pendingMu.Lock()
defer ix.pendingMu.Unlock()
_, words, err := ix.evaluate(ctx, own, false)
@@ -881,12 +846,12 @@ func (ix *Indexer) holdLost(ctx context.Context, ready, lost []PendingEntry, rea
}
for _, g := range lost {
if unread[g.Key] {
ix.logf("take-in: %s (%s#%d) is not taken in and holds its revision, its message is unread for a transport fault: %s; it is read again on the next message heard on the forge, at the start, and by %s",
g.Key, g.Forge.Repository, g.Forge.Number, reasons[g.Key], TakeInPendingTool)
ix.logf("take-in: %s (%s#%d) is not taken in and holds its revision, its message is unread for a transport fault: %s; it is read again on the next message heard on the forge and at the start",
g.Key, g.Forge.Repository, g.Forge.Number, reasons[g.Key])
continue
}
ix.logf("take-in: %s (%s#%d) is not taken in and holds its revision, the forge bridge could not give its message: %s; %s reads it again; indexer_drop_pending drops it once a later merge of %s covers it",
g.Key, g.Forge.Repository, g.Forge.Number, reasons[g.Key], TakeInPendingTool, g.Forge.Repository)
ix.logf("take-in: %s (%s#%d) is not taken in and holds its revision, the forge bridge could not give its message: %s; indexer_drop_pending drops it once a later merge of %s covers it",
g.Key, g.Forge.Repository, g.Forge.Number, reasons[g.Key], g.Forge.Repository)
}
if !ix.sayTakeIn(ctx, t) {
return nil
@@ -1222,8 +1187,7 @@ func (ix *Indexer) nextSource(branch string) int64 {
// takenWaits are the taken events of a take-in: every wait the entries were said under (saidWaits), every wait
// indexer_pending names the claimed entries by now (WaitsOf: a revision by its oldest merge's key, a group by its g. key) and
// the key of every claimed entry, since indexer_pending named it by that key earlier when it waited behind a group. So a wait
// the watchman learned of without the indexer marking it (seeded from indexer_pending, or said by the own-build-unknown
// path) ends too (design 47, Taken in by the indexer it names). No key is said twice.
// the watchman learned of without the indexer marking it (seeded from indexer_pending) ends too. No key is said twice.
func takenWaits(entries []PendingEntry) []takein.TakeIn {
out := saidWaits(entries)
said := map[string]bool{}
@@ -1354,9 +1318,9 @@ func waitOf(entries []PendingEntry, what, behind string) takein.PendingWait {
return pw
}
// TakeInPendingByHand is the hand take-in (indexer_take_in_pending, ADR 0633 §5): the wait named by its key, the
// ready revision (its m. key) or a group gathered so far (its g. key), is taken in with this indexer and its judge,
// whatever build it names, for when the named indexer can never run or the group's end never comes. by and why are
// TakeInPendingByHand is the hand take-in (indexer_take_in_pending, ADR 0633 §5, A-476 b): the group gathered so far, named by
// its g. key, is taken in with this indexer and its judge,
// whatever build it names, for when the group's end never comes. by and why are
// said in the journal and on the take-in.taken event. What waited behind it is evaluated again after.
func (ix *Indexer) TakeInPendingByHand(ctx context.Context, key, by, why string) TakenIn {
r := TakenIn{Module: key}
@@ -1381,8 +1345,9 @@ func (ix *Indexer) TakeInPendingByHand(ctx context.Context, key, by, why string)
take = held
}
}
case len(w.Ready) > 0 && w.Ready[0].Key == key:
take = w.Ready
default:
r.Answer = fmt.Sprintf("%s names no delivery group: only a group is taken in by hand (a revision is taken in by the indexer itself); indexer_pending names the waits", key)
return r
}
if len(take) == 0 {
r.Answer = fmt.Sprintf("nothing waits under %s: indexer_pending names the waits", key)
@@ -1460,8 +1425,8 @@ func (ix *Indexer) DropPending(ctx context.Context, key, by, why string, abandon
}
if !covered {
r.Answer = fmt.Sprintf("%s is not dropped: no later merge of %s, pending or taken in, has %s as an ancestor, so its "+
"changes would never be taken in (ADR 0633 §5); %s reads its message again, or abandon: true drops it all the same",
key, e.Forge.Repository, e.Forge.Merge, TakeInPendingTool)
"changes would never be taken in (ADR 0633 §5); abandon: true drops it all the same",
key, e.Forge.Repository, e.Forge.Merge)
return r
}
}
@@ -186,53 +186,21 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
})
}
})
t.Run("T6 a merge while the own build is unknown writes nothing", func(t *testing.T) {
t.Run("T6 a merge while the own build is unknown is taken in all the same (A-476 b)", func(t *testing.T) {
b := newTakeInBed(t)
b.ix.ownFP, b.ix.ownFixed = "", false // this process is started again and asks the engine, which does not answer
b.ix.OwnBuild = func(context.Context) (string, error) { return "", errors.New("the engine does not answer") }
b.ix.FixOwnBuild(ctx)
b.said = nil
_, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", ""))
if !errors.Is(err, ErrOwnBuildUnknown) || len(b.keys()) != 0 || b.commit(t, "avahi") == sha('b') {
t.Errorf("err %v, pending %v, avahi at %s: want ErrOwnBuildUnknown and nothing written", err, b.keys(), b.commit(t, "avahi"))
if _, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", "tree-indexer-2")); err != nil || len(b.keys()) != 0 || b.commit(t, "avahi") != sha('b') {
t.Errorf("err %v, pending %v, avahi at %s: want the merge taken in at once", err, b.keys(), b.commit(t, "avahi"))
}
// The merge is said pending with why and since when (so the watchman's bound covers it), and indexer_pending says the
// build is unknown, from the own-build state.
if len(b.said) != 1 || b.said[0].Kind != takein.TakeInPending || b.said[0].Key != takein.PendingKey(1) || !strings.Contains(b.said[0].Unknown, "does not answer") ||
b.said[0].Since.IsZero() || b.said[0].WaitsFor != "" {
t.Errorf("said %+v, want one take-in.pending of m.1 saying why the own build is unknown", b.said)
} else if _, err := takein.EncodeTakeIn(b.said[0]); err != nil {
t.Errorf("the unknown-build event is refused by its own contract: %v", err)
if len(b.said) != 1 || b.said[0].Kind != takein.TakeInTaken {
t.Errorf("said %+v, want one take-in.taken and no wait said", b.said)
}
answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{})
if a, _ := answer.(takein.PendingAnswer); err != nil || !strings.Contains(a.OwnBuildUnknown, "does not answer") || a.OwnBuildSince.IsZero() {
t.Errorf("indexer_pending = %+v %v, want the build unknown with why and since", answer, err)
}
// Before the pending state is on the bus nothing waits: the start asks no build, and the merge is taken in at once,
// as before ADR 0633, though the build is unknown (TestIndexerWhole found the start waiting on it for ever).
b.ix.Pending = &memStore{unbound: process.ErrNotYet}
if _, _, err := b.ix.TakeInAtStart(ctx); err != nil {
t.Fatal(err)
}
if _, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", "")); err != nil || b.commit(t, "avahi") != sha('b') {
t.Errorf("no pending state: err %v, avahi at %s: want the merge taken in at once", err, b.commit(t, "avahi"))
}
})
t.Run("1966/AC195-2 AC195-3 a merge said pending with its build unknown and taken in on redelivery ends that wait under its key; a revision wait is said once", func(t *testing.T) {
b := newTakeInBed(t)
ownBuild := b.ix.OwnBuild
b.ix.ownFP, b.ix.ownFixed = "", false
b.ix.OwnBuild = func(context.Context) (string, error) { return "", errors.New("the engine does not answer") }
b.ix.FixOwnBuild(ctx)
b.said = nil
if _, _, err := b.ix.Arrive(ctx, 7, b.merge(30, sha('b'), "tree-avahi-2", "")); !errors.Is(err, ErrOwnBuildUnknown) {
t.Fatal(err)
}
b.ix.OwnBuild, b.ix.ownFP, b.ix.ownFixed = ownBuild, "", false
b.ix.FixOwnBuild(ctx)
b.arrive(t, 7, b.merge(30, sha('b'), "tree-avahi-2", "")) // the stream delivers it again
if want := []string{"pending " + takein.PendingKey(7), "taken " + takein.PendingKey(7)}; !slices.Equal(b.events(), want) || len(b.keys()) != 0 {
t.Fatalf("said %v, pending %v: want %v and nothing pending", b.events(), b.keys(), want)
if a, _ := answer.(takein.PendingAnswer); err != nil || a.OwnBuild != "" || a.OwnBuildSince.IsZero() || len(a.Waits) != 0 {
t.Errorf("indexer_pending = %+v %v, want no wait and no known build", answer, err)
}
})
t.Run("T7 and T3 a merge alone, an indexer change, a merge behind it and a revert", func(t *testing.T) {
@@ -371,7 +339,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
t.Errorf("indexer_pending %+v, want the refusal naming the cycle", waits)
}
})
t.Run("T10 a start takes what waits in once its build is known; a gathered group stays", func(t *testing.T) {
t.Run("T10 a start takes what waits in whatever its build; a gathered group stays", func(t *testing.T) {
b := newTakeInBed(t)
b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
g := b.merge(30, sha('f'), "", "")
@@ -390,30 +358,21 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
n.FixOwnBuild(ctx)
return n
}
asked := 0
next := mk(func(context.Context) (string, error) { asked++; return "", errors.New("not yet") })
if r := next.TakeInPendingByHand(ctx, "m.nothing", "lead@shanks", "a test"); r.Stored || !strings.Contains(r.Answer, "nothing waits") {
t.Errorf("a hand ask while the own build is unknown: %+v", r)
// The build this program runs is not known (the engine does not answer at its start): nothing waits for it, so the start
// takes the revision in all the same (A-476 b), and a hand ask names only groups.
next := mk(func(context.Context) (string, error) { return "", errors.New("not yet") })
if r := next.TakeInPendingByHand(ctx, "m.nothing", "lead@shanks", "a test"); r.Stored || !strings.Contains(r.Answer, "names no delivery group") {
t.Errorf("a hand ask for a revision: %+v", r)
}
if _, words, err := next.TakeInAtStart(ctx); err != nil || len(b.keys()) != 2 || !strings.Contains(words, "not known") {
t.Fatalf("a start with its build unknown: %q %v, pending %v: want it to answer at once and take nothing in", words, err, b.keys())
}
// The engine answers now, and the same process does not ask it again: it learnt nothing it did not know at its start.
next.OwnBuild = func(context.Context) (string, error) { asked++; return own, nil }
if _, words, err := next.TakeInAtStart(ctx); err != nil || asked != 1 || len(b.keys()) != 2 || !strings.Contains(words, "not known") {
t.Fatalf("the window of a replacement: %q %v, asked %d, pending %v: want the build fixed at the start, never learnt later", words, err, asked, b.keys())
}
if answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{}); err != nil || answer.(takein.PendingAnswer).OwnBuild != own || answer.(takein.PendingAnswer).OwnBuildSince.IsZero() {
t.Errorf("indexer_pending = %+v %v, want the known build and since", answer, err)
}
// The module started again fixes the build at its start and takes the revision in.
next = mk(func(context.Context) (string, error) { return own, nil })
if _, _, err := next.TakeInAtStart(ctx); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(b.keys(), []string{takein.PendingKey(5)}) || b.commit(t, "avahi") != sha('c') {
t.Errorf("start: pending %v, avahi at %s: want the revision taken in, the group left", b.keys(), b.commit(t, "avahi"))
}
if answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{}); err != nil || answer.(takein.PendingAnswer).OwnBuild != own {
t.Errorf("indexer_pending = %+v %v, want the build the original program knows", answer, err)
}
})
t.Run("T8 T9 T11 a group gathered until its end, a merge behind it, one alone, an end again", func(t *testing.T) {
b := newTakeInBed(t)
@@ -886,25 +845,33 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
t.Fatalf("pending %v: want m.2 dropped and m.3 pending", b.keys())
}
// Issue 1966 (#650), ADR 0633 §5: the newest pending merge of its repository, which no taken merge covers, is not
// dropped; the answer names the read again. abandon drops it all the same (below).
// dropped; the answer names abandon, which drops it all the same (below).
if r := b.ix.DropPending(ctx, takein.PendingKey(3), "lead@shanks", "lost message", false); r.Stored ||
!strings.Contains(r.Answer, TakeInPendingTool) || len(b.keys()) != 1 {
!strings.Contains(r.Answer, "abandon") || len(b.keys()) != 1 {
t.Fatalf("a drop nothing covers: %+v, pending %v", r, b.keys())
}
if r := b.ix.DropPending(ctx, takein.PendingKey(3), "lead@shanks", "abandoned", true); !r.Stored || len(b.keys()) != 0 {
t.Fatalf("an abandoned drop: %+v, pending %v", r, b.keys())
}
// A delivery group gathered is taken in by hand (A-476 b: only a group is).
b.ix.Pins = newMem()
b.ix.Revisions = newMemObjects(0) // A-448: the taken event names the catalog graph revision the take-in made
gm := b.merge(24, sha('f'), "tree-avahi-6", "")
gm.Group, gm.Head = "feat/h", sha('1')
gm.Members = []meshdeliveryclient.Head{{Repository: catalogueRepo, Number: 24, Head: sha('1')}, {Repository: "novox/mesh-sdk", Number: 25, Head: sha('2')}}
b.arrive(t, 5, gm)
b.said = nil
if r := b.ix.TakeInPendingByHand(ctx, takein.PendingKey(3), "lead@shanks", "the named indexer can never run"); !r.Stored {
if r := b.ix.TakeInPendingByHand(ctx, takein.GroupTakeInKey("feat/h"), "lead@shanks", "the group's end never comes"); !r.Stored {
t.Fatalf("hand take-in: %+v", r)
}
if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('d') || len(b.said) != 1 || b.said[0].Kind != takein.TakeInTaken || b.said[0].By != "lead@shanks" {
if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('f') || len(b.said) < 1 || b.said[0].Kind != takein.TakeInTaken || b.said[0].By != "lead@shanks" {
t.Errorf("hand take-in: pending %v, avahi at %s, said %+v", b.keys(), b.commit(t, "avahi"), b.said)
}
// §476, §493: a hand take-in's taken event carries the branch, the source revision it makes and its entries.
if len(b.said) == 1 {
if len(b.said) >= 1 {
got := b.said[0]
if got.Branch != TrunkBranch || got.Source <= 0 || len(got.Entries) != 2 || got.Entries[1].Module != "avahi" ||
got.Entries[1].Commit != sha('d') || got.Entries[1].Fingerprint == "" {
if got.Branch != TrunkBranch || got.Source <= 0 || len(got.Entries) != 1 || got.Entries[0].Module != "avahi" ||
got.Entries[0].Commit != sha('f') || got.Entries[0].Fingerprint == "" {
t.Errorf("hand take-in said branch %q source %d entries %+v: want main, a source revision and avahi at its commit", got.Branch, got.Source, got.Entries)
} else if newest, any, _ := newestTrunk(ctx, b.ix.Revisions); !any || got.Revision != newest.Revision {
t.Errorf("hand take-in said revision %d, want the revision it made, %d (A-448)", got.Revision, newest.Revision)
@@ -43,15 +43,13 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T
return []stdio.Tool{
{Name: PendingTool,
Description: "What waits for a take-in (ADR 0633): merges the indexer holds in its pending state instead of taking them in. " +
"A revision (what revision, its key its oldest merge's m.<sequence>) waits for the build of the indexer it names " +
"(waits-for): the running indexer wrote only what building and moving that indexer needs, and the new indexer takes " +
"it in at its start. A group (what group, key g.<group>) is a delivery group whose members are gathered until the " +
"A revision (what revision, its key its oldest merge's m.<sequence>) waits only while its message is unread or lost or the " +
"objects are full. A group (what group, key g.<group>) is a delivery group whose members are gathered until the " +
"forge bridge says its end; a merge in one of its members' repositories waits behind it (behind). Each names its " +
"merges, since when it waits, the modules it holds back (holds), the merges whose message the forge bridge's stream no " +
"longer holds (lost: they are not taken in; indexer_drop_pending drops them) and the last take-in event said of it. Empty waits: " +
"every merge is taken in. The watchman raises take-in-waits past 60 minutes for a revision, 30 for a group; " +
"indexer_take_in_pending and indexer_drop_pending steer a wait. own-build says which build the indexer knows it runs and own-build-since since when; own-build-unknown, when set, says the indexer cannot learn the build it " +
"runs (since when): it fixes it once at its start, so every merge waits (a take-in.pending that says so) until the module is started again.",
"indexer_take_in_pending and indexer_drop_pending steer a wait. own-build says which build the indexer knows it runs and own-build-since since when; it only names the writer, and no merge waits for it.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) {
var unread []string
@@ -62,7 +60,7 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T
return nil, err
}
answer := takein.PendingAnswer{Waits: PendingWaits(entries), Unread: unread}
// Whether this indexer knows the build it runs: when it does not, every merge waits, and this says since when and why.
// The build this indexer knows it runs, and since when.
if ownBuild != nil {
if v, _, found, err := ownBuild.Get(context.Background(), OwnBuildKey); err == nil && found {
var o OwnBuild
@@ -70,8 +68,6 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T
answer.OwnBuildSince = o.Since
if o.Known {
answer.OwnBuild = o.FP
} else {
answer.OwnBuildUnknown = o.Why
}
}
}
@@ -79,12 +75,12 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T
return answer, nil
}},
{Name: TakeInPendingTool,
Description: "Take a wait in by hand with the running indexer and its judge (ADR 0633 §5): the revision under its m. key, " +
"whatever build of the indexer it names, or a delivery group under its g. key with the members gathered so far. For " +
"when the named indexer can never run, or a group's end never comes. A hand act: who and why are said in the " +
"indexer's journal and on the take-in.taken event, which clears take-in-waits. indexer_pending names the keys.",
Description: "Take a delivery group in by hand with the running indexer and its judge (ADR 0633 §5): the group under its g. key " +
"with the members gathered so far, for when its end never comes. Only a group is taken in by hand: a revision is taken in " +
"by the indexer itself. A hand act: who and why are said in the indexer's journal and on the take-in.taken event, which " +
"clears take-in-waits. indexer_pending names the keys.",
Input: map[string]any{"type": "object", "required": []string{"key", "why"}, "properties": map[string]any{
"key": map[string]any{"type": "string", "description": "the wait's key, m.<sequence> or g.<group>, as indexer_pending names it"},
"key": map[string]any{"type": "string", "description": "the group's key, g.<group>, as indexer_pending names it"},
"why": map[string]any{"type": "string", "description": "why it is taken in by hand, in plain words"}}},
Run: hand(false)},
{Name: DropPendingTool,
@@ -169,11 +169,11 @@ func NewMergeAsk(repository, merge string, sequence uint64, why, by string, at t
// drop one pending merge's; abandon drops one no later merge covers.
func NewPendingAsk(key, why, by string, drop, abandon bool, at time.Time) (string, []byte, error) {
key = strings.TrimSpace(key)
if !strings.HasPrefix(key, "m.") && !(strings.HasPrefix(key, "g.") && !drop) {
if !(strings.HasPrefix(key, "m.") && drop) && !(strings.HasPrefix(key, "g.") && !drop) {
if drop {
return "", nil, errors.New("name the pending merge to drop by its key, m.<sequence>, as indexer_pending names it")
}
return "", nil, errors.New("name the wait to take in by its key, m.<sequence> or g.<group>, as indexer_pending names it")
return "", nil, errors.New("name the group to take in by its key, g.<group>, as indexer_pending names it")
}
if strings.TrimSpace(why) == "" {
return "", nil, errors.New("say why (why): a hand act on what waits is said in the journal and on its take-in event")
@@ -301,6 +301,14 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc
r.tcs, r.pkgs = toolchainsUsed(toolchains), f.versions
}
}
if _, merged := overlay[module]; !merged && repository != "" && !trunk.SameRepository(k.cur.Repository, repository) &&
maps.Equal(k.held.Packages, map[string]string(r.pkgs)) {
// A module registered for another repository is moved by this repository's commit only when it imports a package the
// commit moved (A-476a, ADR 0593 §3); any other stands as held (A-476 c), and so it is for a base of this repository's.
r = result{fp: k.held.Fp2, none: k.held.None2, tcs: k.held.Toolchains, pkgs: sdkstep.PackageVersions(k.held.Packages)}
done[module] = r
return r
}
done[module] = r
k.cur.Inputs2 = &kept
return r
@@ -310,14 +318,6 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc
for _, key := range slices.Sorted(maps.Keys(all)) {
k := all[key]
r := of(key, nil)
if _, merged := overlay[key]; !merged && repository != "" && !trunk.SameRepository(k.cur.Repository, repository) &&
maps.Equal(k.held.Packages, map[string]string(r.pkgs)) {
// A module registered for another repository is moved by this repository's commit only when it imports a package the
// commit moved (A-476a, ADR 0593 §3); any other stands as held (A-476 c).
r = result{fp: k.held.Fp2, none: k.held.None2, tcs: k.held.Toolchains, pkgs: sdkstep.PackageVersions(k.held.Packages)}
edges[key] = graphEdges{module: key, cur: k.cur, manifest: k.manifest, fp: r.fp, none: r.none, tcs: r.tcs, pkgs: r.pkgs}
continue
}
edges[key] = graphEdges{module: key, cur: k.cur, manifest: k.manifest, fp: r.fp, none: r.none, tcs: r.tcs, pkgs: r.pkgs}
if _, merged := overlay[key]; merged {
final := overlay[key]
@@ -843,12 +843,24 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
}
return a
}
// A-476 c: mesh-tools, of another repository than mesh-sdk, holds a fingerprint a recompute would move (here a stale one); a
// commit of mesh-sdk, which moved nothing it imports, leaves it as held, and only a full recompute moves it.
stale := tools
stale.Fp2 = "src2:" + strings.Repeat("5", 64)
mods.put("mesh-tools", mustEncode(t, stale))
tools = stale
first := sdkMerge(strings.Repeat("1", 40), "tree-sdk-1")
second := sdkMerge(strings.Repeat("2", 40), "tree-sdk-2")
if second.Fp2 == first.Fp2 {
t.Errorf("a mesh-sdk trunk merge kept avahi's fingerprint %q", first.Fp2)
}
n = second
if err := ix.Recompute(context.Background()); err != nil {
t.Fatal(err)
}
if o, _ := moduleOf(t, mods, "mesh-tools"); o.Fp2 == stale.Fp2 {
t.Errorf("a full recompute left mesh-tools at %q: the check above proves nothing", o.Fp2)
}
n, _ = moduleOf(t, mods, "avahi") // moved by the recompute too: its base mesh-tools moved
// Nodes that disagree keep the fingerprint already computed, with its toolchains (issue 1940, 2026-10-25 20:56: a
// stale report blanked every module and the controller's take-in refused modules already built); only a new
// fingerprint needs the agreement: a merge that moves the module's tree gives it none, naming the disagreement.
@@ -47,7 +47,7 @@ func (ix *Indexer) unreadFindings(ctx context.Context, entries []PendingEntry) {
"A call whose answer is kept is not answered",
fmt.Sprintf("The call %s is made again on every transport fault and on the callee's event; it has not been answered since %s. Its last fault: %s.",
UnreadCallee, since.UTC().Format(time.RFC3339), e.Value.Unread), "indexer."+mcc.KindCallUnanswered)
f.Needs = TakeInPendingTool + " reads the merge again now; " + PendingTool + " shows it unread."
f.Needs = "The merge is read again on the next message heard on the forge; " + PendingTool + " shows it unread."
f.Evidence = []mcseat.Evidence{{At: since.UTC(), Said: e.Value.Unread}}
if err := ix.Findings.Write(ctx, f); err != nil {
ix.logf("take-in: the finding %s is not written: %v", views.FindingKey(f.Scope, f.ID, f.Kind), err)
@@ -506,8 +506,8 @@ func (ix *Indexer) writeVerdict(ctx context.Context, digest string, v meshcatalo
}
// OwnBuildState is the indexer's state of the build it runs (the bucket mesh-delivery-indexer_own-build, one key, OwnBuildKey):
// written once, at the process's start by FixOwnBuild, so indexer_pending says whether this process knows its build and, when it
// does not, since when and why.
// written once, at the process's start by FixOwnBuild, so indexer_pending says which build this process knows it runs, and since
// when. The build only names the writer of what is written (A-476 b).
const (
OwnBuildStateName = "own-build"
OwnBuildKey = "state"
@@ -526,8 +526,7 @@ type OwnBuild struct {
// seat (OwnBuild; never the trunk's value of the indexer, which names a build that may be merged and not delivered), before
// this process hears a merge. It is never asked again: while the node-engine replaces the indexer the seat already names the
// successor, and an old process that learnt it later would take revisions in with its old judge. Where the start's ask is not
// answered the build stays unknown for the life of this process: every merge waits, loudly (a take-in.pending that says so, the
// own-build state, indexer_pending), and the module is started again to learn it.
// answered the build stays unknown for the life of this process, said in the own-build state and the journal; nothing waits for it.
func (ix *Indexer) FixOwnBuild(ctx context.Context) {
ix.ownMu.Lock()
if ix.ownFixed || ix.OwnBuild == nil {
@@ -543,7 +542,7 @@ func (ix *Indexer) FixOwnBuild(ctx context.Context) {
st := OwnBuild{V: 1, Since: ix.now().UTC()}
if err != nil {
st.Why = err.Error()
ix.logf("judge: the build this program runs is not known (%v), so every merge waits and a manifest with a field it does not know waits and is not refused; it is learnt only at the start, so the indexer is started again", err)
ix.logf("judge: the build this program runs is not known (%v); merges are taken in all the same, and it is learnt only at the start", err)
} else {
st.Known, st.FP = true, fp
ix.logf("judge: this program runs the indexer build %s, fixed here at its start", fp)
@@ -179,8 +179,6 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
// ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a
// commit is in another (issue 1900).
if want := []string{"seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps",
"seat:node-build-go.accept.check", // design 47 G A-231: the validation's run ask (§712) on the check queue
"seat:node-build-go.output", // design 47 G A-254: a validation's answer kept as an object
"seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) {
t.Errorf("invokes = %v, want %v", m.Invokes, want)
}
@@ -178,8 +178,6 @@
"seat:mesh-hand-acts.hand-act",
"seat:mesh-watchdogs.presence",
"seat:node-build-go.steps",
"seat:node-build-go.accept.check",
"seat:node-build-go.output",
"seat:node-build-node.steps",
"seat:node-build-image.steps",
"mesh-delivery-builder.builder_versions"
+11 -20
View File
@@ -1,5 +1,5 @@
// Package takein is the indexer's take-in contract (novox/hq ADR 0633 §1, §5): the events it publishes on its own stream
// `take-in` while a source revision waits for the indexer it names or a delivery group is gathered, their keys, and the
// `take-in` while a source revision waits (its message unread, lost or held for room) or a delivery group is gathered, their keys, and the
// answer of its tool indexer_pending. The indexer writes them; the watchman and the view read them. It is the indexer's
// own package: one module owns these types, so they live with it and not in mesh-sdk (the operator, 2026-11-04).
package takein
@@ -118,7 +118,7 @@ func checkTakeInKey(key string) error {
}
// TakeIn is the body of a take-in event. Kind is the event's kind and Key its key. What says what waits: a revision
// (Key its oldest merge's PendingKey, WaitsFor the indexer build it names) or a group being gathered (Key its
// (Key its oldest merge's PendingKey) or a group being gathered (Key its
// GroupTakeInKey, Members every approved member with its head, Merges the members held so far). Merges are the merges
// it holds, oldest first; Since is when the wait began. A `taken` event names the same once the wait ended; By and Why
// name a hand act (a hand take-in, or a drop, Dropped), and are carried by nothing else.
@@ -138,16 +138,9 @@ type TakeIn struct {
// Lost are the keys of merges whose message the forge bridge's stream no longer holds: they are not taken in, and the
// revision waits until indexer_drop_pending drops them (issue 1585).
Lost []string `json:"lost,omitempty"`
// Refused is why the manifest of the indexer the revision names was refused when the revision was worked out: with
// the seat definitions the revision carries (ADR 0633 §1), the manifest validator's problems. The revision waits for
// an indexer whose manifest is accepted, and says why here, as it says what it lost.
// Refused is why the revision is refused: the cycle with a build edge its catalog graph would hold (design 47 G §49). The
// revision waits, and says why here, as it says what it lost.
Refused string `json:"refused,omitempty"`
// Failed is why the revision's validation could not validate (design 47 G §713): the build seat's words, never a refusal
// and never passed; it is asked again by a later merge of the branch and by indexer_validate_again.
Failed string `json:"failed,omitempty"`
// Unknown is why the indexer holds a merge it does not take in although no indexer build is named: it cannot learn the build
// it runs itself (ADR 0633 §1), so Since is when it began not to know. A pending revision names WaitsFor or Unknown.
Unknown string `json:"own-build-unknown,omitempty"`
// Branch, Source and Entries are what a take-in took in (design 47 G §476, §493, §648): the watched branch, the source
// revision it makes of that branch, and one entry per module it wrote. Every taken event of one take-in carries the same;
// a taken event that ends a wait without a take-in (a group's end, a drop) carries none and starts no pipeline.
@@ -184,8 +177,8 @@ func (t TakeIn) check() error {
if !strings.HasPrefix(t.Key, "m.") {
return fmt.Errorf("delivery: a revision's take-in key is its oldest merge's m.<sequence>, not %q", t.Key)
}
if t.Kind == TakeInPending && t.WaitsFor == "" && t.Unknown == "" && len(t.Lost) == 0 && t.Refused == "" && t.Failed == "" {
return fmt.Errorf("delivery: a pending revision %s names the indexer build it waits for, why the indexer does not know its own, the merges it lost, its refusal or why it could not validate", t.Key)
if t.Kind == TakeInPending && t.WaitsFor == "" && len(t.Lost) == 0 && t.Refused == "" {
return fmt.Errorf("delivery: a pending revision %s names the indexer build it waits for, the merges it lost or its refusal", t.Key)
}
case TakeInGroup:
if t.Group == "" {
@@ -270,13 +263,11 @@ type PendingWait struct {
// PendingAnswer is indexer_pending's answer: every wait, oldest first, and the pending keys that do not read.
type PendingAnswer struct {
Waits []PendingWait `json:"waits"`
Unread []string `json:"unread,omitempty"`
// OwnBuildUnknown is why the running indexer does not know its own build, and since when: while it is set, every merge waits
// (a take-in.pending with the same words) and the indexer is started again to learn it. Empty when the build is known.
OwnBuildUnknown string `json:"own-build-unknown,omitempty"`
OwnBuildSince time.Time `json:"own-build-since,omitzero"`
// OwnBuild is the build the running indexer knows it runs, fixed at its start; OwnBuildSince is since when. Empty when unknown.
Waits []PendingWait `json:"waits"`
Unread []string `json:"unread,omitempty"`
OwnBuildSince time.Time `json:"own-build-since,omitzero"`
// OwnBuild is the build the running indexer knows it runs, fixed at its start; OwnBuildSince is since when. Empty when unknown:
// it names only the writer of what the indexer writes, and nothing waits for it.
OwnBuild string `json:"own-build,omitempty"`
}
@@ -21,10 +21,7 @@ func sampleTakeIns(since time.Time) []TakeIn {
Members: []meshdeliveryclient.Head{{Repository: "novox/mesh-delivery", Number: 9, Head: "ffff"}, {Repository: "novox/mesh-sdk", Number: 37, Head: "dddd"}}, Since: since},
{Kind: TakeInTaken, Key: PendingKey(12), What: TakeInRevision, Merges: m, WaitsFor: "src2:abc", Since: since,
By: "lead-47@shanks", Why: "the named indexer can never run"},
{Kind: TakeInPending, Key: PendingKey(13), What: TakeInRevision, Merges: m, Unknown: "the engine does not answer", Since: since}, // the indexer cannot learn its own build
// design 47 G §713: a revision refused by its validation, and one that could not validate.
{Kind: TakeInPending, Key: PendingKey(14), What: TakeInRevision, Merges: m, Refused: "a: names no version", Since: since},
{Kind: TakeInPending, Key: PendingKey(15), What: TakeInRevision, Merges: m, Failed: "could not run: no build node", Since: since},
{Kind: TakeInPending, Key: PendingKey(14), What: TakeInRevision, Merges: m, Refused: "a cycle with a build edge", Since: since},
}
}