diff --git a/modules/mesh-delivery-indexer/internal/index/behind.go b/modules/mesh-delivery-indexer/internal/index/behind.go index 81c02951..d7906ae9 100644 --- a/modules/mesh-delivery-indexer/internal/index/behind.go +++ b/modules/mesh-delivery-indexer/internal/index/behind.go @@ -91,7 +91,7 @@ func (ix *Indexer) TakeInBehind(ctx context.Context) []string { } switch { case held[head]: - // A head the pending holds (a member waiting for its group's end or its build) is taken in by that path, whole with + // A head the pending holds (a member waiting for its group's end) is taken in by that path, whole with // its group: it is not behind (A-463a). ix.clearUntaken(ctx, key, "the head of "+repository+" waits in the indexer's pending") case merge == head: diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index a5bc1f15..94b4697a 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -24,9 +24,8 @@ import ( "mesh-delivery-indexer/judge" ) -// The take-in step (novox/hq ADR 0633): a source revision is taken in only by the indexer it names. A merged message is -// kept in the indexer's own state pending, one key per merge, until the take-in that ends it; a revision naming another -// build of the indexer writes only what building and moving that indexer needs, and the rest stays pending for it. +// The take-in step (novox/hq ADR 0633, A-476): a merged message is kept in the indexer's own state pending, one key per +// merge, until the take-in that ends it; a revision is taken in by the running indexer whatever build of the indexer it names. const ( // PendingState is the indexer's state of held merges: the bucket mesh-delivery-indexer_pending, keyed // takein.PendingKey, so the key order is the merge order. An old `state` entry (the indexer is foundation, ADR @@ -45,10 +44,6 @@ var errClaimLost = errors.New("another take-in claimed the merges first, so this // runs such a build, so the revision waits until a later merge gives it one or a hand take-in (indexer_take_in_pending). const NoBuild = "no build: " -// ErrOwnBuildUnknown is what a merged message answers while the indexer cannot learn the build it runs (ADR 0633 §1): -// the message is left unacknowledged, so the bus delivers it again, and nothing is written. -var ErrOwnBuildUnknown = errors.New("the build of the indexer this program runs is not known") - // PendingRef is what a pending value keeps of its merged message instead of a copy (a merge of a repository with many // modules is about 0.5 MB, over the state's 256 KiB value cap; issue 1585): the small fields the waits are worked out // from. The message itself is read from the forge bridge's stream by Pending.Seq when a take-in needs it. @@ -384,26 +379,13 @@ func (ix *Indexer) pendingOnBus(ctx context.Context) bool { } // ownBuildNow is the build this program runs, as FixOwnBuild fixed it at the start: empty while unknown, and never asked again. +// It names only the writer of what is written (LastWriter); nothing waits for it (A-476 b). func (ix *Indexer) ownBuildNow(context.Context) string { ix.ownMu.Lock() defer ix.ownMu.Unlock() return ix.ownFP } -// sayOwnBuildUnknown says, for a merge this process cannot take in because it does not know its own build (ADR 0633 §1), a -// take-in.pending that names why and since when, so the watchman's take-in-waits bound covers it. The merge is not written to -// pending; the stream delivers it again to the process that knows its build. -func (ix *Indexer) sayOwnBuildUnknown(ctx context.Context, seq uint64, f forgemsg.Forge) { - ix.ownMu.Lock() - since, why := ix.ownSince, ix.ownWhy - ix.ownMu.Unlock() - if since.IsZero() { // never fixed: the start did not run (no engine wired) - since, why = ix.now(), "the build this program runs was never asked" - } - ix.sayTakeIn(ctx, takein.TakeIn{Kind: takein.TakeInPending, Key: takein.PendingKey(seq), What: takein.TakeInRevision, - Merges: []takein.Merge{{Repository: f.Repository, Number: f.Number, Merge: f.Merge}}, Since: since, Unknown: why}) -} - // sayTakeIn publishes a take-in event and keeps what it said on the entries, so indexer_pending reads the last event of // each wait from the state. A publish that fails is said in the log: the next evaluation says it again. func (ix *Indexer) sayTakeIn(ctx context.Context, t takein.TakeIn) bool { @@ -451,10 +433,6 @@ func (ix *Indexer) Arrive(ctx context.Context, seq uint64, f forgemsg.Forge) (Ap return a, "", err } own := ix.ownBuildNow(ctx) - if own == "" { - ix.sayOwnBuildUnknown(ctx, seq, f) - return Applied{}, "", ErrOwnBuildUnknown - } ix.pendingMu.Lock() defer ix.pendingMu.Unlock() key := takein.PendingKey(seq) @@ -497,9 +475,6 @@ func (ix *Indexer) EndGroup(ctx context.Context, end forgemsg.ForgeGroupEnd) (Ap return Applied{}, "no pending state: the group's merges were taken in one by one", nil } own := ix.ownBuildNow(ctx) - if own == "" { - return Applied{}, "", ErrOwnBuildUnknown - } ix.pendingMu.Lock() defer ix.pendingMu.Unlock() entries, err := ReadPending(ctx, ix.Pending, ix.logf) @@ -672,21 +647,15 @@ func (ix *Indexer) takenAlready(ctx context.Context, m forgemsg.GroupMerge) (boo } // TakeInAtStart is the take-in at the start (ADR 0633 §2), before the consumer on the forge bridge's stream starts: -// it evaluates pending once with the build this program runs, taking in the whole revisions that name this build; a -// group still being gathered stays, and a revision naming yet another build is held for that one. A claim left by an -// indexer that ran before is void here. When the build is not known (the node-engine did not answer), it waits for -// nothing: it says so and the consumer starts all the same; a merged message is then left unacknowledged until the -// build is known (clause 1), and its redelivery, or a toolchain report (ToolchainReported), asks again. No timer. +// it evaluates pending once, taking in the whole revisions whatever build of the indexer they name (A-476 b); a group +// still being gathered stays. A claim left by an indexer that ran before is void here. The build this program runs, known +// or not, only names the writer. No timer. func (ix *Indexer) TakeInAtStart(ctx context.Context) (Applied, string, error) { if ix.Pending == nil || !ix.pendingOnBus(ctx) { // Nothing can wait in a state not on the bus: the start asks no build and does not wait for one. return Applied{}, "the pending state is not on the bus yet: nothing waits", nil } own := ix.ownBuildNow(ctx) - if own == "" { - ix.logf("take-in: the build this program runs is not known at its start, so nothing pending is taken in now; the forge consumer starts, and a merged message left unacknowledged asks it again") - return Applied{}, "the build this program runs is not known yet: nothing pending is taken in at the start", nil - } ix.pendingMu.Lock() defer ix.pendingMu.Unlock() a, words, err := ix.evaluate(ctx, own, true) @@ -694,8 +663,8 @@ func (ix *Indexer) TakeInAtStart(ctx context.Context) (Applied, string, error) { return a, "the pending state is not on the bus yet: nothing waits", nil } if err == nil && len(a.Written) == 0 { - // A start that takes nothing in says so, with the build it learnt (the wait's own build is in the line holdFor says). - ix.logf("take-in: at the start, this program runs build %s and took nothing in: %s", own, words) + // A start that takes nothing in says so. + ix.logf("take-in: at the start, this program (build %q) took nothing in: %s", own, words) } return a, words, err } @@ -726,16 +695,12 @@ func (ix *Indexer) UnreadAgain(ctx context.Context) error { } // PendingAgain evaluates pending again after something a fingerprint is computed from changed, a toolchain report -// (ADR 0633 §1, §3), or the verdicts state bound (issue 1966): a revision that now names the running build is taken in, and one that names another build is held -// for that one. It does nothing while the pending state is not on the bus or the build this program runs is not known. +// (ADR 0633 §3), or the verdicts state bound (issue 1966): what waits is taken in. It does nothing while the pending state is not on the bus. func (ix *Indexer) PendingAgain(ctx context.Context) error { if ix.Pending == nil || !ix.pendingOnBus(ctx) { return nil } own := ix.ownBuildNow(ctx) - if own == "" { - return nil - } ix.pendingMu.Lock() defer ix.pendingMu.Unlock() _, words, err := ix.evaluate(ctx, own, false) @@ -881,12 +846,12 @@ func (ix *Indexer) holdLost(ctx context.Context, ready, lost []PendingEntry, rea } for _, g := range lost { if unread[g.Key] { - ix.logf("take-in: %s (%s#%d) is not taken in and holds its revision, its message is unread for a transport fault: %s; it is read again on the next message heard on the forge, at the start, and by %s", - g.Key, g.Forge.Repository, g.Forge.Number, reasons[g.Key], TakeInPendingTool) + ix.logf("take-in: %s (%s#%d) is not taken in and holds its revision, its message is unread for a transport fault: %s; it is read again on the next message heard on the forge and at the start", + g.Key, g.Forge.Repository, g.Forge.Number, reasons[g.Key]) continue } - ix.logf("take-in: %s (%s#%d) is not taken in and holds its revision, the forge bridge could not give its message: %s; %s reads it again; indexer_drop_pending drops it once a later merge of %s covers it", - g.Key, g.Forge.Repository, g.Forge.Number, reasons[g.Key], TakeInPendingTool, g.Forge.Repository) + ix.logf("take-in: %s (%s#%d) is not taken in and holds its revision, the forge bridge could not give its message: %s; indexer_drop_pending drops it once a later merge of %s covers it", + g.Key, g.Forge.Repository, g.Forge.Number, reasons[g.Key], g.Forge.Repository) } if !ix.sayTakeIn(ctx, t) { return nil @@ -1222,8 +1187,7 @@ func (ix *Indexer) nextSource(branch string) int64 { // takenWaits are the taken events of a take-in: every wait the entries were said under (saidWaits), every wait // indexer_pending names the claimed entries by now (WaitsOf: a revision by its oldest merge's key, a group by its g. key) and // the key of every claimed entry, since indexer_pending named it by that key earlier when it waited behind a group. So a wait -// the watchman learned of without the indexer marking it (seeded from indexer_pending, or said by the own-build-unknown -// path) ends too (design 47, Taken in by the indexer it names). No key is said twice. +// the watchman learned of without the indexer marking it (seeded from indexer_pending) ends too. No key is said twice. func takenWaits(entries []PendingEntry) []takein.TakeIn { out := saidWaits(entries) said := map[string]bool{} @@ -1354,9 +1318,9 @@ func waitOf(entries []PendingEntry, what, behind string) takein.PendingWait { return pw } -// TakeInPendingByHand is the hand take-in (indexer_take_in_pending, ADR 0633 §5): the wait named by its key, the -// ready revision (its m. key) or a group gathered so far (its g. key), is taken in with this indexer and its judge, -// whatever build it names, for when the named indexer can never run or the group's end never comes. by and why are +// TakeInPendingByHand is the hand take-in (indexer_take_in_pending, ADR 0633 §5, A-476 b): the group gathered so far, named by +// its g. key, is taken in with this indexer and its judge, +// whatever build it names, for when the group's end never comes. by and why are // said in the journal and on the take-in.taken event. What waited behind it is evaluated again after. func (ix *Indexer) TakeInPendingByHand(ctx context.Context, key, by, why string) TakenIn { r := TakenIn{Module: key} @@ -1381,8 +1345,9 @@ func (ix *Indexer) TakeInPendingByHand(ctx context.Context, key, by, why string) take = held } } - case len(w.Ready) > 0 && w.Ready[0].Key == key: - take = w.Ready + default: + r.Answer = fmt.Sprintf("%s names no delivery group: only a group is taken in by hand (a revision is taken in by the indexer itself); indexer_pending names the waits", key) + return r } if len(take) == 0 { r.Answer = fmt.Sprintf("nothing waits under %s: indexer_pending names the waits", key) @@ -1460,8 +1425,8 @@ func (ix *Indexer) DropPending(ctx context.Context, key, by, why string, abandon } if !covered { r.Answer = fmt.Sprintf("%s is not dropped: no later merge of %s, pending or taken in, has %s as an ancestor, so its "+ - "changes would never be taken in (ADR 0633 §5); %s reads its message again, or abandon: true drops it all the same", - key, e.Forge.Repository, e.Forge.Merge, TakeInPendingTool) + "changes would never be taken in (ADR 0633 §5); abandon: true drops it all the same", + key, e.Forge.Repository, e.Forge.Merge) return r } } diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index 6f22c27a..950b0fbf 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -186,53 +186,21 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) }) } }) - t.Run("T6 a merge while the own build is unknown writes nothing", func(t *testing.T) { + t.Run("T6 a merge while the own build is unknown is taken in all the same (A-476 b)", func(t *testing.T) { b := newTakeInBed(t) b.ix.ownFP, b.ix.ownFixed = "", false // this process is started again and asks the engine, which does not answer b.ix.OwnBuild = func(context.Context) (string, error) { return "", errors.New("the engine does not answer") } b.ix.FixOwnBuild(ctx) b.said = nil - _, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", "")) - if !errors.Is(err, ErrOwnBuildUnknown) || len(b.keys()) != 0 || b.commit(t, "avahi") == sha('b') { - t.Errorf("err %v, pending %v, avahi at %s: want ErrOwnBuildUnknown and nothing written", err, b.keys(), b.commit(t, "avahi")) + if _, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", "tree-indexer-2")); err != nil || len(b.keys()) != 0 || b.commit(t, "avahi") != sha('b') { + t.Errorf("err %v, pending %v, avahi at %s: want the merge taken in at once", err, b.keys(), b.commit(t, "avahi")) } - // The merge is said pending with why and since when (so the watchman's bound covers it), and indexer_pending says the - // build is unknown, from the own-build state. - if len(b.said) != 1 || b.said[0].Kind != takein.TakeInPending || b.said[0].Key != takein.PendingKey(1) || !strings.Contains(b.said[0].Unknown, "does not answer") || - b.said[0].Since.IsZero() || b.said[0].WaitsFor != "" { - t.Errorf("said %+v, want one take-in.pending of m.1 saying why the own build is unknown", b.said) - } else if _, err := takein.EncodeTakeIn(b.said[0]); err != nil { - t.Errorf("the unknown-build event is refused by its own contract: %v", err) + if len(b.said) != 1 || b.said[0].Kind != takein.TakeInTaken { + t.Errorf("said %+v, want one take-in.taken and no wait said", b.said) } answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{}) - if a, _ := answer.(takein.PendingAnswer); err != nil || !strings.Contains(a.OwnBuildUnknown, "does not answer") || a.OwnBuildSince.IsZero() { - t.Errorf("indexer_pending = %+v %v, want the build unknown with why and since", answer, err) - } - // Before the pending state is on the bus nothing waits: the start asks no build, and the merge is taken in at once, - // as before ADR 0633, though the build is unknown (TestIndexerWhole found the start waiting on it for ever). - b.ix.Pending = &memStore{unbound: process.ErrNotYet} - if _, _, err := b.ix.TakeInAtStart(ctx); err != nil { - t.Fatal(err) - } - if _, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", "")); err != nil || b.commit(t, "avahi") != sha('b') { - t.Errorf("no pending state: err %v, avahi at %s: want the merge taken in at once", err, b.commit(t, "avahi")) - } - }) - t.Run("1966/AC195-2 AC195-3 a merge said pending with its build unknown and taken in on redelivery ends that wait under its key; a revision wait is said once", func(t *testing.T) { - b := newTakeInBed(t) - ownBuild := b.ix.OwnBuild - b.ix.ownFP, b.ix.ownFixed = "", false - b.ix.OwnBuild = func(context.Context) (string, error) { return "", errors.New("the engine does not answer") } - b.ix.FixOwnBuild(ctx) - b.said = nil - if _, _, err := b.ix.Arrive(ctx, 7, b.merge(30, sha('b'), "tree-avahi-2", "")); !errors.Is(err, ErrOwnBuildUnknown) { - t.Fatal(err) - } - b.ix.OwnBuild, b.ix.ownFP, b.ix.ownFixed = ownBuild, "", false - b.ix.FixOwnBuild(ctx) - b.arrive(t, 7, b.merge(30, sha('b'), "tree-avahi-2", "")) // the stream delivers it again - if want := []string{"pending " + takein.PendingKey(7), "taken " + takein.PendingKey(7)}; !slices.Equal(b.events(), want) || len(b.keys()) != 0 { - t.Fatalf("said %v, pending %v: want %v and nothing pending", b.events(), b.keys(), want) + if a, _ := answer.(takein.PendingAnswer); err != nil || a.OwnBuild != "" || a.OwnBuildSince.IsZero() || len(a.Waits) != 0 { + t.Errorf("indexer_pending = %+v %v, want no wait and no known build", answer, err) } }) t.Run("T7 and T3 a merge alone, an indexer change, a merge behind it and a revert", func(t *testing.T) { @@ -371,7 +339,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) t.Errorf("indexer_pending %+v, want the refusal naming the cycle", waits) } }) - t.Run("T10 a start takes what waits in once its build is known; a gathered group stays", func(t *testing.T) { + t.Run("T10 a start takes what waits in whatever its build; a gathered group stays", func(t *testing.T) { b := newTakeInBed(t) b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) g := b.merge(30, sha('f'), "", "") @@ -390,30 +358,21 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) n.FixOwnBuild(ctx) return n } - asked := 0 - next := mk(func(context.Context) (string, error) { asked++; return "", errors.New("not yet") }) - if r := next.TakeInPendingByHand(ctx, "m.nothing", "lead@shanks", "a test"); r.Stored || !strings.Contains(r.Answer, "nothing waits") { - t.Errorf("a hand ask while the own build is unknown: %+v", r) + // The build this program runs is not known (the engine does not answer at its start): nothing waits for it, so the start + // takes the revision in all the same (A-476 b), and a hand ask names only groups. + next := mk(func(context.Context) (string, error) { return "", errors.New("not yet") }) + if r := next.TakeInPendingByHand(ctx, "m.nothing", "lead@shanks", "a test"); r.Stored || !strings.Contains(r.Answer, "names no delivery group") { + t.Errorf("a hand ask for a revision: %+v", r) } - if _, words, err := next.TakeInAtStart(ctx); err != nil || len(b.keys()) != 2 || !strings.Contains(words, "not known") { - t.Fatalf("a start with its build unknown: %q %v, pending %v: want it to answer at once and take nothing in", words, err, b.keys()) - } - // The engine answers now, and the same process does not ask it again: it learnt nothing it did not know at its start. - next.OwnBuild = func(context.Context) (string, error) { asked++; return own, nil } - if _, words, err := next.TakeInAtStart(ctx); err != nil || asked != 1 || len(b.keys()) != 2 || !strings.Contains(words, "not known") { - t.Fatalf("the window of a replacement: %q %v, asked %d, pending %v: want the build fixed at the start, never learnt later", words, err, asked, b.keys()) - } - if answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{}); err != nil || answer.(takein.PendingAnswer).OwnBuild != own || answer.(takein.PendingAnswer).OwnBuildSince.IsZero() { - t.Errorf("indexer_pending = %+v %v, want the known build and since", answer, err) - } - // The module started again fixes the build at its start and takes the revision in. - next = mk(func(context.Context) (string, error) { return own, nil }) if _, _, err := next.TakeInAtStart(ctx); err != nil { t.Fatal(err) } if !reflect.DeepEqual(b.keys(), []string{takein.PendingKey(5)}) || b.commit(t, "avahi") != sha('c') { t.Errorf("start: pending %v, avahi at %s: want the revision taken in, the group left", b.keys(), b.commit(t, "avahi")) } + if answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{}); err != nil || answer.(takein.PendingAnswer).OwnBuild != own { + t.Errorf("indexer_pending = %+v %v, want the build the original program knows", answer, err) + } }) t.Run("T8 T9 T11 a group gathered until its end, a merge behind it, one alone, an end again", func(t *testing.T) { b := newTakeInBed(t) @@ -886,25 +845,33 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) t.Fatalf("pending %v: want m.2 dropped and m.3 pending", b.keys()) } // Issue 1966 (#650), ADR 0633 §5: the newest pending merge of its repository, which no taken merge covers, is not - // dropped; the answer names the read again. abandon drops it all the same (below). + // dropped; the answer names abandon, which drops it all the same (below). if r := b.ix.DropPending(ctx, takein.PendingKey(3), "lead@shanks", "lost message", false); r.Stored || - !strings.Contains(r.Answer, TakeInPendingTool) || len(b.keys()) != 1 { + !strings.Contains(r.Answer, "abandon") || len(b.keys()) != 1 { t.Fatalf("a drop nothing covers: %+v, pending %v", r, b.keys()) } + if r := b.ix.DropPending(ctx, takein.PendingKey(3), "lead@shanks", "abandoned", true); !r.Stored || len(b.keys()) != 0 { + t.Fatalf("an abandoned drop: %+v, pending %v", r, b.keys()) + } + // A delivery group gathered is taken in by hand (A-476 b: only a group is). b.ix.Pins = newMem() b.ix.Revisions = newMemObjects(0) // A-448: the taken event names the catalog graph revision the take-in made + gm := b.merge(24, sha('f'), "tree-avahi-6", "") + gm.Group, gm.Head = "feat/h", sha('1') + gm.Members = []meshdeliveryclient.Head{{Repository: catalogueRepo, Number: 24, Head: sha('1')}, {Repository: "novox/mesh-sdk", Number: 25, Head: sha('2')}} + b.arrive(t, 5, gm) b.said = nil - if r := b.ix.TakeInPendingByHand(ctx, takein.PendingKey(3), "lead@shanks", "the named indexer can never run"); !r.Stored { + if r := b.ix.TakeInPendingByHand(ctx, takein.GroupTakeInKey("feat/h"), "lead@shanks", "the group's end never comes"); !r.Stored { t.Fatalf("hand take-in: %+v", r) } - if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('d') || len(b.said) != 1 || b.said[0].Kind != takein.TakeInTaken || b.said[0].By != "lead@shanks" { + if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('f') || len(b.said) < 1 || b.said[0].Kind != takein.TakeInTaken || b.said[0].By != "lead@shanks" { t.Errorf("hand take-in: pending %v, avahi at %s, said %+v", b.keys(), b.commit(t, "avahi"), b.said) } // §476, §493: a hand take-in's taken event carries the branch, the source revision it makes and its entries. - if len(b.said) == 1 { + if len(b.said) >= 1 { got := b.said[0] - if got.Branch != TrunkBranch || got.Source <= 0 || len(got.Entries) != 2 || got.Entries[1].Module != "avahi" || - got.Entries[1].Commit != sha('d') || got.Entries[1].Fingerprint == "" { + if got.Branch != TrunkBranch || got.Source <= 0 || len(got.Entries) != 1 || got.Entries[0].Module != "avahi" || + got.Entries[0].Commit != sha('f') || got.Entries[0].Fingerprint == "" { t.Errorf("hand take-in said branch %q source %d entries %+v: want main, a source revision and avahi at its commit", got.Branch, got.Source, got.Entries) } else if newest, any, _ := newestTrunk(ctx, b.ix.Revisions); !any || got.Revision != newest.Revision { t.Errorf("hand take-in said revision %d, want the revision it made, %d (A-448)", got.Revision, newest.Revision) diff --git a/modules/mesh-delivery-indexer/internal/index/pending_tools.go b/modules/mesh-delivery-indexer/internal/index/pending_tools.go index 366654eb..bec7be46 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_tools.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_tools.go @@ -43,15 +43,13 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T return []stdio.Tool{ {Name: PendingTool, Description: "What waits for a take-in (ADR 0633): merges the indexer holds in its pending state instead of taking them in. " + - "A revision (what revision, its key its oldest merge's m.) waits for the build of the indexer it names " + - "(waits-for): the running indexer wrote only what building and moving that indexer needs, and the new indexer takes " + - "it in at its start. A group (what group, key g.) is a delivery group whose members are gathered until the " + + "A revision (what revision, its key its oldest merge's m.) waits only while its message is unread or lost or the " + + "objects are full. A group (what group, key g.) is a delivery group whose members are gathered until the " + "forge bridge says its end; a merge in one of its members' repositories waits behind it (behind). Each names its " + "merges, since when it waits, the modules it holds back (holds), the merges whose message the forge bridge's stream no " + "longer holds (lost: they are not taken in; indexer_drop_pending drops them) and the last take-in event said of it. Empty waits: " + "every merge is taken in. The watchman raises take-in-waits past 60 minutes for a revision, 30 for a group; " + - "indexer_take_in_pending and indexer_drop_pending steer a wait. own-build says which build the indexer knows it runs and own-build-since since when; own-build-unknown, when set, says the indexer cannot learn the build it " + - "runs (since when): it fixes it once at its start, so every merge waits (a take-in.pending that says so) until the module is started again.", + "indexer_take_in_pending and indexer_drop_pending steer a wait. own-build says which build the indexer knows it runs and own-build-since since when; it only names the writer, and no merge waits for it.", Input: map[string]any{"type": "object", "properties": map[string]any{}}, Run: func(map[string]any) (any, error) { var unread []string @@ -62,7 +60,7 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T return nil, err } answer := takein.PendingAnswer{Waits: PendingWaits(entries), Unread: unread} - // Whether this indexer knows the build it runs: when it does not, every merge waits, and this says since when and why. + // The build this indexer knows it runs, and since when. if ownBuild != nil { if v, _, found, err := ownBuild.Get(context.Background(), OwnBuildKey); err == nil && found { var o OwnBuild @@ -70,8 +68,6 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T answer.OwnBuildSince = o.Since if o.Known { answer.OwnBuild = o.FP - } else { - answer.OwnBuildUnknown = o.Why } } } @@ -79,12 +75,12 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T return answer, nil }}, {Name: TakeInPendingTool, - Description: "Take a wait in by hand with the running indexer and its judge (ADR 0633 §5): the revision under its m. key, " + - "whatever build of the indexer it names, or a delivery group under its g. key with the members gathered so far. For " + - "when the named indexer can never run, or a group's end never comes. A hand act: who and why are said in the " + - "indexer's journal and on the take-in.taken event, which clears take-in-waits. indexer_pending names the keys.", + Description: "Take a delivery group in by hand with the running indexer and its judge (ADR 0633 §5): the group under its g. key " + + "with the members gathered so far, for when its end never comes. Only a group is taken in by hand: a revision is taken in " + + "by the indexer itself. A hand act: who and why are said in the indexer's journal and on the take-in.taken event, which " + + "clears take-in-waits. indexer_pending names the keys.", Input: map[string]any{"type": "object", "required": []string{"key", "why"}, "properties": map[string]any{ - "key": map[string]any{"type": "string", "description": "the wait's key, m. or g., as indexer_pending names it"}, + "key": map[string]any{"type": "string", "description": "the group's key, g., as indexer_pending names it"}, "why": map[string]any{"type": "string", "description": "why it is taken in by hand, in plain words"}}}, Run: hand(false)}, {Name: DropPendingTool, diff --git a/modules/mesh-delivery-indexer/internal/index/takein.go b/modules/mesh-delivery-indexer/internal/index/takein.go index ae21ccc0..5ea1004d 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein.go +++ b/modules/mesh-delivery-indexer/internal/index/takein.go @@ -169,11 +169,11 @@ func NewMergeAsk(repository, merge string, sequence uint64, why, by string, at t // drop one pending merge's; abandon drops one no later merge covers. func NewPendingAsk(key, why, by string, drop, abandon bool, at time.Time) (string, []byte, error) { key = strings.TrimSpace(key) - if !strings.HasPrefix(key, "m.") && !(strings.HasPrefix(key, "g.") && !drop) { + if !(strings.HasPrefix(key, "m.") && drop) && !(strings.HasPrefix(key, "g.") && !drop) { if drop { return "", nil, errors.New("name the pending merge to drop by its key, m., as indexer_pending names it") } - return "", nil, errors.New("name the wait to take in by its key, m. or g., as indexer_pending names it") + return "", nil, errors.New("name the group to take in by its key, g., as indexer_pending names it") } if strings.TrimSpace(why) == "" { return "", nil, errors.New("say why (why): a hand act on what waits is said in the journal and on its take-in event") diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index d500c088..c3c1927d 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -301,6 +301,14 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc r.tcs, r.pkgs = toolchainsUsed(toolchains), f.versions } } + if _, merged := overlay[module]; !merged && repository != "" && !trunk.SameRepository(k.cur.Repository, repository) && + maps.Equal(k.held.Packages, map[string]string(r.pkgs)) { + // A module registered for another repository is moved by this repository's commit only when it imports a package the + // commit moved (A-476a, ADR 0593 §3); any other stands as held (A-476 c), and so it is for a base of this repository's. + r = result{fp: k.held.Fp2, none: k.held.None2, tcs: k.held.Toolchains, pkgs: sdkstep.PackageVersions(k.held.Packages)} + done[module] = r + return r + } done[module] = r k.cur.Inputs2 = &kept return r @@ -310,14 +318,6 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc for _, key := range slices.Sorted(maps.Keys(all)) { k := all[key] r := of(key, nil) - if _, merged := overlay[key]; !merged && repository != "" && !trunk.SameRepository(k.cur.Repository, repository) && - maps.Equal(k.held.Packages, map[string]string(r.pkgs)) { - // A module registered for another repository is moved by this repository's commit only when it imports a package the - // commit moved (A-476a, ADR 0593 §3); any other stands as held (A-476 c). - r = result{fp: k.held.Fp2, none: k.held.None2, tcs: k.held.Toolchains, pkgs: sdkstep.PackageVersions(k.held.Packages)} - edges[key] = graphEdges{module: key, cur: k.cur, manifest: k.manifest, fp: r.fp, none: r.none, tcs: r.tcs, pkgs: r.pkgs} - continue - } edges[key] = graphEdges{module: key, cur: k.cur, manifest: k.manifest, fp: r.fp, none: r.none, tcs: r.tcs, pkgs: r.pkgs} if _, merged := overlay[key]; merged { final := overlay[key] diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index c8675341..24898914 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -843,12 +843,24 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) { } return a } + // A-476 c: mesh-tools, of another repository than mesh-sdk, holds a fingerprint a recompute would move (here a stale one); a + // commit of mesh-sdk, which moved nothing it imports, leaves it as held, and only a full recompute moves it. + stale := tools + stale.Fp2 = "src2:" + strings.Repeat("5", 64) + mods.put("mesh-tools", mustEncode(t, stale)) + tools = stale first := sdkMerge(strings.Repeat("1", 40), "tree-sdk-1") second := sdkMerge(strings.Repeat("2", 40), "tree-sdk-2") if second.Fp2 == first.Fp2 { t.Errorf("a mesh-sdk trunk merge kept avahi's fingerprint %q", first.Fp2) } - n = second + if err := ix.Recompute(context.Background()); err != nil { + t.Fatal(err) + } + if o, _ := moduleOf(t, mods, "mesh-tools"); o.Fp2 == stale.Fp2 { + t.Errorf("a full recompute left mesh-tools at %q: the check above proves nothing", o.Fp2) + } + n, _ = moduleOf(t, mods, "avahi") // moved by the recompute too: its base mesh-tools moved // Nodes that disagree keep the fingerprint already computed, with its toolchains (issue 1940, 2026-10-25 20:56: a // stale report blanked every module and the controller's take-in refused modules already built); only a new // fingerprint needs the agreement: a merge that moves the module's tree gives it none, naming the disagreement. diff --git a/modules/mesh-delivery-indexer/internal/index/unreadcall.go b/modules/mesh-delivery-indexer/internal/index/unreadcall.go index d81e49fb..7d02bd10 100644 --- a/modules/mesh-delivery-indexer/internal/index/unreadcall.go +++ b/modules/mesh-delivery-indexer/internal/index/unreadcall.go @@ -47,7 +47,7 @@ func (ix *Indexer) unreadFindings(ctx context.Context, entries []PendingEntry) { "A call whose answer is kept is not answered", fmt.Sprintf("The call %s is made again on every transport fault and on the callee's event; it has not been answered since %s. Its last fault: %s.", UnreadCallee, since.UTC().Format(time.RFC3339), e.Value.Unread), "indexer."+mcc.KindCallUnanswered) - f.Needs = TakeInPendingTool + " reads the merge again now; " + PendingTool + " shows it unread." + f.Needs = "The merge is read again on the next message heard on the forge; " + PendingTool + " shows it unread." f.Evidence = []mcseat.Evidence{{At: since.UTC(), Said: e.Value.Unread}} if err := ix.Findings.Write(ctx, f); err != nil { ix.logf("take-in: the finding %s is not written: %v", views.FindingKey(f.Scope, f.ID, f.Kind), err) diff --git a/modules/mesh-delivery-indexer/internal/index/verdicts.go b/modules/mesh-delivery-indexer/internal/index/verdicts.go index b8d0c6f0..526bb638 100644 --- a/modules/mesh-delivery-indexer/internal/index/verdicts.go +++ b/modules/mesh-delivery-indexer/internal/index/verdicts.go @@ -506,8 +506,8 @@ func (ix *Indexer) writeVerdict(ctx context.Context, digest string, v meshcatalo } // OwnBuildState is the indexer's state of the build it runs (the bucket mesh-delivery-indexer_own-build, one key, OwnBuildKey): -// written once, at the process's start by FixOwnBuild, so indexer_pending says whether this process knows its build and, when it -// does not, since when and why. +// written once, at the process's start by FixOwnBuild, so indexer_pending says which build this process knows it runs, and since +// when. The build only names the writer of what is written (A-476 b). const ( OwnBuildStateName = "own-build" OwnBuildKey = "state" @@ -526,8 +526,7 @@ type OwnBuild struct { // seat (OwnBuild; never the trunk's value of the indexer, which names a build that may be merged and not delivered), before // this process hears a merge. It is never asked again: while the node-engine replaces the indexer the seat already names the // successor, and an old process that learnt it later would take revisions in with its old judge. Where the start's ask is not -// answered the build stays unknown for the life of this process: every merge waits, loudly (a take-in.pending that says so, the -// own-build state, indexer_pending), and the module is started again to learn it. +// answered the build stays unknown for the life of this process, said in the own-build state and the journal; nothing waits for it. func (ix *Indexer) FixOwnBuild(ctx context.Context) { ix.ownMu.Lock() if ix.ownFixed || ix.OwnBuild == nil { @@ -543,7 +542,7 @@ func (ix *Indexer) FixOwnBuild(ctx context.Context) { st := OwnBuild{V: 1, Since: ix.now().UTC()} if err != nil { st.Why = err.Error() - ix.logf("judge: the build this program runs is not known (%v), so every merge waits and a manifest with a field it does not know waits and is not refused; it is learnt only at the start, so the indexer is started again", err) + ix.logf("judge: the build this program runs is not known (%v); merges are taken in all the same, and it is learnt only at the start", err) } else { st.Known, st.FP = true, fp ix.logf("judge: this program runs the indexer build %s, fixed here at its start", fp) diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index ff8aef65..6de6a6e4 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -179,8 +179,6 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a // commit is in another (issue 1900). if want := []string{"seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", - "seat:node-build-go.accept.check", // design 47 G A-231: the validation's run ask (§712) on the check queue - "seat:node-build-go.output", // design 47 G A-254: a validation's answer kept as an object "seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index e4c88a65..6c0a2a72 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -178,8 +178,6 @@ "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", - "seat:node-build-go.accept.check", - "seat:node-build-go.output", "seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions" diff --git a/modules/mesh-delivery-indexer/takein/takein.go b/modules/mesh-delivery-indexer/takein/takein.go index 8e5ba0d1..70b24218 100644 --- a/modules/mesh-delivery-indexer/takein/takein.go +++ b/modules/mesh-delivery-indexer/takein/takein.go @@ -1,5 +1,5 @@ // Package takein is the indexer's take-in contract (novox/hq ADR 0633 §1, §5): the events it publishes on its own stream -// `take-in` while a source revision waits for the indexer it names or a delivery group is gathered, their keys, and the +// `take-in` while a source revision waits (its message unread, lost or held for room) or a delivery group is gathered, their keys, and the // answer of its tool indexer_pending. The indexer writes them; the watchman and the view read them. It is the indexer's // own package: one module owns these types, so they live with it and not in mesh-sdk (the operator, 2026-11-04). package takein @@ -118,7 +118,7 @@ func checkTakeInKey(key string) error { } // TakeIn is the body of a take-in event. Kind is the event's kind and Key its key. What says what waits: a revision -// (Key its oldest merge's PendingKey, WaitsFor the indexer build it names) or a group being gathered (Key its +// (Key its oldest merge's PendingKey) or a group being gathered (Key its // GroupTakeInKey, Members every approved member with its head, Merges the members held so far). Merges are the merges // it holds, oldest first; Since is when the wait began. A `taken` event names the same once the wait ended; By and Why // name a hand act (a hand take-in, or a drop, Dropped), and are carried by nothing else. @@ -138,16 +138,9 @@ type TakeIn struct { // Lost are the keys of merges whose message the forge bridge's stream no longer holds: they are not taken in, and the // revision waits until indexer_drop_pending drops them (issue 1585). Lost []string `json:"lost,omitempty"` - // Refused is why the manifest of the indexer the revision names was refused when the revision was worked out: with - // the seat definitions the revision carries (ADR 0633 §1), the manifest validator's problems. The revision waits for - // an indexer whose manifest is accepted, and says why here, as it says what it lost. + // Refused is why the revision is refused: the cycle with a build edge its catalog graph would hold (design 47 G §49). The + // revision waits, and says why here, as it says what it lost. Refused string `json:"refused,omitempty"` - // Failed is why the revision's validation could not validate (design 47 G §713): the build seat's words, never a refusal - // and never passed; it is asked again by a later merge of the branch and by indexer_validate_again. - Failed string `json:"failed,omitempty"` - // Unknown is why the indexer holds a merge it does not take in although no indexer build is named: it cannot learn the build - // it runs itself (ADR 0633 §1), so Since is when it began not to know. A pending revision names WaitsFor or Unknown. - Unknown string `json:"own-build-unknown,omitempty"` // Branch, Source and Entries are what a take-in took in (design 47 G §476, §493, §648): the watched branch, the source // revision it makes of that branch, and one entry per module it wrote. Every taken event of one take-in carries the same; // a taken event that ends a wait without a take-in (a group's end, a drop) carries none and starts no pipeline. @@ -184,8 +177,8 @@ func (t TakeIn) check() error { if !strings.HasPrefix(t.Key, "m.") { return fmt.Errorf("delivery: a revision's take-in key is its oldest merge's m., not %q", t.Key) } - if t.Kind == TakeInPending && t.WaitsFor == "" && t.Unknown == "" && len(t.Lost) == 0 && t.Refused == "" && t.Failed == "" { - return fmt.Errorf("delivery: a pending revision %s names the indexer build it waits for, why the indexer does not know its own, the merges it lost, its refusal or why it could not validate", t.Key) + if t.Kind == TakeInPending && t.WaitsFor == "" && len(t.Lost) == 0 && t.Refused == "" { + return fmt.Errorf("delivery: a pending revision %s names the indexer build it waits for, the merges it lost or its refusal", t.Key) } case TakeInGroup: if t.Group == "" { @@ -270,13 +263,11 @@ type PendingWait struct { // PendingAnswer is indexer_pending's answer: every wait, oldest first, and the pending keys that do not read. type PendingAnswer struct { - Waits []PendingWait `json:"waits"` - Unread []string `json:"unread,omitempty"` - // OwnBuildUnknown is why the running indexer does not know its own build, and since when: while it is set, every merge waits - // (a take-in.pending with the same words) and the indexer is started again to learn it. Empty when the build is known. - OwnBuildUnknown string `json:"own-build-unknown,omitempty"` - OwnBuildSince time.Time `json:"own-build-since,omitzero"` - // OwnBuild is the build the running indexer knows it runs, fixed at its start; OwnBuildSince is since when. Empty when unknown. + Waits []PendingWait `json:"waits"` + Unread []string `json:"unread,omitempty"` + OwnBuildSince time.Time `json:"own-build-since,omitzero"` + // OwnBuild is the build the running indexer knows it runs, fixed at its start; OwnBuildSince is since when. Empty when unknown: + // it names only the writer of what the indexer writes, and nothing waits for it. OwnBuild string `json:"own-build,omitempty"` } diff --git a/modules/mesh-delivery-indexer/takein/takein_test.go b/modules/mesh-delivery-indexer/takein/takein_test.go index f69c1e49..a2376fe6 100644 --- a/modules/mesh-delivery-indexer/takein/takein_test.go +++ b/modules/mesh-delivery-indexer/takein/takein_test.go @@ -21,10 +21,7 @@ func sampleTakeIns(since time.Time) []TakeIn { Members: []meshdeliveryclient.Head{{Repository: "novox/mesh-delivery", Number: 9, Head: "ffff"}, {Repository: "novox/mesh-sdk", Number: 37, Head: "dddd"}}, Since: since}, {Kind: TakeInTaken, Key: PendingKey(12), What: TakeInRevision, Merges: m, WaitsFor: "src2:abc", Since: since, By: "lead-47@shanks", Why: "the named indexer can never run"}, - {Kind: TakeInPending, Key: PendingKey(13), What: TakeInRevision, Merges: m, Unknown: "the engine does not answer", Since: since}, // the indexer cannot learn its own build - // design 47 G §713: a revision refused by its validation, and one that could not validate. - {Kind: TakeInPending, Key: PendingKey(14), What: TakeInRevision, Merges: m, Refused: "a: names no version", Since: since}, - {Kind: TakeInPending, Key: PendingKey(15), What: TakeInRevision, Merges: m, Failed: "could not run: no build node", Since: since}, + {Kind: TakeInPending, Key: PendingKey(14), What: TakeInRevision, Merges: m, Refused: "a cycle with a build edge", Since: since}, } }