From 2cb3198049e5e5e311cee35901e59d0156ae55e5 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 22:47:19 +0100 Subject: [PATCH 01/18] Checker: judge the new indexer's manifest with the trunk's judge, and fail a held verdict (issue 1585, ADR 0633 clause 6) Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn --- .../internal/check/examiner_test.go | 31 +++++ .../internal/check/manifests_judge.go | 122 +++++++++++++++--- 2 files changed, 133 insertions(+), 20 deletions(-) diff --git a/modules/mesh-delivery-checker/internal/check/examiner_test.go b/modules/mesh-delivery-checker/internal/check/examiner_test.go index b237c86f..e9f490d5 100644 --- a/modules/mesh-delivery-checker/internal/check/examiner_test.go +++ b/modules/mesh-delivery-checker/internal/check/examiner_test.go @@ -1035,8 +1035,19 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) { // mail's tool while mail awaits web's database: both left out, so neither is refused (ADR 0614 §3, §6). cycle string trunkCycle bool + // indexer makes web the holder of the seat mesh-catalog, and the trunk's judge answers for it: "held" with no verdict, + // "aside" with a field it does not know; ownJudge has the group change the judge, so the manifest is also judged by the + // trunk's, in a second ask (ADR 0633 clause 6, 1585/T22, T23) + indexer string + ownJudge bool }{ {name: "accepted"}, + {name: "the indexer accepted by the trunk's judge", indexer: "ok"}, + {name: "the indexer held by the trunk's judge", indexer: "held", fails: "the indexer web, the holder of the seat mesh-catalog, would never run: the trunk's judge at mesh-delivery's trunk " + strings.Repeat("d", 40) + " does not accept its manifest: the judge holds it, with no verdict yet"}, + {name: "the indexer with a field the trunk's judge does not know", indexer: "aside", fails: "the judge does not know its fields new_field, so it waits"}, + {name: "the indexer refused by the trunk's judge", indexer: "refused", refused: "web has no id", fails: "does not accept its manifest: web has no id"}, + {name: "the indexer under a changed judge is judged by the trunk's as well", indexer: "aside", ownJudge: true, fails: "the judge does not know its fields new_field, so it waits"}, + {name: "a touched manifest the judge holds", indexer: "heldplain", fails: "holds the manifest of web: it gave no verdict"}, {name: "an untouched manifest's problem the trunk had", untouched: "mail says no credential form", trunkMail: "mail says no credential form"}, {name: "a problem the change causes in an untouched manifest", untouched: "mail reads a state web no longer keeps", fails: "refuses the manifest of mail: mail reads a state web no longer keeps"}, @@ -1077,6 +1088,10 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) { } b.ex.Read = fixedManifests{strings.Repeat("a", 64): mail, strings.Repeat("b", 64): trunkWeb, web: changed} } + if row.indexer != "" && row.indexer != "heldplain" { + b.ex.Read = fixedManifests{strings.Repeat("a", 64): {Module: "mail"}, strings.Repeat("b", 64): {Module: "web"}, + web: {Module: "web", Claims: []catalogue.Claim{{Name: CatalogSeat}}}} + } var judged []delivery.ComposeJob answer := b.asks.Answer b.asks.Answer = func(a delivery.StepAsk) delivery.Step { @@ -1085,6 +1100,12 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) { } judged = append(judged, *a.Compose) read := composer.ManifestRead{Accepted: row.refused == "", Refused: row.refused} + switch row.indexer { + case "held", "heldplain": + read = composer.ManifestRead{} + case "aside": + read.SetAside = "new_field" + } verdict, _ := json.Marshal(composer.Verdict{V: composer.Version, SDK: "exe:1", Cases: []composer.CaseVerdict{{Name: JudgeCase, Manifests: map[string]composer.ManifestRead{"web": read, "mail": {Accepted: row.untouched == "", Refused: row.untouched}}}, {Name: JudgeTrunk, Manifests: map[string]composer.ManifestRead{"web": {Accepted: true}, "mail": {Accepted: row.trunkMail == "", Refused: row.trunkMail}}}}}) @@ -1092,9 +1113,19 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) { return delivery.Step{ID: a.ID, State: delivery.StepPassed, Result: result} } p := pullOf(repoA, 7, head1) + if row.ownJudge { + p = pullOf(DeliveryRepository, 7, head1) + b.fetch.Changed = []string{JudgeDir + "/judge/judge.go"} + } p.Modules = []string{"web"} p.Fps = map[string]delivery.CandidateModule{"web": {FP2: "src2:" + strings.Repeat("f", 64), Manifest: web}} v := b.judge(t, caseOf(t, p)).Verdict + if row.ownJudge && (len(judged) != 2 || judged[0].Fetch["commit"] != head1 || judged[1].Fetch["commit"] != trunk) { + t.Fatalf("%s: judged %+v, want the head's judge and then the trunk's", row.name, judged) + } + if row.ownJudge { + judged = judged[1:] + } if len(judged) != 1 || judged[0].Fetch["repository"] != DeliveryRepository || judged[0].Fetch["commit"] != trunk || judged[0].Package != JudgePackage || judged[0].Dir != JudgeDir { t.Fatalf("%s: judged %+v (verdict %+v)", row.name, judged, v) diff --git a/modules/mesh-delivery-checker/internal/check/manifests_judge.go b/modules/mesh-delivery-checker/internal/check/manifests_judge.go index c3114fd3..c8320df3 100644 --- a/modules/mesh-delivery-checker/internal/check/manifests_judge.go +++ b/modules/mesh-delivery-checker/internal/check/manifests_judge.go @@ -120,26 +120,12 @@ func (st *state) judgeManifests(ctx context.Context) *delivery.Verdict { req.Predefined = list st.c.note("the manifests are judged against the predefined keys of " + from) } - raw, err := json.Marshal(req) - if err != nil { - return fail(goModule, err.Error()) + cv, trunkCase, v := st.runJudge(ctx, req, touched, commit, at) + if v != nil { + return v } - job := delivery.ComposeJob{Fetch: map[string]any{"repository": DeliveryRepository, "commit": commit}, Dir: JudgeDir, - Package: JudgePackage, Request: raw, Reader: "the indexer's judge at " + at} - a := delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: delivery.CheckerModule, - At: st.j.now().UTC(), Compose: &job} - st.c.Progress.Set(fmt.Sprintf("the judge ask %s of the manifests of %s", a.ID, strings.Join(touched, ", ")), "") - st.c.Asked(a.ID) - if err := st.j.Asks.Put(ctx, a); err != nil { - return fail(goModule, "the judge of the change's manifests could not be asked: "+err.Error()) - } - step, err := st.j.Asks.Wait(ctx, a) - if err != nil { - return fail(goModule, "the judge of the change's manifests: "+err.Error()) - } - cv, trunkCase, why := judgeAnswer(step) - if why != "" { - return fail(goModule, fmt.Sprintf("the indexer's judge at %s did not judge the change's manifests: %s", at, why)) + if v := st.refuseUnmovableIndexer(ctx, req, touched, cv, at); v != nil { + return v } // A new or changed manifest is judged in full; of one the change leaves as it is, a problem the trunk already had is a // warning (it is not refused for a rule it already broke), and a problem the change causes in it, which the trunk case @@ -147,7 +133,13 @@ func (st *state) judgeManifests(ctx context.Context) *delivery.Verdict { var refused, warned []string for _, module := range slices.Sorted(maps.Keys(cv.Manifests)) { r := cv.Manifests[module] - if r.Accepted || r.Refused == "" { + if r.Accepted { + continue + } + if r.Refused == "" { // held: a verdict not given is not one given (ADR 0633 clause 6) + if slices.Contains(touched, module) { + refused = append(refused, fmt.Sprintf("the indexer's judge at %s holds the manifest of %s: it gave no verdict", at, module)) + } continue } had := map[string]bool{} @@ -183,6 +175,96 @@ func (st *state) judgeManifests(ctx context.Context) *delivery.Verdict { return nil } +// runJudge asks the Go build seat to build the judge at the commit and run it over the request, and reads its change and +// trunk cases; a failed verdict when it cannot be asked or does not answer. +func (st *state) runJudge(ctx context.Context, req composer.Request, touched []string, commit, at string) (change, trunk composer.CaseVerdict, v *delivery.Verdict) { + raw, err := json.Marshal(req) + if err != nil { + return change, trunk, fail(goModule, err.Error()) + } + job := delivery.ComposeJob{Fetch: map[string]any{"repository": DeliveryRepository, "commit": commit}, Dir: JudgeDir, + Package: JudgePackage, Request: raw, Reader: "the indexer's judge at " + at} + a := delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: delivery.CheckerModule, + At: st.j.now().UTC(), Compose: &job} + st.c.Progress.Set(fmt.Sprintf("the judge ask %s of the manifests of %s", a.ID, strings.Join(touched, ", ")), "") + st.c.Asked(a.ID) + if err := st.j.Asks.Put(ctx, a); err != nil { + return change, trunk, fail(goModule, "the judge of the change's manifests could not be asked: "+err.Error()) + } + step, err := st.j.Asks.Wait(ctx, a) + if err != nil { + return change, trunk, fail(goModule, "the judge of the change's manifests: "+err.Error()) + } + change, trunk, why := judgeAnswer(step) + if why != "" { + return change, trunk, fail(goModule, fmt.Sprintf("the indexer's judge at %s did not judge the change's manifests: %s", at, why)) + } + return change, trunk, nil +} + +// CatalogSeat is the seat the indexer holds: the module whose manifest claims it is the indexer, found by the claim and +// never by its name (novox/hq ADR 0633 clause 6, ADR 0601 §4). +const CatalogSeat = "mesh-catalog" + +// refuseUnmovableIndexer fails the check when the trunk's judge, the one the running parts hold, would refuse or hold the +// change's new indexer manifest: such an indexer would never run, and every revision behind it would wait for ever +// (ADR 0633 clause 6). The indexer is the touched module whose manifest claims the seat mesh-catalog. When the change +// judges with its own head's judge (it changes the judge), the manifest is judged by the trunk's as well. The verdicts +// read are the trunk's judge's: its change case, where refused, held (not accepted with no reason) or with a field set +// aside (a field the running judge does not know waits, ADR 0601 §3) each fail it, naming the module and the problems. +func (st *state) refuseUnmovableIndexer(ctx context.Context, req composer.Request, touched []string, cv composer.CaseVerdict, judgedAt string) *delivery.Verdict { + if st.j.Read == nil { + return nil + } + var indexer string + for _, module := range touched { + m, err := st.j.Read.Manifest(ctx, req.Cases[0].Manifests[module].Digest) + if err != nil { + return fail(goModule, fmt.Sprintf("the manifest of %s cannot be read, so the indexer cannot be ruled out: %v", module, err)) + } + for _, c := range m.Claims { + if c.Name == CatalogSeat { + indexer = module + } + } + } + if indexer == "" { + return nil + } + if st.judgeHead != "" { + trunk := "" + if st.j.Trunks != nil { + trunk = st.j.Trunks.Heads()[DeliveryRepository] + } + if trunk == "" { + return fail(goModule, "the new indexer cannot be judged by the trunk's judge: the trunk head of "+DeliveryRepository+" is not known") + } + var v *delivery.Verdict + judgedAt = "mesh-delivery's trunk " + trunk + if cv, _, v = st.runJudge(ctx, req, touched, trunk, judgedAt); v != nil { + return v + } + } + r, ok := cv.Manifests[indexer] + var problems []string + switch { + case !ok: + problems = append(problems, "the judge gave no verdict") + case r.Refused != "": + problems = append(problems, strings.Split(r.Refused, "\n")...) + case !r.Accepted: + problems = append(problems, "the judge holds it, with no verdict yet") + case r.SetAside != "": + problems = append(problems, "the judge does not know its fields "+r.SetAside+", so it waits") + } + if len(problems) > 0 { + return fail(goModule, fmt.Sprintf("the indexer %s, the holder of the seat %s, would never run: the trunk's judge at %s does not accept its manifest: %s", + indexer, CatalogSeat, judgedAt, strings.Join(problems, "; "))) + } + st.c.note(fmt.Sprintf("the new indexer's manifest of %s is accepted by the trunk's judge at %s", indexer, judgedAt)) + return nil +} + // judgeAnswer reads the judge's change and trunk cases from its step, or why it cannot. func judgeAnswer(step delivery.Step) (change, trunk composer.CaseVerdict, why string) { if step.State != delivery.StepPassed { From 0579e4e74d559b00862375db66bed900eb4e3c80 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 22:54:21 +0100 Subject: [PATCH 02/18] =?UTF-8?q?Watchman:=20hear=20the=20indexer's=20take?= =?UTF-8?q?-in=20stream=20and=20raise=20take-in-waits=20past=2060=20minute?= =?UTF-8?q?s=20(revision)=20or=2030=20(group)=20(issue=201585,=20ADR=20063?= =?UTF-8?q?3=20=C2=A75)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn --- .../cmd/mesh-delivery-watchman/main.go | 29 +++++++ .../internal/watch/findings_test.go | 75 +++++++++++++++++++ .../internal/watch/signals.go | 1 + .../internal/watch/takein.go | 70 +++++++++++++++++ .../internal/watch/watch.go | 2 +- .../mesh-delivery-watchman/manifest_test.go | 7 +- modules/mesh-delivery-watchman/module.json | 3 + 7 files changed, 185 insertions(+), 2 deletions(-) create mode 100644 modules/mesh-delivery-watchman/internal/watch/takein.go diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go index 8ec5142d..8117b1d9 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go @@ -22,6 +22,7 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/provisions" @@ -375,6 +376,20 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro dead.ages() dead.check() provisioned(ctx, js, w, logger.Printf) + // The indexer's stream take-in (ADR 0633 §5): a revision or a group it holds back is a wait, ended by take-in.taken. The + // durable consumer _ is made by the controller from this module's consumes; the whole stream is heard at + // the start, so a wait that began while the watchman was away is read with its own since. + switch tc, err := cred.EventConsumer(ctx, js, catalogue.StreamBusName(delivery.TakeInStream)); { + case err != nil: + logger.Printf("the consumer %s_%s on %s does not exist yet, so no take-in wait is said until the watchman starts again: %v", + cred.Node, cred.Module, catalogue.StreamBusName(delivery.TakeInStream), err) + default: + hc, err := process.HearEvents(tc, func(m jetstream.Msg) { takeInHeard(w, m, logger.Printf) }, logger.Printf) + if err != nil { + return err + } + defer hc.Stop() + } w.Reconcile() if err := hear(conn, w, dead, logger.Printf); err != nil { return err @@ -439,3 +454,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } } + +// takeInHeard hands one event of a take-in stream to the watchman, by its name relative to the stream, and acknowledges it; +// one that is not a take-in event is said and acknowledged, since hearing it again would not read it. +func takeInHeard(w *watch.Watchman, m jetstream.Msg, logf func(string, ...any)) { + _, event, ok := strings.Cut(m.Subject(), ".event.") + if !ok { + logf("the subject %s of the take-in stream is not an event's, left", m.Subject()) + } else { + w.TakeIn(event, m.Data()) + } + if err := m.Ack(); err != nil { + logf("a take-in event could not be acknowledged: %v", err) + } +} diff --git a/modules/mesh-delivery-watchman/internal/watch/findings_test.go b/modules/mesh-delivery-watchman/internal/watch/findings_test.go index 40e308b2..67cadbe5 100644 --- a/modules/mesh-delivery-watchman/internal/watch/findings_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/findings_test.go @@ -34,6 +34,50 @@ func TestTheWatchmansFindings(t *testing.T) { // want are the finding keys standing at the end, with their severity. want map[string]string }{ + {"ADR 0633 §5 (1585/T21): a revision waiting for an indexer build is a finding past 60 minutes from its since, not within; a group past 30", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.TakeIn(takeInEvent(t, delivery.TakeInPending, revKey), takeInBody(t, delivery.TakeInPending, revKey, t0)) + if o.ttls[signalKey("take-in", revKey)] != TakeInRevisionBound || len(o.findings) != 0 { + t.Fatalf("a revision's wait: ttls %v, findings %v", o.ttls, o.findings) + } + w.TakeIn(takeInEvent(t, delivery.TakeInPending, groupKey), takeInBody(t, delivery.TakeInPending, groupKey, t0)) + if o.ttls[signalKey("take-in", groupKey)] != TakeInGroupBound { + t.Fatalf("a group's wait: ttls %v", o.ttls) + } + expire(w, c, signalKey("take-in", groupKey), TakeInGroupBound) + if len(o.findings) != 1 { + t.Fatalf("findings at 30 minutes: %v", o.findings) + } + expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound-TakeInGroupBound) + }, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning, FindingKey(ScopeDelivery, groupKey, KindTakeInWaits): Warning}}, + {"ADR 0633 §5 (1585/T21): the take-in.taken of a named indexer, a hand take-in or a drop clears the finding", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + for _, by := range []string{"", "operator@shanks"} { + w.TakeIn(takeInEvent(t, delivery.TakeInPending, revKey), takeInBody(t, delivery.TakeInPending, revKey, t0)) + expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound) + if len(o.findings) != 1 { + t.Fatalf("findings past the bound: %v", o.findings) + } + taken, err := delivery.EncodeTakeIn(delivery.TakeIn{Kind: delivery.TakeInTaken, Key: revKey, What: delivery.TakeInRevision, Since: t0, + Merges: []delivery.GroupMerge{{Repository: "novox/a", Merge: "c0ffee"}}, By: by, Why: map[bool]string{true: "dropped", false: ""}[by != ""], Dropped: by != ""}) + if err != nil { + t.Fatal(err) + } + w.TakeIn(takeInEvent(t, delivery.TakeInTaken, revKey), taken) + c.t = t0 + } + }, map[string]string{}}, + {"ADR 0633 §5 (1585/T21): a group's take-in.taken ends its 30 minutes, and the revision then pending under m. starts its own 60", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.TakeIn(takeInEvent(t, delivery.TakeInPending, groupKey), takeInBody(t, delivery.TakeInPending, groupKey, t0)) + expire(w, c, signalKey("take-in", groupKey), 25*time.Minute) + w.TakeIn(takeInEvent(t, delivery.TakeInTaken, groupKey), takeInBody(t, delivery.TakeInTaken, groupKey, t0)) + w.TakeIn(takeInEvent(t, delivery.TakeInPending, revKey), takeInBody(t, delivery.TakeInPending, revKey, c.t)) + if len(o.findings) != 0 || o.ttls[signalKey("take-in", revKey)] != TakeInRevisionBound { + t.Fatalf("after the hand-over: findings %v, ttls %v", o.findings, o.ttls) + } + expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound) + }, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}}, + {"ADR 0633 §5 (1585/T21): a pending event heard again at the start, since past its bound, is a finding at once", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.TakeIn(takeInEvent(t, delivery.TakeInPending, revKey), takeInBody(t, delivery.TakeInPending, revKey, t0.Add(-2*time.Hour))) + }, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}}, {"ADR 0578 §1: a module whose value carries a refusal of its manifest is a finding until it is accepted", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { value := func(verdict string, problems ...string) []byte { b, err := delivery.EncodeModule(delivery.Module{FP2: "src2:" + strings.Repeat("a", 64), Manifest: strings.Repeat("b", 64), @@ -528,3 +572,34 @@ func seedUnhealthy(t *testing.T, w *Watchman, o *fakeOut, sev string, since time o.findings[f.Key()], _ = DecodeFinding(v) w.SeedFinding(f.Key(), v) } + +const ( + revKey = "m.00000000000000000042" + groupKey = "g.feat/take-in" +) + +func takeInEvent(t *testing.T, kind, key string) string { + t.Helper() + e, err := delivery.TakeInEvent(kind, key) + if err != nil { + t.Fatal(err) + } + return e +} + +// takeInBody is a take-in event's body for a key: a group's when it starts with g. +func takeInBody(t *testing.T, kind, key string, since time.Time) []byte { + t.Helper() + body := delivery.TakeIn{Kind: kind, Key: key, What: delivery.TakeInRevision, Since: since, + Merges: []delivery.GroupMerge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}} + if key[0] == 'g' { + body.What, body.Group = delivery.TakeInGroup, "feat/take-in" + } else if kind == delivery.TakeInPending { + body.WaitsFor = "src2:abc" + } + b, err := delivery.EncodeTakeIn(body) + if err != nil { + t.Fatal(err) + } + return b +} diff --git a/modules/mesh-delivery-watchman/internal/watch/signals.go b/modules/mesh-delivery-watchman/internal/watch/signals.go index 042a5321..b1409b4a 100644 --- a/modules/mesh-delivery-watchman/internal/watch/signals.go +++ b/modules/mesh-delivery-watchman/internal/watch/signals.go @@ -919,6 +919,7 @@ var SignalRows = []struct{ Row, Signal, Bound, Kind string }{ {"S11", "a machine's tool runner heartbeat", "3 × its interval (" + (3 * HeartbeatEvery).String() + " when it says none)", KindToolsSilent}, {"S21", "a machine's have failed or its provisioner client's files not placed", ApplyEvery.String(), KindApplyFailing}, {"health", "a have unhealthy in two writes in a row; urgent after " + UnhealthyUrgentAfter.String(), "two writes", KindUnhealthy}, + {"take-in", "a source revision or a delivery group the indexer holds back (ADR 0633 §5)", TakeInRevisionBound.String() + " a revision, " + TakeInGroupBound.String() + " a group", KindTakeInWaits}, {"provisions", "a provision of mesh-provisioner in waits (ADR 0570 §7)", "at once", KindNeedsOperator}, } diff --git a/modules/mesh-delivery-watchman/internal/watch/takein.go b/modules/mesh-delivery-watchman/internal/watch/takein.go new file mode 100644 index 00000000..8d9abf23 --- /dev/null +++ b/modules/mesh-delivery-watchman/internal/watch/takein.go @@ -0,0 +1,70 @@ +package watch + +import ( + "fmt" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// KindTakeInWaits is a source revision or a delivery group the indexer holds back and has not taken in within its bound +// (novox/hq ADR 0633 §5, issue 1585): a revision for the indexer build it names, a group for its end. It stands until a +// `take-in.taken` ends the wait, whoever ended it (the named indexer, a revert, a hand take-in or a drop). +const KindTakeInWaits = "take-in-waits" + +// The bounds of a take-in wait, from the `take-in.pending` event's own since. +const ( + TakeInRevisionBound = 60 * time.Minute + TakeInGroupBound = 30 * time.Minute +) + +// takeInRow is the signals table's row of a take-in wait. +const takeInRow = "take-in" + +// TakeIn hears one event of the indexer's stream `take-in`, by its name relative to the stream (`take-in..`): +// a pending one starts the wait for its key, whose bound runs from the event's since, and a taken one ends it. A group's +// taken event ends the group's wait only: the revision it becomes is its own key and starts its own, longer wait. +func (w *Watchman) TakeIn(event string, body []byte) { + kind, key, err := delivery.ParseTakeInEvent(event) + if err != nil { + w.say("a take-in event is left: %v", err) + return + } + t, err := delivery.DecodeTakeIn(body) + if err != nil { + w.say("the body of %s cannot be read, left: %v", event, err) + return + } + w.mu.Lock() + defer w.mu.Unlock() + finding := FindingKey(ScopeDelivery, key, KindTakeInWaits) + if kind == delivery.TakeInTaken { + w.unsignal(takeInRow, key) + w.unfind(finding) + return + } + bound, what := TakeInRevisionBound, "a source revision waits for the indexer build "+t.WaitsFor + if t.What == delivery.TakeInGroup { + bound, what = TakeInGroupBound, "a delivery group is gathered and its end has not come" + } + var merges []string + for _, m := range t.Merges { + merges = append(merges, m.Repository) + } + f := Finding{Scope: ScopeDelivery, ID: key, Kind: KindTakeInWaits, Severity: Warning, Source: "take-in", Since: t.Since, + Summary: fmt.Sprintf("%s: %s, since %s (%s)", key, what, stamp(t.Since), strings.Join(merges, ", ")), + Said: fmt.Sprintf("held for %s", bound), + Headline: "A merged change is held back by the indexer", + Explanation: "The indexer holds " + what + " and has not taken it in within " + bound.String() + ", so what the merge changes does " + + "not reach the machines.", + Needs: "the indexer build it waits for to run, or the take-in by hand (indexer_take_in_pending), or a drop (indexer_drop_pending)"} + left := bound - w.now().Sub(t.Since) + if left <= 0 { + w.unsignal(takeInRow, key) + w.find(f) + return + } + w.unfind(finding) + w.signal(takeInRow, key, left, signalDue{Do: dueFind, Finding: f}) +} diff --git a/modules/mesh-delivery-watchman/internal/watch/watch.go b/modules/mesh-delivery-watchman/internal/watch/watch.go index 0e8b7925..67b641fe 100644 --- a/modules/mesh-delivery-watchman/internal/watch/watch.go +++ b/modules/mesh-delivery-watchman/internal/watch/watch.go @@ -366,7 +366,7 @@ func signalKind(value []byte) bool { return false } switch f.Source { - case "provisioner.failing", "S8", "S9", "S10", "S11": + case "provisioner.failing", "S8", "S9", "S10", "S11", "take-in": return true } return false diff --git a/modules/mesh-delivery-watchman/manifest_test.go b/modules/mesh-delivery-watchman/manifest_test.go index a577361e..6a54308e 100644 --- a/modules/mesh-delivery-watchman/manifest_test.go +++ b/modules/mesh-delivery-watchman/manifest_test.go @@ -51,6 +51,7 @@ type manifest struct { Version string `json:"version"` Slug string `json:"slug"` Reads []string `json:"reads"` + Consumes []string `json:"consumes"` State []state `json:"state"` OwnSecrets map[string]string `json:"own-secrets"` SecretsOwner string `json:"secrets-owner"` @@ -94,6 +95,10 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6 "mesh-delivery-assigner.wants", "mesh-delivery-builder.builds", "mesh-delivery-indexer.modules", "mesh-host.health", "mesh-provisioner.provisions", "node-engine.have"}) { t.Errorf("reads = %v", reads) } + // 1585/T20: it hears the indexer's stream take-in, whole, through a durable consumer the controller makes from consumes. + if !reflect.DeepEqual(m.Consumes, []string{"mesh-delivery-indexer.take-in.**"}) { + t.Errorf("consumes = %v", m.Consumes) + } // 1952/WM1: the watchman holds the seat mesh-watchdogs at mesh scope, its first seat (novox/hq ADR 0564 §1, §4). if !reflect.DeepEqual(m.Claims, []claim{{Name: "mesh-watchdogs", Scope: "mesh", Serves: []string{"signals", "stuck", "presence", "dead-letters", "resync"}}}) { t.Errorf("claims = %+v", m.Claims) @@ -101,7 +106,7 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6 if !reflect.DeepEqual(m.Tools, []string{"watchman_findings"}) { t.Errorf("tools = %v", m.Tools) } - for _, k := range []string{"emits", "context", "items", "consumes", "provides"} { + for _, k := range []string{"emits", "context", "items", "provides"} { if _, ok := top[k]; ok { t.Errorf("the manifest declares %q: the watchman has none", k) } diff --git a/modules/mesh-delivery-watchman/module.json b/modules/mesh-delivery-watchman/module.json index 4c85f5d7..2de43956 100644 --- a/modules/mesh-delivery-watchman/module.json +++ b/modules/mesh-delivery-watchman/module.json @@ -26,6 +26,9 @@ "mesh-provisioner.provisions", "node-engine.have" ], + "consumes": [ + "mesh-delivery-indexer.take-in.**" + ], "tools": [ "watchman_findings" ], From 72633722c5aaeaea8325dda41ced444b275b7588 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 22:55:27 +0100 Subject: [PATCH 03/18] Checker test: the accepted-indexer row asserts the trunk's judge said so (issue 1585, T22) Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn --- .../mesh-delivery-checker/internal/check/examiner_test.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/modules/mesh-delivery-checker/internal/check/examiner_test.go b/modules/mesh-delivery-checker/internal/check/examiner_test.go index e9f490d5..fca34153 100644 --- a/modules/mesh-delivery-checker/internal/check/examiner_test.go +++ b/modules/mesh-delivery-checker/internal/check/examiner_test.go @@ -1119,7 +1119,13 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) { } p.Modules = []string{"web"} p.Fps = map[string]delivery.CandidateModule{"web": {FP2: "src2:" + strings.Repeat("f", 64), Manifest: web}} - v := b.judge(t, caseOf(t, p)).Verdict + cs := caseOf(t, p) + v := b.judge(t, cs).Verdict + if row.indexer == "ok" && !slices.ContainsFunc(cs.notes, func(n string) bool { + return strings.Contains(n, "the new indexer's manifest of web is accepted by the trunk's judge") + }) { + t.Errorf("%s: the case says %v, want the new indexer's manifest said accepted by the trunk's judge", row.name, cs.notes) + } if row.ownJudge && (len(judged) != 2 || judged[0].Fetch["commit"] != head1 || judged[1].Fetch["commit"] != trunk) { t.Fatalf("%s: judged %+v, want the head's judge and then the trunk's", row.name, judged) } From 071a28f218c24d3738026216c2fc13d236d116ba Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 23:07:02 +0100 Subject: [PATCH 04/18] =?UTF-8?q?indexer:=20a=20source=20revision=20is=20t?= =?UTF-8?q?aken=20in=20only=20by=20the=20indexer=20it=20names=20(ADR=20063?= =?UTF-8?q?3=20=C2=A71-=C2=A75,=20issue=201585)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A merged message is kept in the indexer's own state pending and the merges there are computed together on overlays of the indexer's states: taken in whole when the indexer's fingerprint is the running build, otherwise only the indexer's own entry, graph key and verdict are written and the rest waits for the named indexer. A group's members are gathered until its end. Waits are said on the stream take-in, read by indexer_pending and steered by indexer_take_in_pending and indexer_drop_pending. Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn --- .../cmd/mesh-delivery-indexer-tools/main.go | 27 +- .../cmd/mesh-delivery-indexer/takein.go | 10 +- .../internal/index/candidates.go | 6 + .../internal/index/overlay.go | 110 +++ .../internal/index/pending.go | 837 ++++++++++++++++++ .../internal/index/pending_test.go | 321 +++++++ .../internal/index/pending_tools.go | 69 ++ .../internal/index/source_test.go | 22 +- .../internal/index/takein.go | 32 +- .../mesh-delivery-indexer/manifest_test.go | 24 +- modules/mesh-delivery-indexer/module.json | 11 +- 11 files changed, 1447 insertions(+), 22 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/overlay.go create mode 100644 modules/mesh-delivery-indexer/internal/index/pending.go create mode 100644 modules/mesh-delivery-indexer/internal/index/pending_test.go create mode 100644 modules/mesh-delivery-indexer/internal/index/pending_tools.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index baa62b17..37bf9725 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -8,6 +8,7 @@ package main import ( "context" "encoding/json" + "errors" "fmt" "os" "strings" @@ -191,8 +192,32 @@ func heldCounts(s stdio.KeptState) func() (index.OldSectionsCount, error) { } } +// keptRead is a state the tools bundle reads, as the index package reads a store: Get and Keys; it writes nothing. +type keptRead struct{ s stdio.KeptState } + +func (k keptRead) Get(_ context.Context, key string) ([]byte, uint64, bool, error) { + e, err := k.s.Get(key) + if err != nil || e == nil { + return nil, 0, false, err + } + return e.Value, e.Revision, true, nil +} + +func (k keptRead) Keys(context.Context) ([]string, error) { return k.s.Keys() } + +func (keptRead) Create(context.Context, string, []byte) error { return errReadOnly } + +func (keptRead) Update(context.Context, string, []byte, uint64) error { return errReadOnly } + +func (keptRead) Delete(context.Context, string, uint64) error { return errReadOnly } + +var errReadOnly = errors.New("the tools bundle only reads this state") + func main() { - if err := stdio.Serve("", append(Tools(askTakeIn(stdio.State(index.TakeInsState), 25*time.Second), time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState))), catalogTools()...)); err != nil { + ask := askTakeIn(stdio.State(index.TakeInsState), 25*time.Second) + tools := append(Tools(ask, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState))), catalogTools()...) + tools = append(tools, index.PendingTools(keptRead{stdio.State(index.PendingState)}, ask, time.Now)...) + if err := stdio.Serve("", tools); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err) os.Exit(1) } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go index 7a98edb7..2ef5b43e 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go @@ -16,6 +16,8 @@ type takerIn interface { TakeIn(ctx context.Context, module string) index.TakenIn TakeInRepository(ctx context.Context, repository string) index.TakenIn JudgeAgain(ctx context.Context, module, digest string) index.TakenIn + TakeInPendingByHand(ctx context.Context, key, by, why string) index.TakenIn + DropPending(ctx context.Context, key, by, why string) index.TakenIn } // takenIn answers one value of the takeins state (the tool indexer_take_in, issue 1945): a value not yet answered takes the @@ -34,7 +36,13 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream return } asked, r := "take-in", index.TakenIn{} - if a.Repository != "" { + if a.Pending != "" && a.Drop { + asked, r = "drop of a pending merge", ix.DropPending(ctx, a.Pending, a.By, a.Why) + a.Module = a.Pending + } else if a.Pending != "" { + asked, r = "hand take-in of what waits", ix.TakeInPendingByHand(ctx, a.Pending, a.By, a.Why) + a.Module = a.Pending + } else if a.Repository != "" { asked, r = "take-in of a repository", ix.TakeInRepository(ctx, a.Repository) a.Module = a.Repository } else if a.Judge { diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 85d1c415..75b6ea90 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -121,6 +121,12 @@ type Indexer struct { ownMu sync.Mutex // guards ownFP ownFP string // the fingerprint of the build this program runs, asked at the start pass (learnOwnBuild) judgePending atomic.Bool // the start's judge pass failed: judged again at the next message (verdicts.go) + // Pending is the state of merges held for the indexer their revision names (pending.go, novox/hq ADR 0633 §1); nil + // takes every merge in at once, as before the take-in step. + Pending Store + // SayTakeIn publishes a take-in event on the indexer's own stream take-in (ADR 0633 §5); nil says none. + SayTakeIn func(ctx context.Context, t delivery.TakeIn) error + pendingMu sync.Mutex // one take-in step at a time: a merge, a group's end, the start and a hand act (pending.go) } func (ix *Indexer) logf(format string, args ...any) { diff --git a/modules/mesh-delivery-indexer/internal/index/overlay.go b/modules/mesh-delivery-indexer/internal/index/overlay.go new file mode 100644 index 00000000..6405f0a4 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/overlay.go @@ -0,0 +1,110 @@ +package index + +import ( + "context" + "maps" + "slices" + "sync" +) + +// overlayStore is a store whose writes stay in memory over the store it reads through (novox/hq ADR 0633 §1): a merge +// applied to it is computed whole, its every fingerprint and verdict, and nothing reaches the bus. A key it wrote reads +// as written, a key it deleted as absent, every other as the store under it holds it. Its compare-and-set holds against +// what it reads: a key it wrote has a revision of its own, counted above every revision under it. A nil store under +// it holds nothing. +type overlayStore struct { + under Store + mu sync.Mutex + vals map[string]overlaid + next uint64 +} + +type overlaid struct { + value []byte + revision uint64 + deleted bool +} + +func newOverlay(under Store) *overlayStore { + return &overlayStore{under: under, vals: map[string]overlaid{}, next: 1 << 62} +} + +func (o *overlayStore) Get(ctx context.Context, key string) ([]byte, uint64, bool, error) { + o.mu.Lock() + v, ok := o.vals[key] + o.mu.Unlock() + if ok { + if v.deleted { + return nil, v.revision, false, nil + } + return slices.Clone(v.value), v.revision, true, nil + } + if o.under == nil { + return nil, 0, false, nil + } + return o.under.Get(ctx, key) +} + +// put checks the revision read against what the key holds now and writes it. +func (o *overlayStore) put(ctx context.Context, key string, value []byte, revision uint64, create, del bool) error { + _, rev, found, err := o.Get(ctx, key) + if err != nil { + return err + } + o.mu.Lock() + defer o.mu.Unlock() + switch { + case create && found, !create && (!found || rev != revision): + return ErrConflict + } + o.next++ + o.vals[key] = overlaid{value: slices.Clone(value), revision: o.next, deleted: del} + return nil +} + +func (o *overlayStore) Create(ctx context.Context, key string, value []byte) error { + return o.put(ctx, key, value, 0, true, false) +} + +func (o *overlayStore) Update(ctx context.Context, key string, value []byte, revision uint64) error { + return o.put(ctx, key, value, revision, false, false) +} + +func (o *overlayStore) Delete(ctx context.Context, key string, revision uint64) error { + return o.put(ctx, key, nil, revision, false, true) +} + +func (o *overlayStore) Keys(ctx context.Context) ([]string, error) { + var keys []string + if o.under != nil { + var err error + if keys, err = o.under.Keys(ctx); err != nil { + return nil, err + } + } + o.mu.Lock() + defer o.mu.Unlock() + keys = slices.DeleteFunc(keys, func(k string) bool { _, ok := o.vals[k]; return ok }) + for k, v := range o.vals { + if !v.deleted { + keys = append(keys, k) + } + } + slices.Sort(keys) + return keys, nil +} + +// written is what the overlay holds of key: its value, and whether it wrote or deleted it at all. +func (o *overlayStore) written(key string) (value []byte, deleted, ok bool) { + o.mu.Lock() + defer o.mu.Unlock() + v, ok := o.vals[key] + return slices.Clone(v.value), v.deleted, ok +} + +// changed is every key the overlay wrote or deleted, in order. +func (o *overlayStore) changed() []string { + o.mu.Lock() + defer o.mu.Unlock() + return slices.Sorted(maps.Keys(o.vals)) +} diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go new file mode 100644 index 00000000..1fdb7977 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -0,0 +1,837 @@ +package index + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "slices" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/process" + + "git.novox.be/novox/mesh-delivery/sdk/graph" +) + +// The take-in step (novox/hq ADR 0633): a source revision is taken in only by the indexer it names. A merged message is +// kept in the indexer's own state pending, one key per merge, until the take-in that ends it; a revision naming another +// build of the indexer writes only what building and moving that indexer needs, and the rest stays pending for it. +const ( + // PendingState is the indexer's state of held merges: the bucket mesh-delivery-indexer_pending, keyed + // delivery.PendingKey, so the key order is the merge order. An old `state` entry (the indexer is foundation, ADR + // 0595 §11), history 1, reached by the indexer alone. + PendingState = "pending" + PendingBucket = delivery.IndexerModule + "_" + PendingState + // PendingTool reads it (ADR 0633 §5); TakeInPendingTool and DropPendingTool steer it. + PendingTool = "indexer_pending" + TakeInPendingTool = "indexer_take_in_pending" + DropPendingTool = "indexer_drop_pending" +) + +// errClaimLost says another take-in claimed a pending merge first: this one took nothing in. +var errClaimLost = errors.New("another take-in claimed the merges first, so this one took nothing in") + +// NoBuild opens what a revision waits for when it gives the indexer no fingerprint: the reason follows. No indexer +// runs such a build, so the revision waits until a later merge gives it one or a hand take-in (indexer_take_in_pending). +const NoBuild = "no build: " + +// ErrOwnBuildUnknown is what a merged message answers while the indexer cannot learn the build it runs (ADR 0633 §1): +// the message is left unacknowledged, so the bus delivers it again, and nothing is written. +var ErrOwnBuildUnknown = errors.New("the build of the indexer this program runs is not known") + +// Pending is one value of the pending state: the merged message whole, as the forge bridge encoded it, the group it +// was merged in (Ended once the group's end was heard), since when it is held, the indexer build its revision waits +// for (absent while it waits for nothing yet, or behind a group being gathered), the build of the indexer that last +// wrote it, the wait it was last said under on the take-in stream (Said, SaidAt), and Taking, set by a take-in that +// claimed it: a second indexer that finds it claimed takes nothing in (ADR 0633 §1: never two take-ins of one merge). +type Pending struct { + V int `json:"v"` + Forge json.RawMessage `json:"forge"` + Group string `json:"group,omitempty"` + Ended bool `json:"ended,omitempty"` + Since time.Time `json:"since"` + WaitsFor string `json:"waits-for,omitempty"` + LastWriter string `json:"last-writer"` + Said string `json:"said,omitempty"` + SaidKind string `json:"said-kind,omitempty"` + SaidAt time.Time `json:"said-at,omitzero"` + Taking string `json:"taking,omitempty"` +} + +// PendingEntry is one key of the pending state as read: its key, revision, value and the merged message decoded. +type PendingEntry struct { + Key string + Revision uint64 + Value Pending + Forge delivery.Forge +} + +// DecodePending reads one pending value and its merged message. +func DecodePending(data []byte) (Pending, delivery.Forge, error) { + var p Pending + if err := json.Unmarshal(data, &p); err != nil { + return p, delivery.Forge{}, fmt.Errorf("not a pending value: %w", err) + } + if p.V != 1 { + return p, delivery.Forge{}, fmt.Errorf("a pending value of version %d, not 1", p.V) + } + f, err := delivery.DecodeForge(p.Forge) + if err != nil { + return p, f, fmt.Errorf("its merged message: %w", err) + } + if f.Kind != delivery.ForgeMerged { + return p, f, fmt.Errorf("a pending value holds a %s message, not a merged one", f.Kind) + } + return p, f, nil +} + +func encodePending(p Pending) ([]byte, error) { + p.V = 1 + return json.Marshal(p) +} + +// ReadPending is every value the pending state holds, in key order (the merge order); a key that does not read is +// said through logf and left. +func ReadPending(ctx context.Context, st Store, logf func(string, ...any)) ([]PendingEntry, error) { + keys, err := st.Keys(ctx) + if err != nil { + return nil, fmt.Errorf("listing %s: %w", PendingState, err) + } + slices.Sort(keys) + var out []PendingEntry + for _, key := range keys { + value, rev, found, err := st.Get(ctx, key) + if err != nil { + return nil, fmt.Errorf("%s.%s: %w", PendingState, key, err) + } + if !found { + continue + } + p, f, err := DecodePending(value) + if err != nil { + if logf != nil { + logf("%s.%s is left: %v", PendingState, key, err) + } + continue + } + out = append(out, PendingEntry{Key: key, Revision: rev, Value: p, Forge: f}) + } + return out, nil +} + +// Waits is the pending state read as what waits (ADR 0633 §3, §4): Ready are the merges no group still being gathered +// holds back, in merge order, which are one revision together; Groups are the groups still being gathered, each with +// its members held so far; Behind are the merges in a gathered group's member repositories that wait behind it, by the +// group's take-in key. +type Waits struct { + Ready []PendingEntry + Groups map[string][]PendingEntry + Behind map[string][]PendingEntry +} + +// WaitsOf works out what waits from the entries, in key order. A member of a group whose end was not heard is +// gathered; from the first such member on, a later merge in one of the group's members' repositories waits behind it, +// because its tree holds that member's commit. Every other merge is ready. +func WaitsOf(entries []PendingEntry) Waits { + w := Waits{Groups: map[string][]PendingEntry{}, Behind: map[string][]PendingEntry{}} + blocked := map[string]string{} // repository, lower case, to the group key it waits behind + for _, e := range entries { + if e.Value.Group != "" && !e.Value.Ended { + w.Groups[e.Value.Group] = append(w.Groups[e.Value.Group], e) + for _, m := range e.Forge.Members { + if _, ok := blocked[strings.ToLower(m.Repository)]; !ok { + blocked[strings.ToLower(m.Repository)] = delivery.GroupTakeInKey(e.Value.Group) + } + } + continue + } + if g, ok := blocked[strings.ToLower(e.Forge.Repository)]; ok { + w.Behind[g] = append(w.Behind[g], e) + continue + } + w.Ready = append(w.Ready, e) + } + return w +} + +// mergesOf are the merges of entries, oldest first. +func mergesOf(entries []PendingEntry) []delivery.GroupMerge { + out := make([]delivery.GroupMerge, len(entries)) + for i, e := range entries { + out[i] = delivery.GroupMerge{Repository: e.Forge.Repository, Number: e.Forge.Number, Merge: e.Forge.Merge} + } + return out +} + +func sinceOf(entries []PendingEntry) time.Time { + var at time.Time + for _, e := range entries { + if at.IsZero() || e.Value.Since.Before(at) { + at = e.Value.Since + } + } + return at +} + +// groupWait is the take-in body of a group being gathered. +func groupWait(kind, group string, held []PendingEntry) delivery.TakeIn { + t := delivery.TakeIn{Kind: kind, Key: delivery.GroupTakeInKey(group), What: delivery.TakeInGroup, Group: group, + Merges: mergesOf(held), Since: sinceOf(held)} + if len(held) > 0 { + t.Members = held[len(held)-1].Forge.Members + } + return t +} + +// revisionWait is the take-in body of a revision: its key is its oldest merge's. +func revisionWait(kind string, ready []PendingEntry, waitsFor string) delivery.TakeIn { + return delivery.TakeIn{Kind: kind, Key: ready[0].Key, What: delivery.TakeInRevision, Merges: mergesOf(ready), + WaitsFor: waitsFor, Since: sinceOf(ready)} +} + +// ownBuildNow is the build this program runs: the one learnt, else asked of the node-engine now. +func (ix *Indexer) ownBuildNow(ctx context.Context) string { + ix.ownMu.Lock() + fp := ix.ownFP + ix.ownMu.Unlock() + if fp != "" || ix.OwnBuild == nil { + return fp + } + ix.learnOwnBuild(ctx) + ix.ownMu.Lock() + defer ix.ownMu.Unlock() + return ix.ownFP +} + +// sayTakeIn publishes a take-in event and keeps what it said on the entries, so indexer_pending reads the last event of +// each wait from the state. A publish that fails is said in the log: the next evaluation says it again. +func (ix *Indexer) sayTakeIn(ctx context.Context, t delivery.TakeIn) bool { + if ix.SayTakeIn == nil { + return true + } + if err := ix.SayTakeIn(ctx, t); err != nil { + ix.logf("take-in: %s.%s is not said: %v", t.Kind, t.Key, err) + return false + } + return true +} + +// markSaid writes on each entry the wait it was said under, by compare-and-set; a lost one is left (its next +// evaluation says it again). +func (ix *Indexer) markSaid(ctx context.Context, entries []PendingEntry, t *delivery.TakeIn, change func(*Pending)) { + for _, e := range entries { + p := e.Value + if t != nil { + p.Said, p.SaidKind, p.SaidAt = t.Key, t.Kind, ix.now() + } + if change != nil { + change(&p) + } + data, err := encodePending(p) + if err != nil { + continue + } + if err := ix.Pending.Update(ctx, e.Key, data, e.Revision); err != nil { + ix.logf("take-in: %s.%s is not written again: %v", PendingState, e.Key, err) + } + } +} + +// Arrive takes one merged message of the forge bridge's stream in under ADR 0633: seq is its stream sequence. It is +// written to pending first (a redelivered message finds its key there and writes no second), and then what pending +// holds is evaluated (Evaluate): taken in whole when its revision names the build this program runs, or held for the +// indexer it names. It answers what was taken in, and a line saying what waits. A message is never written while the +// build this program runs is not known (ErrOwnBuildUnknown). +func (ix *Indexer) Arrive(ctx context.Context, seq uint64, f delivery.Forge) (Applied, string, error) { + if ix.Pending == nil || ix.Modules == nil { + a, err := ix.ApplyMessage(ctx, f) + return a, "", err + } + own := ix.ownBuildNow(ctx) + if own == "" { + return Applied{}, "", ErrOwnBuildUnknown + } + ix.pendingMu.Lock() + defer ix.pendingMu.Unlock() + raw, err := delivery.EncodeForge(f) + if err != nil { + return Applied{}, "", fmt.Errorf("%w: %v", ErrUnreadable, err) + } + key := delivery.PendingKey(seq) + data, err := encodePending(Pending{Forge: raw, Group: f.Group, Since: ix.now(), LastWriter: own}) + if err != nil { + return Applied{}, "", err + } + if err := ix.Pending.Create(ctx, key, data); err != nil && !errors.Is(err, ErrConflict) { + return Applied{}, "", fmt.Errorf("%s.%s: %w", PendingState, key, err) + } + if f.Group != "" { + entries, err := ReadPending(ctx, ix.Pending, ix.logf) + if err != nil { + return Applied{}, "", err + } + if held := WaitsOf(entries).Groups[f.Group]; len(held) > 0 { + t := groupWait(delivery.TakeInPending, f.Group, held) + if ix.sayTakeIn(ctx, t) { + ix.markSaid(ctx, held, &t, nil) + } + } + } + return ix.evaluate(ctx, own, false) +} + +// EndGroup takes a group's end in (ADR 0633 §4): once every member it names is held, the group's members are marked +// ended, the group's wait is said taken (take-in.taken under its group key), and pending is evaluated, the group now +// one revision. A member named and neither held nor taken in yet answers an error, so the end is delivered again once +// the member arrived. An end of a group taken in already (every member's merge at or behind its repository's taken +// merge) changes nothing and is said: it is never delivered again for ever. +func (ix *Indexer) EndGroup(ctx context.Context, end delivery.ForgeGroupEnd) (Applied, string, error) { + if ix.Pending == nil { + return Applied{}, "no pending state: the group's merges were taken in one by one", nil + } + own := ix.ownBuildNow(ctx) + if own == "" { + return Applied{}, "", ErrOwnBuildUnknown + } + ix.pendingMu.Lock() + defer ix.pendingMu.Unlock() + entries, err := ReadPending(ctx, ix.Pending, ix.logf) + if err != nil { + return Applied{}, "", err + } + var held []PendingEntry + for _, e := range entries { + if e.Value.Group == end.Group { + held = append(held, e) + } + } + isHeld := func(m delivery.GroupMerge) bool { + return slices.ContainsFunc(held, func(e PendingEntry) bool { + return delivery.SameRepository(e.Forge.Repository, m.Repository) && e.Forge.Number == m.Number + }) + } + var missing []string + for _, m := range end.Members { + if isHeld(m) { + continue + } + taken, err := ix.takenAlready(ctx, m) + if err != nil { + return Applied{}, "", err + } + if !taken { + missing = append(missing, fmt.Sprintf("%s#%d", m.Repository, m.Number)) + } + } + if len(missing) > 0 { + return Applied{}, "", fmt.Errorf("the end of the group %q names %s, not held yet: it waits for their merged messages", end.Group, strings.Join(missing, ", ")) + } + if len(held) == 0 { + ix.logf("take-in: the end of the group %q comes after its take-in: every member is taken in already, nothing changes", end.Group) + return Applied{}, "the group is taken in already", nil + } + if !slices.ContainsFunc(held, func(e PendingEntry) bool { return !e.Value.Ended }) { + return ix.evaluate(ctx, own, false) + } + // The group's wait ends first (take-in.taken under its group key); a revision it then is, pending for another + // indexer, is said pending under its oldest merge's key by the evaluation that follows. + t := groupWait(delivery.TakeInTaken, end.Group, held) + said := &t + if !ix.sayTakeIn(ctx, t) { + said = nil // said again by the take-in or the hold that follows (saidWaits) + } + ix.markSaid(ctx, held, said, func(p *Pending) { p.Ended = true }) + ix.logf("take-in: the group %q ended with %d members held: one revision now", end.Group, len(held)) + return ix.evaluate(ctx, own, false) +} + +// takenAlready says whether a group member's merge is at or behind its repository's taken merge. +func (ix *Indexer) takenAlready(ctx context.Context, m delivery.GroupMerge) (bool, error) { + if ix.Taken == nil { + return false, nil + } + key, err := delivery.RepositoryKey(m.Repository) + if err != nil { + return false, err + } + value, _, found, err := ix.Taken.Get(ctx, key) + if errors.Is(err, process.ErrNotYet) || (err == nil && !found) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("taken.%s: %w", key, err) + } + cur, err := delivery.DecodeTaken(value) + if err != nil || cur.Merge == "" { + return false, nil + } + if cur.Merge == m.Merge { + return true, nil + } + return ix.older(ctx, m.Repository, m.Merge, cur.Merge) +} + +// TakeInAtStart is the take-in at the start (ADR 0633 §2), before the consumer on the forge bridge's stream starts: +// it asks the build this program runs until it is known (wait says when to ask again, false ends it), then evaluates +// pending, taking in the whole revisions that name this build; a group still being gathered stays, and a revision +// naming yet another build is held for that one. A claim left by an indexer that ran before is void here. +func (ix *Indexer) TakeInAtStart(ctx context.Context, wait func(context.Context) bool) (Applied, string, error) { + if ix.Pending == nil { + return Applied{}, "", nil + } + own := ix.ownBuildNow(ctx) + for own == "" { + ix.logf("take-in: the build this program runs is not known yet, so nothing pending is taken in; asked again") + if !wait(ctx) { + return Applied{}, "", ErrOwnBuildUnknown + } + own = ix.ownBuildNow(ctx) + } + ix.pendingMu.Lock() + defer ix.pendingMu.Unlock() + return ix.evaluate(ctx, own, true) +} + +// evaluate works out what pending's ready merges name, together (ADR 0633 §3), and takes them in or holds them. +func (ix *Indexer) evaluate(ctx context.Context, own string, start bool) (Applied, string, error) { + entries, err := ReadPending(ctx, ix.Pending, ix.logf) + if err != nil { + return Applied{}, "", err + } + w := WaitsOf(entries) + if len(w.Ready) == 0 { + return Applied{}, ix.waitingLine(w), nil + } + if !start { + for _, e := range w.Ready { + if e.Value.Taking != "" { + ix.logf("take-in: %s.%s is claimed by the take-in %s, so this indexer takes nothing in", PendingState, e.Key, e.Value.Taking) + return Applied{}, "claimed by another take-in", nil + } + } + } + names, dry, err := ix.computeOver(ctx, w.Ready) + if err != nil { + return Applied{}, "", err + } + if names == own { + a, err := ix.takeInWhole(ctx, w.Ready, own, "") + if errors.Is(err, errClaimLost) { + return a, "claimed by another take-in", nil + } + return a, ix.waitingLine(w), err + } + return Applied{}, ix.waitingLine(w), ix.holdFor(ctx, w.Ready, names, own, dry) +} + +// waitingLine says what waits after an evaluation, for the heard state. +func (ix *Indexer) waitingLine(w Waits) string { + var parts []string + for _, g := range slices.Sorted(maps.Keys(w.Groups)) { + parts = append(parts, fmt.Sprintf("the group %q gathered (%d held)", g, len(w.Groups[g]))) + } + for _, g := range slices.Sorted(maps.Keys(w.Behind)) { + parts = append(parts, fmt.Sprintf("%d merges behind %s", len(w.Behind[g]), g)) + } + return strings.Join(parts, "; ") +} + +// dryRun is an indexer computing a take-in without writing it, and the overlays of what it would have written. +type dryRun struct { + ix *Indexer + modules, graph, verdicts *overlayStore + candidates, taken, predefined, configs *overlayStore + oldSections *overlayStore +} + +// dry is an indexer over overlays of this one's states (overlay.go): what a take-in would write stays in memory and +// nothing reaches the bus but the manifests and the packages the revision publishes, which clause 1 writes all the +// same. It says no event, kicks nothing in the background, and knows what this one knows of the source, the toolchains +// and the previews. Every field of Indexer is named here (TestDryNamesEveryField holds it so). +func (ix *Indexer) dry() *dryRun { + over := func(st Store) (Store, *overlayStore) { + if st == nil { + return nil, nil + } + o := newOverlay(st) + return o, o + } + d := &dryRun{} + x := &Indexer{Manifests: ix.Manifests, Packages: ix.Packages, JudgeID: ix.JudgeID, Files: ix.Files, GoSum: ix.GoSum, + GoMod: ix.GoMod, Now: ix.Now, Builds: ix.Builds, Named: ix.Named, Trunk: ix.Trunk, Read: ix.Read, Contains: ix.Contains, + ReadOpen: nil, OwnBuild: nil, SayRefused: nil, SayAccepted: nil, SayTakeIn: nil, Pending: nil} + x.Logf = func(format string, args ...any) { ix.logf("take-in, computed and not written: "+format, args...) } + x.Store, d.candidates = over(ix.Store) + x.Modules, d.modules = over(ix.Modules) + x.Graph, d.graph = over(ix.Graph) + x.Verdicts, d.verdicts = over(ix.Verdicts) + x.Taken, d.taken = over(ix.Taken) + x.Predefined, d.predefined = over(ix.Predefined) + x.ConfigCandidates, d.configs = over(ix.ConfigCandidates) + x.OldSections, d.oldSections = over(ix.OldSections) + ix.mu.Lock() + x.source = sourceNow{rev: ix.source.rev, heads: maps.Clone(ix.source.heads)} + x.previews = maps.Clone(ix.previews) + x.toolchains = map[string]map[string]delivery.Toolchain{} + for l, by := range ix.toolchains { + x.toolchains[l] = maps.Clone(by) + } + x.moving = maps.Clone(ix.moving) + x.untaken = maps.Clone(ix.untaken) + ix.mu.Unlock() + ix.ownMu.Lock() + x.ownFP = ix.ownFP + ix.ownMu.Unlock() + d.ix = x + return d +} + +// computeOver applies the ready merges, in merge order, to a dry indexer, as one overlay: each later merge's trees +// govern (ADR 0633 §3). It answers the fingerprint the indexer's own module has once they are applied (none is "", +// with its reason), and the dry run, whose overlays hold what the take-in would write. A merge that can never be read +// is left out, as its take-in will drop it. +func (ix *Indexer) computeOver(ctx context.Context, ready []PendingEntry) (string, *dryRun, error) { + d := ix.dry() + for _, e := range ready { + if _, err := d.ix.ApplyMessage(ctx, e.Forge); err != nil { + if errors.Is(err, ErrUnreadable) { + continue + } + return "", d, fmt.Errorf("computing the take-in of %s (%s#%d): %w", e.Key, e.Forge.Repository, e.Forge.Number, err) + } + } + key, _ := delivery.ModulesKey(delivery.IndexerModule) + value, _, found, err := d.ix.Modules.Get(ctx, key) + if err != nil { + return "", d, err + } + if !found { + return NoBuild + "the indexer holds no record of itself", d, nil + } + m, err := delivery.DecodeModule(value) + if err != nil { + return "", d, nil + } + if m.FP2 == "" { + return NoBuild + m.None2, d, nil + } + return m.FP2, d, nil +} + +// claim marks the entries as taken in by this take-in, by compare-and-set on the revisions read; a lost one leaves +// every claim made and answers false, and nothing is taken in. +func (ix *Indexer) claim(ctx context.Context, entries []PendingEntry, own, by string) ([]PendingEntry, bool) { + id := fmt.Sprintf("%s at %s", own, ix.now().Format(time.RFC3339Nano)) + if by != "" { + id = by + ": " + id + } + claimed := make([]PendingEntry, 0, len(entries)) + for _, e := range entries { + p := e.Value + p.Taking, p.LastWriter = id, own + data, err := encodePending(p) + if err == nil { + err = ix.Pending.Update(ctx, e.Key, data, e.Revision) + } + if err != nil { + ix.logf("take-in: the claim of %s.%s is lost (%v): another take-in took it, so this one takes nothing in", PendingState, e.Key, err) + return nil, false + } + _, rev, _, err := ix.Pending.Get(ctx, e.Key) + if err != nil { + return nil, false + } + e.Revision, e.Value = rev, p + claimed = append(claimed, e) + } + return claimed, true +} + +// takeInWhole takes the entries in with this indexer, as merges are taken in (ApplyMessage), oldest first, then +// removes them from pending and says each wait they were said under taken. by names a hand act ("" for none). +func (ix *Indexer) takeInWhole(ctx context.Context, entries []PendingEntry, own, by string) (Applied, error) { + return ix.takeInWholeBy(ctx, entries, own, by, "", "") +} + +// takeInWholeBy is takeInWhole; who and why, of a hand act, are carried on the take-in.taken events. +func (ix *Indexer) takeInWholeBy(ctx context.Context, entries []PendingEntry, own, by, who, why string) (Applied, error) { + claimed, ok := ix.claim(ctx, entries, own, by) + if !ok { + return Applied{}, errClaimLost + } + var applied Applied + for _, e := range claimed { + a, err := ix.ApplyMessage(ctx, e.Forge) + applied.Written = append(applied.Written, a.Written...) + if a.Late != "" { + applied.Late = a.Late + } + if errors.Is(err, ErrUnreadable) { + ix.logf("take-in: %s (%s#%d) can never be read, dropped: %v", e.Key, e.Forge.Repository, e.Forge.Number, err) + continue + } + if err != nil { + return applied, fmt.Errorf("taking %s (%s#%d) in: %w", e.Key, e.Forge.Repository, e.Forge.Number, err) + } + } + for _, e := range claimed { + if err := ix.Pending.Delete(ctx, e.Key, e.Revision); err != nil { + ix.logf("take-in: %s.%s is taken in and not removed: %v", PendingState, e.Key, err) + } + } + ix.logf("take-in: %d merges taken in as one revision by %s%s, oldest %s", len(claimed), own, byWords(by), claimed[0].Key) + for _, t := range saidWaits(claimed) { + t.By, t.Why = who, why + ix.sayTakeIn(ctx, t) + } + return applied, nil +} + +func byWords(by string) string { + if by == "" { + return "" + } + return " (a hand act: " + by + ")" +} + +// saidWaits are the taken events of every wait the entries were said under that is not ended yet. +func saidWaits(entries []PendingEntry) []delivery.TakeIn { + by := map[string][]PendingEntry{} + for _, e := range entries { + if e.Value.Said != "" && e.Value.SaidKind == delivery.TakeInPending { + by[e.Value.Said] = append(by[e.Value.Said], e) + } + } + var out []delivery.TakeIn + for _, key := range slices.Sorted(maps.Keys(by)) { + held := by[key] + if strings.HasPrefix(key, "g.") { + out = append(out, groupWait(delivery.TakeInTaken, held[0].Value.Group, held)) + continue + } + t := revisionWait(delivery.TakeInTaken, held, held[0].Value.WaitsFor) + t.Key = key + out = append(out, t) + } + return out +} + +// holdFor keeps the ready merges pending for the indexer build they name and writes only what building and moving +// that indexer needs (ADR 0633 §1): the indexer's own module entry, its key of the module graph and the verdict on its +// manifest, from the dry run; the manifests and the packages are written by the dry run itself. The revision is said +// pending under its oldest merge's key, and a wait it was said under before by another key is said taken. +func (ix *Indexer) holdFor(ctx context.Context, ready []PendingEntry, names, own string, d *dryRun) error { + key, _ := delivery.ModulesKey(delivery.IndexerModule) + if err := copyKey(ctx, d.modules, ix.Modules, key); err != nil { + return fmt.Errorf("the indexer's own module entry: %w", err) + } + if gk, err := graph.Key(delivery.IndexerModule); err == nil { + if err := copyKey(ctx, d.graph, ix.Graph, gk); err != nil && !errors.Is(err, process.ErrNotYet) { + return fmt.Errorf("the indexer's key of the module graph: %w", err) + } + } + if value, _, found, err := d.ix.Modules.Get(ctx, key); err == nil && found { + if m, err := delivery.DecodeModule(value); err == nil && m.Manifest != "" { + if vk, err := delivery.VerdictsKey(m.Manifest); err == nil { + if err := copyKey(ctx, d.verdicts, ix.Verdicts, vk); err != nil && !errors.Is(err, process.ErrNotYet) { + return fmt.Errorf("the verdict on the indexer's manifest: %w", err) + } + } + } + } + t := revisionWait(delivery.TakeInPending, ready, names) + var ended, unmarked []PendingEntry + said, changed := false, false + for _, e := range ready { + switch { + case e.Value.Said == t.Key && e.Value.SaidKind == delivery.TakeInPending: + said = true + changed = changed || e.Value.WaitsFor != names + if e.Value.WaitsFor != names { + unmarked = append(unmarked, e) + } + case e.Value.Said != "" && e.Value.SaidKind == delivery.TakeInPending: + ended = append(ended, e) + unmarked = append(unmarked, e) + default: + unmarked = append(unmarked, e) + } + } + for _, old := range saidWaits(ended) { + ix.sayTakeIn(ctx, old) + } + if !said || changed { + ix.logf("take-in: %d merges, oldest %s, name the indexer build %s, not %s, which runs: they wait for it", len(ready), t.Key, names, own) + if !ix.sayTakeIn(ctx, t) { + ix.markSaid(ctx, unmarked, nil, func(p *Pending) { p.WaitsFor, p.LastWriter = names, own }) + return nil + } + unmarked = ready + } + ix.markSaid(ctx, unmarked, &t, func(p *Pending) { p.WaitsFor, p.LastWriter = names, own }) + return nil +} + +// copyKey writes what the overlay holds of key to the store, when it wrote or deleted it. +func copyKey(ctx context.Context, from *overlayStore, to Store, key string) error { + if from == nil || to == nil { + return nil + } + value, deleted, ok := from.written(key) + if !ok { + return nil + } + return write(ctx, to, key, func(cur []byte, found bool) ([]byte, bool, error) { + switch { + case deleted: + return nil, found, nil + case found && string(cur) == string(value): + return nil, false, nil + } + return value, false, nil + }) +} + +// PendingWaits is what indexer_pending answers of the pending state's entries (ADR 0633 §5): the ready merges as one +// revision, each group being gathered, and the merges behind a group, each with the modules it holds back and the +// last take-in event said of it. +func PendingWaits(entries []PendingEntry) []delivery.PendingWait { + w := WaitsOf(entries) + var out []delivery.PendingWait + if len(w.Ready) > 0 { + out = append(out, waitOf(w.Ready, delivery.TakeInRevision, "")) + } + for _, g := range slices.Sorted(maps.Keys(w.Groups)) { + out = append(out, waitOf(w.Groups[g], delivery.TakeInGroup, "")) + } + for _, g := range slices.Sorted(maps.Keys(w.Behind)) { + out = append(out, waitOf(w.Behind[g], delivery.TakeInRevision, g)) + } + slices.SortStableFunc(out, func(a, b delivery.PendingWait) int { return a.Since.Compare(b.Since) }) + return out +} + +func waitOf(entries []PendingEntry, what, behind string) delivery.PendingWait { + pw := delivery.PendingWait{Key: entries[0].Key, What: what, Merges: mergesOf(entries), Behind: behind, Since: sinceOf(entries), + WaitsFor: entries[len(entries)-1].Value.WaitsFor, Holds: []string{}} + if what == delivery.TakeInGroup { + pw.Group = entries[0].Value.Group + pw.Key = delivery.GroupTakeInKey(pw.Group) + pw.Members = entries[len(entries)-1].Forge.Members + pw.WaitsFor = "" + } + for _, e := range entries { + pw.Keys = append(pw.Keys, e.Key) + for _, raw := range e.Forge.Manifests { + if name := manifestModule(raw); name != "" && !slices.Contains(pw.Holds, name) { + pw.Holds = append(pw.Holds, name) + } + } + if e.Value.Said != "" && e.Value.SaidAt.After(pw.SaidAt) { + pw.Said, pw.SaidAt = delivery.TakeInStream+"."+e.Value.SaidKind+"."+e.Value.Said, e.Value.SaidAt + } + } + slices.Sort(pw.Holds) + return pw +} + +// TakeInPendingByHand is the hand take-in (indexer_take_in_pending, ADR 0633 §5): the wait named by its key, the +// ready revision (its m. key) or a group gathered so far (its g. key), is taken in with this indexer and its judge, +// whatever build it names, for when the named indexer can never run or the group's end never comes. by and why are +// said in the journal and on the take-in.taken event. What waited behind it is evaluated again after. +func (ix *Indexer) TakeInPendingByHand(ctx context.Context, key, by, why string) TakenIn { + r := TakenIn{Module: key} + if ix.Pending == nil { + r.Answer = "the pending state is not on the bus: nothing waits" + return r + } + own := ix.ownBuildNow(ctx) + ix.pendingMu.Lock() + defer ix.pendingMu.Unlock() + entries, err := ReadPending(ctx, ix.Pending, ix.logf) + if err != nil { + r.Answer = err.Error() + return r + } + w := WaitsOf(entries) + var take []PendingEntry + switch { + case strings.HasPrefix(key, "g."): + for g, held := range w.Groups { + if delivery.GroupTakeInKey(g) == key { + take = held + } + } + case len(w.Ready) > 0 && w.Ready[0].Key == key: + take = w.Ready + } + if len(take) == 0 { + r.Answer = fmt.Sprintf("nothing waits under %s: indexer_pending names the waits", key) + return r + } + ix.logf("take-in: a hand take-in of %s (%d merges) asked by %s: %s", key, len(take), by, why) + hand := fmt.Sprintf("%s (%s)", by, why) + if _, err := ix.takeInWholeBy(ctx, take, own, hand, by, why); err != nil { + r.Answer = err.Error() + return r + } + r.Stored = true + r.Answer = fmt.Sprintf("%d merges taken in by the running indexer %s", len(take), own) + if _, _, err := ix.evaluate(ctx, own, false); err != nil { + r.Answer += "; what waited behind it is not evaluated: " + err.Error() + } + return r +} + +// DropPending is the hand drop (indexer_drop_pending, ADR 0633 §5): the one pending merge under key, abandoned +// because a later merge replaced it, is removed and nothing of it is taken in. Its wait is said taken, naming the drop; +// the merges left under that wait are said pending again by the evaluation that follows. +func (ix *Indexer) DropPending(ctx context.Context, key, by, why string) TakenIn { + r := TakenIn{Module: key} + if ix.Pending == nil { + r.Answer = "the pending state is not on the bus: nothing waits" + return r + } + own := ix.ownBuildNow(ctx) + ix.pendingMu.Lock() + defer ix.pendingMu.Unlock() + entries, err := ReadPending(ctx, ix.Pending, ix.logf) + if err != nil { + r.Answer = err.Error() + return r + } + at := slices.IndexFunc(entries, func(e PendingEntry) bool { return e.Key == key }) + if at < 0 { + r.Answer = fmt.Sprintf("%s holds no merge under %s", PendingState, key) + return r + } + e := entries[at] + if err := ix.Pending.Delete(ctx, e.Key, e.Revision); err != nil { + r.Answer = fmt.Sprintf("%s.%s is not removed: %v", PendingState, key, err) + return r + } + ix.logf("take-in: %s (%s#%d at %s) dropped by hand, asked by %s: %s; nothing of it is taken in", key, e.Forge.Repository, e.Forge.Number, e.Forge.Merge, by, why) + if e.Value.Said != "" && e.Value.SaidKind == delivery.TakeInPending { + var same []PendingEntry + for _, o := range entries { + if o.Key != key && o.Value.Said == e.Value.Said { + same = append(same, o) + } + } + for _, t := range saidWaits([]PendingEntry{e}) { + t.By, t.Why, t.Dropped = by, why, true + ix.sayTakeIn(ctx, t) + } + ix.markSaid(ctx, same, nil, func(p *Pending) { p.Said, p.SaidKind = "", "" }) + } + r.Stored, r.Answer = true, fmt.Sprintf("%s dropped: %s#%d at %s is not taken in", key, e.Forge.Repository, e.Forge.Number, e.Forge.Merge) + if own != "" { + if _, _, err := ix.evaluate(ctx, own, false); err != nil { + r.Answer += "; what waits is not evaluated again: " + err.Error() + } + } + return r +} diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go new file mode 100644 index 00000000..21e6870a --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -0,0 +1,321 @@ +package index + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "slices" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// takeInBed is an indexer holding the catalogue's avahi and the indexer's own module, both with a fingerprint, its +// pending state and the take-in events it says; base is the catalogue merge it was built from. +type takeInBed struct { + ix *Indexer + mods *memStore + pending *memStore + said []delivery.TakeIn + base delivery.Forge + logs *strings.Builder +} + +func newTakeInBed(t *testing.T) *takeInBed { + t.Helper() + ix, mods, f := newFormIndexerUnmerged(t, mod{"avahi", catalogueManifest(t, "avahi")}, + mod{delivery.IndexerModule, renamed(t, "avahi", delivery.IndexerModule)}) + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") + reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.27.1") + b := &takeInBed{ix: ix, mods: mods, pending: newMem(), base: f, logs: &strings.Builder{}} + ix.Verdicts = newMem() + ix.Pending = b.pending + ix.SayTakeIn = func(_ context.Context, t delivery.TakeIn) error { b.said = append(b.said, t); return nil } + logf := ix.Logf + ix.Logf = func(format string, args ...any) { + b.logs.WriteString(format + "\n") + if logf != nil { + logf(format, args...) + } + } + own := b.fp(t, delivery.IndexerModule) + if own == "" || b.fp(t, "avahi") == "" { + t.Fatalf("the bed's modules have no fingerprint") + } + ix.OwnBuild = func(context.Context) (string, error) { return own, nil } + return b +} + +func (b *takeInBed) fp(t *testing.T, module string) string { + t.Helper() + m, _ := moduleOf(t, b.mods, module) + return m.FP2 +} + +func (b *takeInBed) commit(t *testing.T, module string) string { + t.Helper() + m, _ := moduleOf(t, b.mods, module) + return m.Commit +} + +// merge is a catalogue merge at commit c whose trees of avahi and the indexer are given ("" keeps the base's). +func (b *takeInBed) merge(number int, c, avahi, indexer string) delivery.Forge { + f := b.base + f.Number, f.Merge, f.Head = number, c, c + f.Trees = map[string]string{} + for dir, tree := range b.base.Trees { + f.Trees[dir] = tree + } + if avahi != "" { + f.Trees["modules/avahi"] = avahi + } + if indexer != "" { + f.Trees["modules/"+delivery.IndexerModule] = indexer + } + return f +} + +func (b *takeInBed) arrive(t *testing.T, seq uint64, f delivery.Forge) { + t.Helper() + if _, _, err := b.ix.Arrive(context.Background(), seq, f); err != nil { + t.Fatalf("arrive %d: %v", seq, err) + } +} + +func (b *takeInBed) keys() []string { + k, _ := b.pending.Keys(context.Background()) + slices.Sort(k) + return k +} + +func (b *takeInBed) events() []string { + var out []string + for _, s := range b.said { + out = append(out, s.Kind+" "+s.Key) + } + return out +} + +func sha(c byte) string { return strings.Repeat(string(c), 40) } + +func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) + ctx := context.Background() + t.Run("T6 a merge while the own build is unknown writes nothing", func(t *testing.T) { + b := newTakeInBed(t) + b.ix.OwnBuild = func(context.Context) (string, error) { return "", errors.New("the engine does not answer") } + _, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", "")) + if !errors.Is(err, ErrOwnBuildUnknown) || len(b.keys()) != 0 || b.commit(t, "avahi") == sha('b') { + t.Errorf("err %v, pending %v, avahi at %s: want ErrOwnBuildUnknown and nothing written", err, b.keys(), b.commit(t, "avahi")) + } + }) + t.Run("T7 and T3 a merge alone, an indexer change, a merge behind it and a revert", func(t *testing.T) { + b := newTakeInBed(t) + own, avahi0 := b.fp(t, delivery.IndexerModule), b.fp(t, "avahi") + // A merge with no change to the indexer and nothing pending is taken in alone, as before. + b.arrive(t, 1, b.merge(20, sha('b'), "tree-avahi-2", "")) + if avahi1 := b.fp(t, "avahi"); avahi1 == avahi0 || len(b.keys()) != 0 || len(b.said) != 0 { + t.Fatalf("alone: avahi %s (was %s), pending %v, said %v: want taken in", avahi1, avahi0, b.keys(), b.events()) + } + avahi1, verdicts := b.fp(t, "avahi"), len(b.ix.Verdicts.(*memStore).vals) + // T4: a change to the indexer writes only its entry (with its graph key and verdict); the rest is pending. + b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) + named := b.fp(t, delivery.IndexerModule) + if named == own || b.fp(t, "avahi") != avahi1 || b.commit(t, "avahi") != sha('b') { + t.Fatalf("indexer change: indexer %s (own %s), avahi %s at %s: want only the indexer written", named, own, b.fp(t, "avahi"), b.commit(t, "avahi")) + } + if g := graphOf(t, b.ix, delivery.IndexerModule); g.FP != named { + t.Errorf("graph of the indexer names %s, want %s", g.FP, named) + } + if n := len(b.ix.Verdicts.(*memStore).vals); n > verdicts+1 { + t.Errorf("verdicts %d, want at most %d: only the new indexer manifest's", n, verdicts+1) + } + if want := []string{delivery.PendingKey(2)}; !reflect.DeepEqual(b.keys(), want) || !slices.Equal(b.events(), []string{"pending " + delivery.PendingKey(2)}) || b.said[0].WaitsFor != named { + t.Fatalf("pending %v, said %+v: want m.2 pending for %s", b.keys(), b.said, named) + } + // T5: the merge delivered again writes no second key and says nothing again. + b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) + if len(b.keys()) != 1 || len(b.said) != 1 { + t.Errorf("redelivered: pending %v, said %v", b.keys(), b.events()) + } + // A later merge joins: the newest governs and still names the new indexer. + b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")) + if len(b.keys()) != 2 || b.commit(t, "avahi") != sha('b') || len(b.said) != 1 { + t.Fatalf("behind: pending %v, avahi at %s, said %v: want both pending and nothing said again", b.keys(), b.commit(t, "avahi"), b.events()) + } + // A revert brings the indexer back to the running build: everything is taken in, oldest first. + b.arrive(t, 4, b.merge(23, sha('e'), "tree-avahi-5", b.base.Trees["modules/"+delivery.IndexerModule])) + if len(b.keys()) != 0 || b.fp(t, delivery.IndexerModule) != own || b.commit(t, "avahi") != sha('e') { + t.Fatalf("revert: pending %v, indexer %s, avahi at %s: want all taken in", b.keys(), b.fp(t, delivery.IndexerModule), b.commit(t, "avahi")) + } + if want := []string{"pending " + delivery.PendingKey(2), "taken " + delivery.PendingKey(2)}; !slices.Equal(b.events(), want) { + t.Errorf("said %v, want %v", b.events(), want) + } + }) + t.Run("T10 the named indexer takes the revision in at its start; a gathered group stays", func(t *testing.T) { + b := newTakeInBed(t) + b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) + g := b.merge(30, sha('f'), "", "") + g.Repository, g.Trees, g.Manifests, g.Locks, g.Group = "novox/mesh-sdk", map[string]string{}, nil, nil, "feat/x" + g.Members = []delivery.Head{{Repository: "novox/mesh-sdk", Number: 30, Head: g.Head}, {Repository: "novox/mesh-host", Number: 4, Head: sha('9')}} + b.arrive(t, 5, g) + named := b.fp(t, delivery.IndexerModule) + // The new indexer starts; its build is not known at first: a hand ask is answered meanwhile (R5), and the start waits. + next := &Indexer{} + *next = Indexer{Store: b.ix.Store, Modules: b.ix.Modules, Manifests: b.ix.Manifests, Graph: b.ix.Graph, Verdicts: b.ix.Verdicts, + Packages: b.ix.Packages, Builds: b.ix.Builds, Pending: b.pending, SayTakeIn: b.ix.SayTakeIn, Logf: b.ix.Logf, GoSum: b.ix.GoSum} + next.toolchains = b.ix.toolchains + asked := 0 + next.OwnBuild = func(context.Context) (string, error) { + if asked++; asked < 2 { + return "", errors.New("not yet") + } + return named, nil + } + if r := next.TakeInPendingByHand(ctx, "m.nothing", "lead@shanks", "a test"); r.Stored || !strings.Contains(r.Answer, "nothing waits") { + t.Errorf("a hand ask while the own build is unknown: %+v", r) + } + if _, _, err := next.TakeInAtStart(ctx, func(context.Context) bool { return true }); err != nil { + t.Fatal(err) + } + if asked < 2 || !reflect.DeepEqual(b.keys(), []string{delivery.PendingKey(5)}) || b.commit(t, "avahi") != sha('c') { + t.Errorf("start: asked %d, pending %v, avahi at %s: want the revision taken in, the group left", asked, b.keys(), b.commit(t, "avahi")) + } + }) + t.Run("T8 T9 T11 a group gathered until its end, a merge behind it, one alone, an end again", func(t *testing.T) { + b := newTakeInBed(t) + b.ix.Taken = newMem() + // The commits in the order they were made: one is in another when it comes first. + order := "a756bc" + b.ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { + return strings.IndexByte(order, commit[0]) <= strings.IndexByte(order, in[0]), nil + } + member := func(repo string, number int, c string) delivery.Forge { + f := b.merge(number, c, "", "") + f.Group = "feat/take-in" + f.Members = []delivery.Head{{Repository: catalogueRepo, Number: 40, Head: sha('1')}, {Repository: "novox/mesh-sdk", Number: 41, Head: sha('2')}} + if repo != catalogueRepo { + f.Repository, f.Trees, f.Manifests, f.Locks = repo, map[string]string{}, nil, nil + } + f.Head = map[int]string{40: sha('1'), 41: sha('2')}[number] + return f + } + b.arrive(t, 1, member(catalogueRepo, 40, sha('7'))) + if b.commit(t, "avahi") == sha('7') || !slices.Equal(b.events(), []string{"pending g.feat/take-in"}) { + t.Fatalf("first member: avahi at %s, said %v: want held, the group said pending", b.commit(t, "avahi"), b.events()) + } + // A merge in another repository is taken in alone; one in a member's repository waits behind the group. + alone := b.merge(50, sha('5'), "", "") + alone.Repository, alone.Trees, alone.Manifests, alone.Locks = "novox/hq", map[string]string{}, nil, nil + b.arrive(t, 2, alone) + b.arrive(t, 3, b.merge(51, sha('6'), "tree-avahi-9", "")) + if !reflect.DeepEqual(b.keys(), []string{delivery.PendingKey(1), delivery.PendingKey(3)}) { + t.Fatalf("pending %v: want the member and the merge behind it", b.keys()) + } + end := delivery.ForgeGroupEnd{Group: "feat/take-in", Members: []delivery.GroupMerge{{Repository: catalogueRepo, Number: 40, Merge: sha('7')}, {Repository: "novox/mesh-sdk", Number: 41, Merge: sha('b')}}} + if _, _, err := b.ix.EndGroup(ctx, end); err == nil { + t.Fatal("an end before its last member is taken in") + } + b.arrive(t, 4, member("novox/mesh-sdk", 41, sha('b'))) + if _, _, err := b.ix.EndGroup(ctx, end); err != nil { + t.Fatal(err) + } + if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('6') { + t.Fatalf("end: pending %v, avahi at %s: want the group and the merge behind it taken in", b.keys(), b.commit(t, "avahi")) + } + if want := []string{"pending g.feat/take-in", "pending g.feat/take-in", "taken g.feat/take-in"}; !slices.Equal(b.events(), want) { + t.Errorf("said %v, want %v", b.events(), want) + } + // R1: the end published again after the take-in changes nothing and is not delivered again. + if _, words, err := b.ix.EndGroup(ctx, end); err != nil || !strings.Contains(words, "taken in already") { + t.Errorf("an end again: %q %v", words, err) + } + // T11 (the lead's row): a group whose revision changes the indexer: the group's wait is taken, then the revision pending. + b.said = nil + g := member(catalogueRepo, 40, sha('c')) + g.Group, g.Trees = "feat/two", b.merge(0, "", "", "tree-indexer-7").Trees + g.Members = g.Members[:1] + b.arrive(t, 9, g) + if _, _, err := b.ix.EndGroup(ctx, delivery.ForgeGroupEnd{Group: "feat/two", Members: []delivery.GroupMerge{{Repository: catalogueRepo, Number: 40, Merge: sha('c')}}}); err != nil { + t.Fatal(err) + } + if want := []string{"pending g.feat/two", "taken g.feat/two", "pending " + delivery.PendingKey(9)}; !slices.Equal(b.events(), want) { + t.Errorf("a group naming another indexer: said %v, want %v", b.events(), want) + } + }) + t.Run("T5 two take-ins of one pending revision: the one that loses the claim writes nothing", func(t *testing.T) { + b := newTakeInBed(t) + b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) + named := b.fp(t, delivery.IndexerModule) + b.ix.ownMu.Lock() + b.ix.ownFP = named + b.ix.ownMu.Unlock() + // The other indexer claims the merge between this one's read and its claim. + b.pending.beforeWrite = func() { + v := b.pending.vals[delivery.PendingKey(2)] + var p Pending + _ = json.Unmarshal(v, &p) + p.Taking = "the other indexer" + data, _ := json.Marshal(p) + b.pending.put(delivery.PendingKey(2), data) + } + if _, _, err := b.ix.TakeInAtStart(ctx, func(context.Context) bool { return false }); err != nil { + t.Fatal(err) + } + if b.commit(t, "avahi") == sha('c') || len(b.keys()) != 1 || !strings.Contains(b.logs.String(), "is lost") { + t.Errorf("avahi at %s, pending %v: want nothing taken in by the loser, and it said", b.commit(t, "avahi"), b.keys()) + } + }) + t.Run("T12 T13 indexer_pending names the wait; a hand take-in and a drop say what they ended", func(t *testing.T) { + b := newTakeInBed(t) + b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) + b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")) + var answer delivery.PendingAnswer + for _, tool := range PendingTools(b.pending, nil, nil) { + if tool.Name == PendingTool { + out, err := tool.Run(map[string]any{}) + if err != nil { + t.Fatal(err) + } + answer = out.(delivery.PendingAnswer) + } + } + if len(answer.Waits) != 1 || answer.Waits[0].Key != delivery.PendingKey(2) || !slices.Equal(answer.Waits[0].Holds, []string{"avahi", delivery.IndexerModule}) || + answer.Waits[0].Said != "take-in.pending."+delivery.PendingKey(2) || len(answer.Waits[0].Keys) != 2 { + t.Fatalf("indexer_pending = %+v", answer) + } + b.said = nil + if r := b.ix.DropPending(ctx, delivery.PendingKey(2), "lead@shanks", "replaced by m.3"); !r.Stored { + t.Fatalf("drop: %+v", r) + } + if len(b.said) < 2 || !b.said[0].Dropped || b.said[0].Key != delivery.PendingKey(2) || b.said[0].By != "lead@shanks" || b.said[1].Key != delivery.PendingKey(3) { + t.Fatalf("drop said %+v: want m.2 taken (dropped, by) and m.3 pending", b.said) + } + b.said = nil + if r := b.ix.TakeInPendingByHand(ctx, delivery.PendingKey(3), "lead@shanks", "the named indexer can never run"); !r.Stored { + t.Fatalf("hand take-in: %+v", r) + } + if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('d') || len(b.said) != 1 || b.said[0].Kind != delivery.TakeInTaken || b.said[0].By != "lead@shanks" { + t.Errorf("hand take-in: pending %v, avahi at %s, said %+v", b.keys(), b.commit(t, "avahi"), b.said) + } + }) + t.Run("the dry run names every field of the indexer", func(t *testing.T) { + // dry (pending.go) builds an indexer field by field: a field added to Indexer is named there, or this fails. + named := map[string]bool{} + for _, f := range []string{"Store", "Modules", "Manifests", "Predefined", "ConfigCandidates", "Taken", "Verdicts", "OldSections", + "JudgeID", "SayRefused", "SayAccepted", "Packages", "Graph", "Files", "GoSum", "GoMod", "Now", "Builds", "Named", "Trunk", + "Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Pending", "SayTakeIn"} { + named[f] = true + } + ty := reflect.TypeOf(Indexer{}) + for i := range ty.NumField() { + if f := ty.Field(i); f.IsExported() && !named[f.Name] { + t.Errorf("Indexer.%s is not named by dry", f.Name) + } + } + }) +} diff --git a/modules/mesh-delivery-indexer/internal/index/pending_tools.go b/modules/mesh-delivery-indexer/internal/index/pending_tools.go new file mode 100644 index 00000000..33c08f4c --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/pending_tools.go @@ -0,0 +1,69 @@ +package index + +import ( + "context" + "fmt" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// PendingTools are the indexer's tools of what waits for a take-in (novox/hq ADR 0633 §5): indexer_pending reads the +// pending state, and indexer_take_in_pending and indexer_drop_pending steer it by an ask in takeins, as indexer_take_in +// does. pending is the pending state as the tools bundle reads it; ask writes an ask and waits for the indexer's answer. +func PendingTools(pending Store, ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time) []stdio.Tool { + str := func(args map[string]any, k string) string { s, _ := args[k].(string); return s } + hand := func(drop bool) func(map[string]any) (any, error) { + return func(args map[string]any) (any, error) { + key, value, err := NewPendingAsk(str(args, "key"), str(args, "why"), stdio.CallerOf(args), drop, now()) + if err != nil { + return nil, err + } + a, err := ask(key, value) + if err != nil { + return nil, err + } + return a.Result, nil + } + } + return []stdio.Tool{ + {Name: PendingTool, + Description: "What waits for a take-in (ADR 0633): merges the indexer holds in its pending state instead of taking them in. " + + "A revision (what revision, its key its oldest merge's m.) waits for the build of the indexer it names " + + "(waits-for): the running indexer wrote only what building and moving that indexer needs, and the new indexer takes " + + "it in at its start. A group (what group, key g.) is a delivery group whose members are gathered until the " + + "forge bridge says its end; a merge in one of its members' repositories waits behind it (behind). Each names its " + + "merges, since when it waits, the modules it holds back (holds) and the last take-in event said of it. Empty waits: " + + "every merge is taken in. The watchman raises take-in-waits past 60 minutes for a revision, 30 for a group; " + + "indexer_take_in_pending and indexer_drop_pending steer a wait.", + Input: map[string]any{"type": "object", "properties": map[string]any{}}, + Run: func(map[string]any) (any, error) { + var unread []string + entries, err := ReadPending(context.Background(), pending, func(format string, args ...any) { + unread = append(unread, fmt.Sprintf(format, args...)) + }) + if err != nil { + return nil, err + } + return delivery.PendingAnswer{Waits: PendingWaits(entries), Unread: unread}, nil + }}, + {Name: TakeInPendingTool, + Description: "Take a wait in by hand with the running indexer and its judge (ADR 0633 §5): the revision under its m. key, " + + "whatever build of the indexer it names, or a delivery group under its g. key with the members gathered so far. For " + + "when the named indexer can never run, or a group's end never comes. A hand act: who and why are said in the " + + "indexer's journal and on the take-in.taken event, which clears take-in-waits. indexer_pending names the keys.", + Input: map[string]any{"type": "object", "required": []string{"key", "why"}, "properties": map[string]any{ + "key": map[string]any{"type": "string", "description": "the wait's key, m. or g., as indexer_pending names it"}, + "why": map[string]any{"type": "string", "description": "why it is taken in by hand, in plain words"}}}, + Run: hand(false)}, + {Name: DropPendingTool, + Description: "Drop one pending merge by hand, abandoned because a later merge replaced it (ADR 0633 §5): it is removed from " + + "the pending state and nothing of it is taken in; what waited with it is worked out again. A hand act: who and why are " + + "said in the journal and on the take-in.taken event (dropped).", + Input: map[string]any{"type": "object", "required": []string{"key", "why"}, "properties": map[string]any{ + "key": map[string]any{"type": "string", "description": "the pending merge's key, m., as indexer_pending names its merges' wait"}, + "why": map[string]any{"type": "string", "description": "why it is dropped, in plain words"}}}, + Run: hand(true)}, + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/source_test.go b/modules/mesh-delivery-indexer/internal/index/source_test.go index 8bb56ebd..31961771 100644 --- a/modules/mesh-delivery-indexer/internal/index/source_test.go +++ b/modules/mesh-delivery-indexer/internal/index/source_test.go @@ -51,6 +51,17 @@ func TestASourceRevisionStampsTheModulesOfTheRepositoriesHeadItNames(t *testing. } func TestAMergeHeardAfterTheSourceNamesItsHeadIsStampedWhenWritten(t *testing.T) { // 1344/AC1 (the source first) + // A source revision older than the one held is left (merged here from a test of its own, for the test budget). + t.Run("an older source revision is left", func(t *testing.T) { + ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs)) + ctx := context.Background() + _ = ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})) + _ = ix.ApplySource(ctx, 9, sourceOf("novox/mesh-delivery", merge)) + _ = ix.ApplySource(ctx, 7, sourceOf("novox/mesh-delivery", merge)) + if m, _ := moduleOf(t, mods, "a"); m.Src != 9 { + t.Errorf("module a Src = %d, want 9", m.Src) + } + }) ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs)) ctx := context.Background() if err := ix.ApplySource(ctx, 7, sourceOf("novox/mesh-delivery", merge)); err != nil { @@ -93,14 +104,3 @@ func TestAMergeThatNoSourceRevisionNamesIsWrittenAsToday(t *testing.T) { // 1344 t.Errorf("module a = %+v, want the new merge with Src 0", m) } } - -func TestASourceRevisionOlderThanTheOneHeldIsLeft(t *testing.T) { - ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs)) - ctx := context.Background() - _ = ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})) - _ = ix.ApplySource(ctx, 9, sourceOf("novox/mesh-delivery", merge)) - _ = ix.ApplySource(ctx, 7, sourceOf("novox/mesh-delivery", merge)) - if m, _ := moduleOf(t, mods, "a"); m.Src != 9 { - t.Errorf("module a Src = %d, want 9", m.Src) - } -} diff --git a/modules/mesh-delivery-indexer/internal/index/takein.go b/modules/mesh-delivery-indexer/internal/index/takein.go index 579ba62b..f1982e05 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein.go +++ b/modules/mesh-delivery-indexer/internal/index/takein.go @@ -33,7 +33,11 @@ type TakeInAsk struct { // Judge asks the manifests to be judged again now (indexer_judge_again) instead of the module to be taken in. Judge bool `json:"judge,omitempty"` // Digest names, with Judge, the one manifest digest of the module to judge (indexer_judge with a digest). - Digest string `json:"digest,omitempty"` + Digest string `json:"digest,omitempty"` + // Pending asks the wait under this take-in key to be taken in by hand (indexer_take_in_pending), or with Drop the + // one pending merge under it to be dropped (indexer_drop_pending), instead of a module (pending.go, ADR 0633 §5). + Pending string `json:"pending,omitempty"` + Drop bool `json:"drop,omitempty"` By string `json:"by,omitempty"` At time.Time `json:"at"` Result *TakenIn `json:"result,omitempty"` @@ -52,8 +56,8 @@ func DecodeTakeIn(data []byte) (TakeInAsk, error) { if a.V != 1 { return a, fmt.Errorf("a takeins value of version %d, not 1", a.V) } - if a.Module == "" && a.Repository == "" { - return a, errors.New("a takeins value naming no module or repository") + if a.Module == "" && a.Repository == "" && a.Pending == "" { + return a, errors.New("a takeins value naming no module, repository or pending wait") } return a, nil } @@ -82,3 +86,25 @@ func NewTakeInRepository(repository, why, by string, at time.Time) (string, []by data, err := json.Marshal(TakeInAsk{V: 1, Repository: repository, Why: strings.TrimSpace(why), By: by, At: at.UTC()}) return "repository." + key, data, err } + +// NewPendingAsk is the value indexer_take_in_pending (drop false) or indexer_drop_pending (drop true) writes: its key +// in takeins, `pending..`, and the value. key is a wait's key as indexer_pending names it, for a +// drop one pending merge's. +func NewPendingAsk(key, why, by string, drop bool, at time.Time) (string, []byte, error) { + key = strings.TrimSpace(key) + if !strings.HasPrefix(key, "m.") && !(strings.HasPrefix(key, "g.") && !drop) { + if drop { + return "", nil, errors.New("name the pending merge to drop by its key, m., as indexer_pending names it") + } + return "", nil, errors.New("name the wait to take in by its key, m. or g., as indexer_pending names it") + } + if strings.TrimSpace(why) == "" { + return "", nil, errors.New("say why (why): a hand act on what waits is said in the journal and on its take-in event") + } + act := "take-in" + if drop { + act = "drop" + } + data, err := json.Marshal(TakeInAsk{V: 1, Pending: key, Drop: drop, Why: strings.TrimSpace(why), By: by, At: at.UTC()}) + return "pending." + act + "." + key, data, err +} diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 4911041b..d8f68d3b 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -16,6 +16,8 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-delivery/sdk/graph" + + "mesh-delivery-indexer/internal/index" ) // The controller's manifest reader is mesh-controller internal/catalogue (Manifest, manifest.go: Consumes :521, @@ -28,6 +30,12 @@ type state struct { History int `json:"history,omitempty"` } +type streamDecl struct { + Name string `json:"name"` + Events []string `json:"events"` + MaxAgeDays int `json:"max-age-days,omitempty"` +} + type artifact struct { Name string `json:"name"` Kind string `json:"kind"` @@ -51,6 +59,7 @@ type manifest struct { Emits []string `json:"emits"` Consumes []string `json:"consumes"` State []state `json:"state"` + Streams []streamDecl `json:"streams"` Reads []string `json:"reads"` Invokes []string `json:"invokes"` OwnSecrets map[string]string `json:"own-secrets"` @@ -99,7 +108,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build } // 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard. if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: delivery.PackagesState, History: delivery.PackagesHistory}, - {Name: graph.State, History: graph.History}}; !reflect.DeepEqual(m.State, want) { + {Name: graph.State, History: graph.History}, {Name: index.PendingState, History: 1}}; !reflect.DeepEqual(m.State, want) { t.Errorf("state = %+v, want %+v", m.State, want) } if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-delivery-assigner.placement", "mesh-delivery-builder.build-log", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) { @@ -125,7 +134,9 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none. The same bundle // reads the verdicts state (indexer_verdicts) and asks the indexer to judge again through the takeins state // (indexer_judge; issue 774, clause 3). 1966/AC11: indexer_old_sections reads the count of the old sections (state oldsections). - if want := []string{"indexer_take_in", "indexer_old_sections", "indexer_verdicts", "indexer_judge"}; !reflect.DeepEqual(m.Tools, want) { + // 1585/T12, T13 (ADR 0633 §5): indexer_pending reads the pending state; indexer_take_in_pending and indexer_drop_pending + // ask through takeins. + if want := []string{"indexer_take_in", "indexer_old_sections", "indexer_verdicts", "indexer_judge", index.PendingTool, index.TakeInPendingTool, index.DropPendingTool}; !reflect.DeepEqual(m.Tools, want) { t.Errorf("tools = %v, want %v", m.Tools, want) } // ADR 0604 §2: it holds the seat mesh-catalog, serving its four verbs from its tools bundle (which stays without a @@ -134,8 +145,13 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build !slices.Equal(m.Claims[0].Serves, []string{"modules", "module", "provides", "dependents"}) { t.Errorf("claims = %+v, want mesh-catalog with its four verbs", m.Claims) } - // No streams, no bus, no other seat (ADR 0437 §4); it says no event. - for _, k := range []string{"streams", "logs", "provides", "requires", "uses", "defines-seats", "seat", "seats", "bus"} { + // 1585/T11 (ADR 0633 §1, §5): its one stream, take-in, an old streams entry (the indexer is foundation, ADR 0595 §11), + // carries the take-in events. + if want := []streamDecl{{Name: delivery.TakeInStream, Events: []string{"take-in.**"}, MaxAgeDays: 7}}; !reflect.DeepEqual(m.Streams, want) { + t.Errorf("streams = %+v, want %+v", m.Streams, want) + } + // No other bus section, no other seat (ADR 0437 §4). + for _, k := range []string{"logs", "provides", "requires", "uses", "defines-seats", "seat", "seats", "bus"} { if _, ok := top[k]; ok { t.Errorf("the manifest declares %q: the indexer has none", k) } diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 3fa9e8a7..fafd9bcc 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -18,7 +18,10 @@ "indexer_take_in", "indexer_old_sections", "indexer_verdicts", - "indexer_judge" + "indexer_judge", + "indexer_pending", + "indexer_take_in_pending", + "indexer_drop_pending" ], "consumes": [ "mesh-build-go.toolchain.reported", @@ -38,7 +41,11 @@ { "name": "verdicts", "history": 5 }, { "name": "oldsections", "history": 1 }, { "name": "packages", "history": 5 }, - { "name": "graph", "history": 5 } + { "name": "graph", "history": 5 }, + { "name": "pending", "history": 1 } + ], + "streams": [ + { "name": "take-in", "events": ["take-in.**"], "max-age-days": 7 } ], "emits": [ "manifest-refused", From 531d05ab4b9f2c35d039338f5a57a4483da44358 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 23:08:37 +0100 Subject: [PATCH 05/18] =?UTF-8?q?indexer:=20hear=20a=20group's=20end,=20ke?= =?UTF-8?q?ep=20merges=20pending,=20take=20in=20at=20start=20(ADR=200633?= =?UTF-8?q?=20=C2=A72,=20=C2=A74,=20issue=201585)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The consumer tells a group's end apart by its name before the forge decode, a merged message goes through the pending state, and the pending revisions naming this build are taken in before the consumer starts; take-in events go on the indexer's own stream. Until the pending state is on the bus every merge is taken in at once, as before. Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn --- .../cmd/mesh-delivery-indexer/main.go | 58 ++++++++++++++++++- .../cmd/mesh-delivery-indexer/main_test.go | 18 ++++++ .../cmd/mesh-delivery-indexer/verdicts.go | 25 ++++++++ .../internal/index/pending.go | 17 +++++- 4 files changed, 114 insertions(+), 4 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 0d896889..ccdbe7ac 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -16,6 +16,7 @@ import ( "log" "os" "os/signal" + "strings" "sync/atomic" "syscall" "time" @@ -153,6 +154,30 @@ type delivered interface { Metadata() (*jetstream.MsgMetadata, error) } +// handleGroupEnd takes a group's end in (index.EndGroup): a body that does not read is terminated and said; an end +// whose members are not all held yet, or while the build this program runs is not known, is left for redelivery. +func handleGroupEnd(ctx context.Context, m delivered, ix *index.Indexer, h *index.Heard, logger *log.Logger) { + end, err := delivery.DecodeForgeGroupEnd(m.Data()) + if err != nil { + logger.Printf("%s: not a group's end, dropped: %v", m.Subject(), err) + h.Outcome, h.Words = index.HeardDropped, err.Error() + _ = m.Term() + return + } + h.Kind, h.Branch = delivery.ForgeGroupEndEvent, end.Group + applied, words, err := ix.EndGroup(ctx, end) + h.Written, h.Late = applied.Written, applied.Late + if err != nil { + logger.Printf("%s: the end of the group %q left for redelivery: %v", m.Subject(), end.Group, err) + h.Outcome, h.Words = index.HeardLeft, err.Error() + return + } + h.Outcome, h.Words = index.HeardApplied, words + if err := m.DoubleAck(ctx); err != nil { + logger.Printf("%s: the end of the group %q applied, its acknowledgement was not confirmed: %v", m.Subject(), end.Group, err) + } +} + // handle takes one message of FORGE into the candidates and acknowledges it. A message it can never read is terminated, // never redelivered, and the log names its subject and why. A write that failed stays unacknowledged: the server // delivers it again (the server's ack wait is the only clock). Each message handled is written to the heard state @@ -171,6 +196,11 @@ func handle(ctx context.Context, m delivered, ix *index.Indexer, heard index.Sto logger.Printf("%s: sequence %d is not written to the heard state: %v", h.Subject, h.Sequence, err) } }() + // A delivery group's end (novox/hq ADR 0633 §4) has a body of its own: told apart by its name, before the decode. + if strings.HasSuffix(m.Subject(), "."+delivery.ForgeGroupEndEvent) { + handleGroupEnd(ctx, m, ix, &h, logger) + return + } f, err := delivery.DecodeForge(m.Data()) if err != nil { logger.Printf("%s: not a forge message, dropped: %v", m.Subject(), err) @@ -182,7 +212,13 @@ func handle(ctx context.Context, m delivered, ix *index.Indexer, heard index.Sto if f.Merge != "" { h.Commit = f.Merge } - applied, err := ix.ApplyMessage(ctx, f) + var applied index.Applied + if f.Kind == delivery.ForgeMerged && h.Sequence > 0 { + // A merge is taken in only by the indexer its revision names (ADR 0633 §1): kept in pending until then. + applied, h.Words, err = ix.Arrive(ctx, h.Sequence, f) + } else { + applied, err = ix.ApplyMessage(ctx, f) + } h.Written, h.Late = applied.Written, applied.Late switch { case errors.Is(err, index.ErrUnreadable): @@ -308,7 +344,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Predefined: awaitedStore{predefined}, ConfigCandidates: awaitedStore{configCandidates}, Logf: logger.Printf, Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call), ReadOpen: openVia(calls.Call), Packages: awaitedStore{packages}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call), - OwnBuild: ownBuildVia(calls.Call, cred.Node), Graph: awaitedStore{graph}} + OwnBuild: ownBuildVia(calls.Call, cred.Node), Graph: awaitedStore{graph}, + // The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear; + // until then every merge is taken in at once, as before. + Pending: awaitedStore{process.AwaitBucket(ctx, js, conn, index.PendingBucket, "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)}, + SayTakeIn: sayTakeIn(js, cred)} // The log's entries at the start are read before any fingerprint is computed or registered module indexed (below); each // new one indexes the module it names when that module is registered and the bucket holds none (registered.go). var started atomic.Bool @@ -458,6 +498,20 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } defer pc.Stop() } + // The pending revisions that name this build are taken in now, oldest first, before the consumer on the forge bridge's + // stream starts (ADR 0633 §2); the build this program runs is asked until it is known, said every minute. A hand + // take-in (indexer_take_in_pending) is answered meanwhile: the takeins watch runs already. + if _, words, err := ix.TakeInAtStart(ctx, func(ctx context.Context) bool { + // A context's bound between two asks of the node-engine, no timer (ADR 0437 §8), as askTakeIn bounds its wait. + again, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + <-again.Done() + return ctx.Err() == nil + }); err != nil { + return fmt.Errorf("the take-in at start: %w", err) + } else if words != "" { + logger.Printf("take-in at start: still waiting: %s", words) + } cons, err := cred.EventConsumer(ctx, js, forgeStream) if err != nil { // Essential: the forge's merges are what the indexer indexes. diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go index eb70b62f..f42d7972 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go @@ -102,6 +102,24 @@ func TestAMessageThatIsNoForgeMessageIsTerminatedAndLogged(t *testing.T) { // 12 if h := heardOf(t, heard); h.Outcome != index.HeardDropped || h.Sequence != 41 || h.Words == "" { t.Errorf("heard = %+v", h) } + // 1585/T9 (ADR 0633 §4): a group's end is told apart by its name before the forge decode: one that reads is acked + // (an indexer with no pending state takes the members in one by one), one that does not is dropped as a group's end. + for _, c := range []struct { + data string + termd int + logged string + }{{`{"v":1,"group":"feat/x","members":[{"repository":"novox/mesh-sdk","number":3,"merge":"` + head + `"}]}`, 0, ""}, + {`{"v":1,"group":"feat/x"}`, 1, "not a group's end"}} { + logs.Reset() + m := &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event." + delivery.ForgeGroupEndEvent, data: []byte(c.data)} + handle(context.Background(), m, &index.Indexer{Store: st, Builds: index.NewBuilds()}, heard, log.New(&logs, "", 0)) + if m.termd != c.termd || m.acked+m.termd != 1 || !strings.Contains(logs.String(), c.logged) { + t.Errorf("group end %s: terminated %d acked %d log %q", c.data, m.termd, m.acked, logs.String()) + } + if h := heardOf(t, heard); h.Kind != delivery.ForgeGroupEndEvent && c.termd == 0 { + t.Errorf("heard of a group's end = %+v", h) + } + } } func TestTheSourceMainKeyIsTakenAtItsRevision(t *testing.T) { // 1344/AC1 (the watch's side: key, revision, value) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/verdicts.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/verdicts.go index 79e1b316..495cb684 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/verdicts.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/verdicts.go @@ -62,3 +62,28 @@ func sayAccepted(js jetstream.JetStream, cred process.Account, logf func(string, } } } + +// sayTakeIn publishes a take-in event on the indexer's own stream take-in (novox/hq ADR 0633 §5): +// `mesh.stream..event.take-in..`. A publish that fails is the caller's to say. +func sayTakeIn(js jetstream.JetStream, cred process.Account) func(context.Context, delivery.TakeIn) error { + return func(ctx context.Context, t delivery.TakeIn) error { + event, err := delivery.TakeInEvent(t.Kind, t.Key) + if err != nil { + return err + } + data, err := delivery.EncodeTakeIn(t) + if err != nil { + return err + } + msg := nats.NewMsg(process.ModuleStreamSubject(cred.Module, event)) + msg.Header.Set("x-source", cred.Module) + msg.Header.Set("x-node", cred.Node) + msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano)) + msg.Header.Set("content-type", "application/json") + msg.Data = data + pub, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + _, err = js.PublishMsg(pub, msg) + return err + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index 1fdb7977..e88b20c0 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -265,7 +265,13 @@ func (ix *Indexer) Arrive(ctx context.Context, seq uint64, f delivery.Forge) (Ap if err != nil { return Applied{}, "", err } - if err := ix.Pending.Create(ctx, key, data); err != nil && !errors.Is(err, ErrConflict) { + switch err := ix.Pending.Create(ctx, key, data); { + case errors.Is(err, process.ErrNotYet): + // The state this delivery brings is not on the bus yet (issue 1790): the merge is taken in at once, as before. + ix.logf("take-in: the state %s is not on the bus yet, so %s#%d is taken in at once, as before ADR 0633", PendingBucket, f.Repository, f.Number) + a, err := ix.ApplyMessage(ctx, f) + return a, "", err + case err != nil && !errors.Is(err, ErrConflict): return Applied{}, "", fmt.Errorf("%s.%s: %w", PendingState, key, err) } if f.Group != "" { @@ -299,6 +305,9 @@ func (ix *Indexer) EndGroup(ctx context.Context, end delivery.ForgeGroupEnd) (Ap ix.pendingMu.Lock() defer ix.pendingMu.Unlock() entries, err := ReadPending(ctx, ix.Pending, ix.logf) + if errors.Is(err, process.ErrNotYet) { + return Applied{}, "the pending state is not on the bus yet: the group's merges were taken in one by one", nil + } if err != nil { return Applied{}, "", err } @@ -392,7 +401,11 @@ func (ix *Indexer) TakeInAtStart(ctx context.Context, wait func(context.Context) } ix.pendingMu.Lock() defer ix.pendingMu.Unlock() - return ix.evaluate(ctx, own, true) + a, words, err := ix.evaluate(ctx, own, true) + if errors.Is(err, process.ErrNotYet) { + return a, "the pending state is not on the bus yet: nothing waits", nil + } + return a, words, err } // evaluate works out what pending's ready merges name, together (ADR 0633 §3), and takes them in or holds them. From d11a5af113c603233cd770732b3490dca477aee1 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 23:09:20 +0100 Subject: [PATCH 06/18] =?UTF-8?q?View:=20view=5Ftiers=20names=20a=20module?= =?UTF-8?q?=20the=20indexer=20holds=20back,=20read=20through=20indexer=5Fp?= =?UTF-8?q?ending=20(issue=201585,=20ADR=200633=20=C2=A75)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn --- .../cmd/mesh-delivery-view-tools/main.go | 12 ++++- .../tiers_rows_test.go | 30 +++++++++++ .../mesh-delivery-view/internal/view/read.go | 2 + .../mesh-delivery-view/internal/view/tiers.go | 52 +++++++++++++++++++ modules/mesh-delivery-view/manifest_test.go | 5 ++ modules/mesh-delivery-view/module.json | 3 ++ 6 files changed, 103 insertions(+), 1 deletion(-) diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go index 746dc902..834e8c39 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go @@ -324,7 +324,8 @@ func main() { PackagesState: read(stdio.State(delivery.IndexerModule + "." + delivery.PackagesState)), GraphState: read(stdio.State(graph.Read)), SourceState: read(stdio.State(delivery.GateModule + ".source")), ProvisionsState: read(stdio.State(provisions.ProvisionsRead)), - BuildLog: stdio.Log(delivery.BuildLogRead).Since} + BuildLog: stdio.Log(delivery.BuildLogRead).Since, + Pending: pendingOf} u := view.UsageReader{Haves: r.HavesState, Resolved: read(stdio.State("mesh-config.resolved")), Tries: read(stdio.State("mesh-delivery-checker.tries")), Steps: map[string]view.StateRead{}, Pauses: map[string]view.StateRead{}, Backlog: backlogOf(socket())} for _, lang := range delivery.AskLanguages { @@ -337,3 +338,12 @@ func main() { os.Exit(1) } } + +// pendingOf asks the indexer's tool indexer_pending (declared in invokes): what it holds back and why (ADR 0633 §5). +func pendingOf() (delivery.PendingAnswer, error) { + raw, err := stdio.Ask(delivery.IndexerModule+".indexer_pending", map[string]any{}) + if err != nil { + return delivery.PendingAnswer{}, err + } + return delivery.DecodePendingAnswer(raw) +} diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/tiers_rows_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/tiers_rows_test.go index bf2c81dd..c9f8bba5 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/tiers_rows_test.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/tiers_rows_test.go @@ -102,6 +102,36 @@ func tiersToolSaysEachModulesWait(t *testing.T) { web.Waits[1].Edge != "provision database" || !strings.Contains(web.Waits[1].Words, "asked: the build db runs does not offer it") { t.Errorf("web = %+v, want after db, held on ace by the provision rule and its provision asked (g14 is away)", web) } + // ADR 0633 §5 (1585/T24): a module the indexer holds back, read through indexer_pending, waits on the edge take-in from the + // wait's since; a held module the revision does not move is named in the first tier with that wait alone, a group's wait + // says its group, and a tool that cannot be asked is said and names no module. + heldSince := time.Date(2026, 11, 4, 22, 0, 0, 0, time.UTC) + r.Pending = func() (delivery.PendingAnswer, error) { + return delivery.PendingAnswer{Waits: []delivery.PendingWait{ + {Key: "m.00000000000000000042", What: delivery.TakeInRevision, WaitsFor: "src2:abc", Since: heldSince, Holds: []string{"web", "newmod"}}, + {Key: "g.feat", What: delivery.TakeInGroup, Group: "feat", Since: heldSince.Add(time.Hour), Holds: []string{"lib"}}}}, nil + } + out, err = run(t, r, view.TiersTool, map[string]any{"revision": float64(9)}) + held, _ := out.(view.Tiers) + if err != nil || len(held.Tiers) != 2 || held.Pending != "" { + t.Fatalf("view_tiers with waits = %+v, %v", out, err) + } + web = held.Tiers[1][0] + if last := web.Waits[len(web.Waits)-1]; last.Edge != view.TakeInEdge || !last.Since.Equal(heldSince) || !strings.Contains(last.Words, "src2:abc") { + t.Errorf("web = %+v, want its last wait on the edge take-in since %s", web, heldSince) + } + if lib := held.Tiers[0][1]; lib.Module != "lib" || lib.Waits[len(lib.Waits)-1].Edge != view.TakeInEdge || !strings.Contains(lib.Waits[len(lib.Waits)-1].Words, "group feat") { + t.Errorf("lib = %+v, want its last wait the group's", lib) + } + if nm := held.Tiers[0][len(held.Tiers[0])-1]; nm.Module != "newmod" || len(nm.Waits) != 1 || nm.Waits[0].Edge != view.TakeInEdge { + t.Errorf("the first tier = %+v, want newmod, which the revision does not move, named with its take-in wait alone", held.Tiers[0]) + } + r.Pending = func() (delivery.PendingAnswer, error) { return delivery.PendingAnswer{}, errors.New("no answer") } + if out, err = run(t, r, view.TiersTool, map[string]any{"revision": float64(9)}); err != nil || !strings.Contains(out.(view.Tiers).Pending, "no answer") || + strings.Contains(fmt.Sprint(out.(view.Tiers).Tiers), view.TakeInEdge) { + t.Errorf("view_tiers with no answer of indexer_pending = %+v, %v", out, err) + } + r.Pending = nil // No revision named: the gate's newest, its source key's revision. r.SourceState = view.StateRead{GetEntry: func(string) (json.RawMessage, uint64, error) { return json.RawMessage(`{}`), 9, nil }} if out, err := run(t, r, view.TiersTool, map[string]any{}); err != nil || out.(view.Tiers).Revision != 9 || out.(view.Tiers).Modules != 3 { diff --git a/modules/mesh-delivery-view/internal/view/read.go b/modules/mesh-delivery-view/internal/view/read.go index 5da2b0bb..7e7afa6a 100644 --- a/modules/mesh-delivery-view/internal/view/read.go +++ b/modules/mesh-delivery-view/internal/view/read.go @@ -58,6 +58,8 @@ type Reader struct { // GraphState is the indexer's module graph (graph.Bucket, ADR 0603 §2), SourceState the gate's source and // ProvisionsState the provisioner's keys: what view_tiers reads (tiers.go). Unset reads none, said in the answer. GraphState, SourceState, ProvisionsState StateRead + // Pending reads the indexer's waits through its tool indexer_pending (ADR 0633 §5); unset names no module held back. + Pending func() (delivery.PendingAnswer, error) // BuildLog reads the builder's log from a time (stdio.Log(delivery.BuildLogRead).Since); unset reads none. BuildLog LogSince } diff --git a/modules/mesh-delivery-view/internal/view/tiers.go b/modules/mesh-delivery-view/internal/view/tiers.go index 35a1e0a7..59d05bf1 100644 --- a/modules/mesh-delivery-view/internal/view/tiers.go +++ b/modules/mesh-delivery-view/internal/view/tiers.go @@ -43,6 +43,8 @@ type Tiers struct { Graph string `json:"graph,omitempty"` Cycle []string `json:"cycle,omitempty"` Away []string `json:"away,omitempty"` + // Pending says why the indexer's waits could not be read, so no module is named held by its revision's indexer (ADR 0633 §5). + Pending string `json:"pending,omitempty"` } // TierModule is one module of a tier: its fingerprint, the modules of the same revision it waits for (After), each @@ -115,6 +117,7 @@ func (r Reader) Tiers(revision uint64) (Tiers, error) { out.Away = slices.Sorted(maps.Keys(away)) wants, _ := values(r.WantsState) provided, perr := values(r.ProvisionsState) + held, heldAll := r.heldBack(&out) // The ordering among the revision's modules: each one's tier is one past the highest of the modules it waits for. in := map[string]bool{} @@ -177,8 +180,26 @@ func (r Reader) Tiers(revision uint64) (Tiers, error) { m := moved[n] tm := TierModule{Module: n, FP: m.FP2, None: m.None2, After: after[n], LeftOut: gr[n].LeftOut} tm.Waits = r.waitsOf(n, m, gr[n], wants, provided, perr, away) + if w, ok := held[n]; ok { + tm.Waits = append(tm.Waits, w) + } out.Tiers[t] = append(out.Tiers[t], tm) } + // A module the indexer holds back is not at the revision yet, so no tier of it names it: it is named in the first tier with + // its wait alone (ADR 0633 §5), with the fingerprint the indexer last took in. + for _, n := range heldAll { + if in[n] { + continue + } + tm := TierModule{Module: n, Waits: []TierWait{held[n]}} + if m, err := delivery.DecodeModule(modules[n]); err == nil { + tm.FP, tm.None = m.FP2, m.None2 + } + if len(out.Tiers) == 0 { + out.Tiers = append(out.Tiers, nil) + } + out.Tiers[0] = append(out.Tiers[0], tm) + } compact := out.Tiers[:0] for _, t := range out.Tiers { if len(t) > 0 { @@ -189,6 +210,37 @@ func (r Reader) Tiers(revision uint64) (Tiers, error) { return out, nil } +// TakeInEdge is the edge of a wait on the indexer a source revision or a group is held for (ADR 0633 §5). +const TakeInEdge = "take-in" + +// heldBack are the modules the indexer holds back, by module, each with its wait, read through the indexer's tool +// indexer_pending (ADR 0633 §5), and the same modules sorted. A tool that cannot be asked says so in out.Pending. +func (r Reader) heldBack(out *Tiers) (map[string]TierWait, []string) { + held := map[string]TierWait{} + if r.Pending == nil { + return held, nil + } + a, err := r.Pending() + if err != nil { + out.Pending = "the indexer's pending revisions cannot be read, so no module is named waiting for its revision's indexer: " + err.Error() + return held, nil + } + for _, w := range a.Waits { + words := "the indexer holds back " + w.Key + ", which waits for the indexer build " + w.WaitsFor + if w.What == delivery.TakeInGroup { + words = "the indexer gathers the group " + w.Group + " and its end has not come" + } else if w.Behind != "" { + words = "the indexer holds back " + w.Key + " behind " + w.Behind + } + for _, module := range w.Holds { + if _, ok := held[module]; !ok { // the oldest wait names it + held[module] = TierWait{Edge: TakeInEdge, Words: words, Since: w.Since} + } + } + } + return held, slices.Sorted(maps.Keys(held)) +} + // away are the nodes the watchman's presence says are offline. func (r Reader) away() map[string]bool { out := map[string]bool{} diff --git a/modules/mesh-delivery-view/manifest_test.go b/modules/mesh-delivery-view/manifest_test.go index 4e65b88f..b593a849 100644 --- a/modules/mesh-delivery-view/manifest_test.go +++ b/modules/mesh-delivery-view/manifest_test.go @@ -51,6 +51,7 @@ type manifest struct { Version string `json:"version"` Slug string `json:"slug"` Reads []string `json:"reads"` + Invokes []string `json:"invokes"` Uses []string `json:"uses"` Tools []string `json:"tools"` Emits []string `json:"emits"` @@ -122,6 +123,10 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 if want := []string{"view_status", "indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "indexer_predefined", "indexer_config_candidates", "indexer_packages", "builder_build", "builder_builds", "builder_history", "assigner_wants", "assigner_want", "assigner_differences", "node_haves", "node_trusts", "build_usage", "view_reach", "indexer_module_history", "indexer_heard", "view_tiers"}; !reflect.DeepEqual(m.Tools, want) { t.Errorf("tools = %v, want %v", m.Tools, want) } + // 1585/T24: view_tiers names a module waiting for its revision's indexer through the indexer's tool, a verb edge (ADR 0614 §2). + if !reflect.DeepEqual(m.Invokes, []string{"mesh-delivery-indexer.indexer_pending"}) { + t.Errorf("invokes = %v", m.Invokes) + } for _, a := range m.Build.Artifacts { // A bundle compiled to a binary is one executable: what the runtime loads is that binary's name, or the controller's // module check refuses the manifest (mesh-sdk catalogue build.go, ADR 0193). diff --git a/modules/mesh-delivery-view/module.json b/modules/mesh-delivery-view/module.json index 55bbbfb6..74236726 100644 --- a/modules/mesh-delivery-view/module.json +++ b/modules/mesh-delivery-view/module.json @@ -26,6 +26,9 @@ "indexer_heard", "view_tiers" ], + "invokes": [ + "mesh-delivery-indexer.indexer_pending" + ], "emits": [ "delivered", "moving", From 7609f2d9a2481aef858e0883d997149b1248044f Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 23:09:20 +0100 Subject: [PATCH 07/18] =?UTF-8?q?Watchman:=20a=20replayed=20pending=20that?= =?UTF-8?q?=20its=20taken=20ends=20is=20never=20raised;=20a=20drop=20then?= =?UTF-8?q?=20a=20pending=20again=20is=20covered=20(issue=201585,=20ADR=20?= =?UTF-8?q?0633=20=C2=A75)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn --- .../internal/watch/findings_test.go | 27 ++++++++++++++++++- .../internal/watch/takein.go | 9 ++++--- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/modules/mesh-delivery-watchman/internal/watch/findings_test.go b/modules/mesh-delivery-watchman/internal/watch/findings_test.go index 67cadbe5..accfbcfb 100644 --- a/modules/mesh-delivery-watchman/internal/watch/findings_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/findings_test.go @@ -75,9 +75,34 @@ func TestTheWatchmansFindings(t *testing.T) { } expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound) }, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}}, - {"ADR 0633 §5 (1585/T21): a pending event heard again at the start, since past its bound, is a finding at once", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + {"ADR 0633 §5 (1585/T21): a pending event heard again at the start, since past its bound, is said after the grace, not at once", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { w.TakeIn(takeInEvent(t, delivery.TakeInPending, revKey), takeInBody(t, delivery.TakeInPending, revKey, t0.Add(-2*time.Hour))) + if len(o.findings) != 0 || o.ttls[signalKey("take-in", revKey)] != TakeInReplayGrace { + t.Fatalf("a replayed old wait: findings %v, ttls %v", o.findings, o.ttls) + } + expire(w, c, signalKey("take-in", revKey), TakeInReplayGrace) }, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}}, + {"ADR 0633 §5 (1585/T21): a replayed pending followed by its taken never raises", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.TakeIn(takeInEvent(t, delivery.TakeInPending, revKey), takeInBody(t, delivery.TakeInPending, revKey, t0.Add(-2*time.Hour))) + w.TakeIn(takeInEvent(t, delivery.TakeInTaken, revKey), takeInBody(t, delivery.TakeInTaken, revKey, t0.Add(-2*time.Hour))) + if o.findingWrites != 0 || len(o.pulses) != 0 { + t.Fatalf("a wait that ended in the replay wrote %d findings and left pulses %v", o.findingWrites, o.pulses) + } + }, map[string]string{}}, + {"ADR 0633 §5 (1585/T21): a drop clears the wait, and the merges left under it said pending again with the same since are raised from it", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.TakeIn(takeInEvent(t, delivery.TakeInPending, revKey), takeInBody(t, delivery.TakeInPending, revKey, t0)) + expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound) + if len(o.findings) != 1 { + t.Fatalf("findings past the bound: %v", o.findings) + } + w.TakeIn(takeInEvent(t, delivery.TakeInTaken, revKey), takeInBody(t, delivery.TakeInTaken, revKey, t0)) + if len(o.findings) != 0 { + t.Fatalf("the drop left %v", o.findings) + } + next := "m.00000000000000000043" + w.TakeIn(takeInEvent(t, delivery.TakeInPending, next), takeInBody(t, delivery.TakeInPending, next, t0)) + expire(w, c, signalKey("take-in", next), TakeInReplayGrace) + }, map[string]string{FindingKey(ScopeDelivery, "m.00000000000000000043", KindTakeInWaits): Warning}}, {"ADR 0578 §1: a module whose value carries a refusal of its manifest is a finding until it is accepted", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { value := func(verdict string, problems ...string) []byte { b, err := delivery.EncodeModule(delivery.Module{FP2: "src2:" + strings.Repeat("a", 64), Manifest: strings.Repeat("b", 64), diff --git a/modules/mesh-delivery-watchman/internal/watch/takein.go b/modules/mesh-delivery-watchman/internal/watch/takein.go index 8d9abf23..d0bfb861 100644 --- a/modules/mesh-delivery-watchman/internal/watch/takein.go +++ b/modules/mesh-delivery-watchman/internal/watch/takein.go @@ -19,6 +19,11 @@ const ( TakeInGroupBound = 30 * time.Minute ) +// TakeInReplayGrace is how long a pending event whose bound has already passed waits before it is said: the stream keeps every +// event, so a restarted watchman hears a wait's pending event and then, in the same pass, the taken event that ended it, and +// that wait must never be raised. A wait still open after the grace is said. +const TakeInReplayGrace = 30 * time.Second + // takeInRow is the signals table's row of a take-in wait. const takeInRow = "take-in" @@ -61,9 +66,7 @@ func (w *Watchman) TakeIn(event string, body []byte) { Needs: "the indexer build it waits for to run, or the take-in by hand (indexer_take_in_pending), or a drop (indexer_drop_pending)"} left := bound - w.now().Sub(t.Since) if left <= 0 { - w.unsignal(takeInRow, key) - w.find(f) - return + left = TakeInReplayGrace } w.unfind(finding) w.signal(takeInRow, key, left, signalDue{Do: dueFind, Finding: f}) From ed4f07b0d8cf8eaae4c665fd90b2ff6b22180d0c Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 23:10:23 +0100 Subject: [PATCH 08/18] =?UTF-8?q?Watchman:=20a=20take-in=20wait=20past=20i?= =?UTF-8?q?ts=20bound=20heard=20in=20the=20replay=20is=20said=20once=20the?= =?UTF-8?q?=20consumer=20has=20caught=20up,=20not=20after=20a=20time=20(is?= =?UTF-8?q?sue=201585,=20ADR=200633=20=C2=A75)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn --- .../cmd/mesh-delivery-watchman/main.go | 12 ++++++ .../internal/watch/findings_test.go | 14 ++++--- .../internal/watch/takein.go | 37 ++++++++++++++----- .../internal/watch/watch.go | 5 ++- 4 files changed, 53 insertions(+), 15 deletions(-) diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go index 8117b1d9..239ad6e4 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go @@ -384,6 +384,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro logger.Printf("the consumer %s_%s on %s does not exist yet, so no take-in wait is said until the watchman starts again: %v", cred.Node, cred.Module, catalogue.StreamBusName(delivery.TakeInStream), err) default: + // The replay has caught up when a message says none is pending after it, or when the consumer had none pending at its + // bind (the stream holds nothing it has not acknowledged): the waits past their bound are said then, not before. + if info, err := tc.Info(ctx); err != nil || (info.NumPending == 0 && info.NumAckPending == 0) { + if err != nil { + logger.Printf("the take-in consumer cannot be read, so its replay is caught up at its last message: %v", err) + } else { + w.TakeInCaughtUp() + } + } hc, err := process.HearEvents(tc, func(m jetstream.Msg) { takeInHeard(w, m, logger.Printf) }, logger.Printf) if err != nil { return err @@ -464,6 +473,9 @@ func takeInHeard(w *watch.Watchman, m jetstream.Msg, logf func(string, ...any)) } else { w.TakeIn(event, m.Data()) } + if meta, err := m.Metadata(); err == nil && meta.NumPending == 0 { + w.TakeInCaughtUp() + } if err := m.Ack(); err != nil { logf("a take-in event could not be acknowledged: %v", err) } diff --git a/modules/mesh-delivery-watchman/internal/watch/findings_test.go b/modules/mesh-delivery-watchman/internal/watch/findings_test.go index accfbcfb..0454d76f 100644 --- a/modules/mesh-delivery-watchman/internal/watch/findings_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/findings_test.go @@ -35,6 +35,7 @@ func TestTheWatchmansFindings(t *testing.T) { want map[string]string }{ {"ADR 0633 §5 (1585/T21): a revision waiting for an indexer build is a finding past 60 minutes from its since, not within; a group past 30", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.TakeInCaughtUp() // a live message, heard after the replay w.TakeIn(takeInEvent(t, delivery.TakeInPending, revKey), takeInBody(t, delivery.TakeInPending, revKey, t0)) if o.ttls[signalKey("take-in", revKey)] != TakeInRevisionBound || len(o.findings) != 0 { t.Fatalf("a revision's wait: ttls %v, findings %v", o.ttls, o.findings) @@ -50,6 +51,7 @@ func TestTheWatchmansFindings(t *testing.T) { expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound-TakeInGroupBound) }, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning, FindingKey(ScopeDelivery, groupKey, KindTakeInWaits): Warning}}, {"ADR 0633 §5 (1585/T21): the take-in.taken of a named indexer, a hand take-in or a drop clears the finding", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.TakeInCaughtUp() // a live message, heard after the replay for _, by := range []string{"", "operator@shanks"} { w.TakeIn(takeInEvent(t, delivery.TakeInPending, revKey), takeInBody(t, delivery.TakeInPending, revKey, t0)) expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound) @@ -66,6 +68,7 @@ func TestTheWatchmansFindings(t *testing.T) { } }, map[string]string{}}, {"ADR 0633 §5 (1585/T21): a group's take-in.taken ends its 30 minutes, and the revision then pending under m. starts its own 60", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.TakeInCaughtUp() // a live message, heard after the replay w.TakeIn(takeInEvent(t, delivery.TakeInPending, groupKey), takeInBody(t, delivery.TakeInPending, groupKey, t0)) expire(w, c, signalKey("take-in", groupKey), 25*time.Minute) w.TakeIn(takeInEvent(t, delivery.TakeInTaken, groupKey), takeInBody(t, delivery.TakeInTaken, groupKey, t0)) @@ -75,21 +78,23 @@ func TestTheWatchmansFindings(t *testing.T) { } expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound) }, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}}, - {"ADR 0633 §5 (1585/T21): a pending event heard again at the start, since past its bound, is said after the grace, not at once", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + {"ADR 0633 §5 (1585/T21): a pending event heard again at the start, since past its bound, is said once the replay has caught up, not before", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { w.TakeIn(takeInEvent(t, delivery.TakeInPending, revKey), takeInBody(t, delivery.TakeInPending, revKey, t0.Add(-2*time.Hour))) - if len(o.findings) != 0 || o.ttls[signalKey("take-in", revKey)] != TakeInReplayGrace { - t.Fatalf("a replayed old wait: findings %v, ttls %v", o.findings, o.ttls) + if len(o.findings) != 0 || len(o.pulses) != 0 { + t.Fatalf("a replayed old wait before the replay caught up: findings %v, pulses %v", o.findings, o.pulses) } - expire(w, c, signalKey("take-in", revKey), TakeInReplayGrace) + w.TakeInCaughtUp() }, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}}, {"ADR 0633 §5 (1585/T21): a replayed pending followed by its taken never raises", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { w.TakeIn(takeInEvent(t, delivery.TakeInPending, revKey), takeInBody(t, delivery.TakeInPending, revKey, t0.Add(-2*time.Hour))) w.TakeIn(takeInEvent(t, delivery.TakeInTaken, revKey), takeInBody(t, delivery.TakeInTaken, revKey, t0.Add(-2*time.Hour))) + w.TakeInCaughtUp() if o.findingWrites != 0 || len(o.pulses) != 0 { t.Fatalf("a wait that ended in the replay wrote %d findings and left pulses %v", o.findingWrites, o.pulses) } }, map[string]string{}}, {"ADR 0633 §5 (1585/T21): a drop clears the wait, and the merges left under it said pending again with the same since are raised from it", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.TakeInCaughtUp() // a live message, heard after the replay w.TakeIn(takeInEvent(t, delivery.TakeInPending, revKey), takeInBody(t, delivery.TakeInPending, revKey, t0)) expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound) if len(o.findings) != 1 { @@ -101,7 +106,6 @@ func TestTheWatchmansFindings(t *testing.T) { } next := "m.00000000000000000043" w.TakeIn(takeInEvent(t, delivery.TakeInPending, next), takeInBody(t, delivery.TakeInPending, next, t0)) - expire(w, c, signalKey("take-in", next), TakeInReplayGrace) }, map[string]string{FindingKey(ScopeDelivery, "m.00000000000000000043", KindTakeInWaits): Warning}}, {"ADR 0578 §1: a module whose value carries a refusal of its manifest is a finding until it is accepted", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { value := func(verdict string, problems ...string) []byte { diff --git a/modules/mesh-delivery-watchman/internal/watch/takein.go b/modules/mesh-delivery-watchman/internal/watch/takein.go index d0bfb861..9cff07b2 100644 --- a/modules/mesh-delivery-watchman/internal/watch/takein.go +++ b/modules/mesh-delivery-watchman/internal/watch/takein.go @@ -2,6 +2,8 @@ package watch import ( "fmt" + "maps" + "slices" "strings" "time" @@ -19,11 +21,6 @@ const ( TakeInGroupBound = 30 * time.Minute ) -// TakeInReplayGrace is how long a pending event whose bound has already passed waits before it is said: the stream keeps every -// event, so a restarted watchman hears a wait's pending event and then, in the same pass, the taken event that ended it, and -// that wait must never be raised. A wait still open after the grace is said. -const TakeInReplayGrace = 30 * time.Second - // takeInRow is the signals table's row of a take-in wait. const takeInRow = "take-in" @@ -45,6 +42,7 @@ func (w *Watchman) TakeIn(event string, body []byte) { defer w.mu.Unlock() finding := FindingKey(ScopeDelivery, key, KindTakeInWaits) if kind == delivery.TakeInTaken { + delete(w.takeInHeld, key) w.unsignal(takeInRow, key) w.unfind(finding) return @@ -65,9 +63,30 @@ func (w *Watchman) TakeIn(event string, body []byte) { "not reach the machines.", Needs: "the indexer build it waits for to run, or the take-in by hand (indexer_take_in_pending), or a drop (indexer_drop_pending)"} left := bound - w.now().Sub(t.Since) - if left <= 0 { - left = TakeInReplayGrace - } w.unfind(finding) - w.signal(takeInRow, key, left, signalDue{Do: dueFind, Finding: f}) + w.unsignal(takeInRow, key) + delete(w.takeInHeld, key) + switch { + case left > 0: + w.signal(takeInRow, key, left, signalDue{Do: dueFind, Finding: f}) + case !w.takeInCaught: + // Past its bound while the stream is still being replayed: the taken event that ended it may be a few messages on, + // so it is said once the replay has caught up (TakeInCaughtUp), and a taken event before that cancels it. + w.takeInHeld[key] = f + default: + w.find(f) + } +} + +// TakeInCaughtUp tells the watchman its consumer has read the stream up to its end (the last message it delivered said no +// message pending, or the consumer had none pending at its bind): every wait past its bound that no taken event ended is +// said now, and the ones heard after are said as they come. +func (w *Watchman) TakeInCaughtUp() { + w.mu.Lock() + defer w.mu.Unlock() + w.takeInCaught = true + for _, key := range slices.Sorted(maps.Keys(w.takeInHeld)) { + w.find(w.takeInHeld[key]) + } + w.takeInHeld = map[string]Finding{} } diff --git a/modules/mesh-delivery-watchman/internal/watch/watch.go b/modules/mesh-delivery-watchman/internal/watch/watch.go index 67b641fe..e8bd18b9 100644 --- a/modules/mesh-delivery-watchman/internal/watch/watch.go +++ b/modules/mesh-delivery-watchman/internal/watch/watch.go @@ -74,6 +74,9 @@ type Watchman struct { seededFindings map[string]bool // signals is every signal's pulse standing, by pulse key, with what its expiry does. signals map[string]signalDue + // takeInHeld are the take-in waits past their bound heard while the stream replays, said once it has caught up (takein.go). + takeInHeld map[string]Finding + takeInCaught bool // away is each machine that said it sleeps or shuts down, until it says it is back. away map[string]bool // haveState is each module's have as last read, unhealthy its writes in a row that said it failed, advised each @@ -107,7 +110,7 @@ type Watchman struct { func New(out Out, now func() time.Time) *Watchman { return &Watchman{out: out, now: now, counts: map[string]int{}, presence: map[string]presence{}, pulses: map[string]*pulse{}, stuck: map[string]delivery.Stuck{}, wants: map[string]string{}, haves: map[string]string{}, held: map[string]bool{}, starts: map[string]startMark{}, bound: delivery.PulseMoveBound, stepBound: StepBound, - findings: map[string][]byte{}, seededFindings: map[string]bool{}, signals: map[string]signalDue{}, away: map[string]bool{}, + findings: map[string][]byte{}, seededFindings: map[string]bool{}, signals: map[string]signalDue{}, takeInHeld: map[string]Finding{}, away: map[string]bool{}, haveState: map[string]delivery.Have{}, unhealthy: map[string]int{}, advised: map[string]int{}, advisedSince: map[string]time.Time{}, control: map[string]bool{}, failedSince: map[string]time.Time{}, provFailed: map[string]map[string]ProvReport{}, closures: map[string]map[string]time.Time{}, waiting: map[string]map[string]string{}, health: map[string]delivery.NodeHealth{}, tunnels: map[string]string{}, From d6b6f24e98d2fd2c56df6080f08bd07957a293c0 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 23:11:53 +0100 Subject: [PATCH 09/18] =?UTF-8?q?forge=20bridge:=20a=20member's=20merged?= =?UTF-8?q?=20message=20names=20its=20group,=20and=20the=20group's=20end?= =?UTF-8?q?=20is=20said=20once=20(ADR=200633=20=C2=A74,=20issue=201585)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The group is read from the gate's merges state the bridge already reads; the end comes only once every approved member is merged and its merged message written, never on a refusal, with a message id the stream's duplicate window holds to one. Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn --- .../cmd/mesh-delivery-forge-bridge/groups.go | 215 ++++++++++++++++++ .../mesh-delivery-forge-bridge/groups_test.go | 94 ++++++++ .../cmd/mesh-delivery-forge-bridge/main.go | 26 ++- .../mesh-delivery-forge-bridge/main_test.go | 9 +- 4 files changed, 337 insertions(+), 7 deletions(-) create mode 100644 modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups.go create mode 100644 modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups_test.go diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups.go new file mode 100644 index 00000000..486990b9 --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups.go @@ -0,0 +1,215 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "log" + "slices" + "strconv" + "strings" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// A delivery group's merges reach the indexer together (novox/hq ADR 0633 §4): the bridge sets the group on each +// member's merged message, from the gate's merges state it reads (ADR 0437 §8), and once every approved member is +// merged and its merged message written on FORGE it says the group's end, forge.group.merged, once. +// +// What it keeps of it is in its state performed, beside the gate's keys it acted on: `published.` for each +// member's merged message written (the hash is the first 16 hex digits of the sha256 of `#@`), +// and `groups.` once the group's end is written, its value the revision of the gate's merges key it was +// worked out from. A group whose end is not written yet is open (bridge_groups reads them). + +// groupsKey is the key of performed that says a group's end is written. +func groupsKey(mergesKey string) string { return "groups." + mergesKey } + +// publishedKey is the key of performed that says a member's merged message is written. +func publishedKey(repository string, number int, merge string) string { + sum := sha256.Sum256([]byte(fmt.Sprintf("%s#%d@%s", strings.ToLower(repository), number, merge))) + return "published." + hex.EncodeToString(sum[:8]) +} + +// groupEndID is the message id of a group's end: the stream's duplicate window drops a second publish of one end. +func groupEndID(group, lastMerge string) string { return "group-end." + group + "." + lastMerge } + +// merges reads one key of the gate's merges state. +type mergesRead func(ctx context.Context, key string) (value []byte, rev uint64, found bool, err error) + +// grouper sets a member's group and says a group's end. +type grouper struct { + merges mergesRead + send func(ctx context.Context, subject string, data []byte, id string) error // writes on FORGE with the message id + done revisions + logger *log.Logger +} + +// sendVia writes on FORGE through pub, with the message id. +func sendVia(pub publisher) func(context.Context, string, []byte, string) error { + return func(ctx context.Context, subject string, data []byte, id string) error { + _, err := pub.Publish(ctx, subject, data, jetstream.WithMsgID(id)) + return err + } +} + +func kvMerges(kv jetstream.KeyValue) mergesRead { + return func(ctx context.Context, key string) ([]byte, uint64, bool, error) { + e, err := kv.Get(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, 0, false, nil + } + if err != nil { + return nil, 0, false, err + } + return e.Value(), e.Revision(), true, nil + } +} + +// groupOf is the gate's merge of the group a merged message's branch names, when the message's pull request is one of +// its approved members and the group has more than one: a lone pull request is no group. +func (g *grouper) groupOf(ctx context.Context, f delivery.Forge) (delivery.Merge, uint64, bool) { + if g == nil || g.merges == nil || f.Branch == "" { + return delivery.Merge{}, 0, false + } + key, err := delivery.MergesKey(f.Branch) + if err != nil { + return delivery.Merge{}, 0, false + } + value, rev, found, err := g.merges(ctx, key) + if err != nil { + g.logger.Printf("%s#%d: the gate's merges.%s cannot be read, so its merged message names no group: %v", f.Repository, f.Number, key, err) + return delivery.Merge{}, 0, false + } + if !found { + return delivery.Merge{}, 0, false + } + m, err := delivery.DecodeMerge(value) + if err != nil { + g.logger.Printf("%s#%d: the gate's merges.%s is not a merge, so its merged message names no group: %v", f.Repository, f.Number, key, err) + return delivery.Merge{}, 0, false + } + if _, ok := m.Approved[delivery.ApprovedKey(f.Repository, f.Number)]; !ok || len(m.Approved) < 2 { + return delivery.Merge{}, 0, false + } + return m, rev, true +} + +// membersOf are a merge's approved members with their heads, in the order of their keys. +func membersOf(m delivery.Merge) []delivery.Head { + var out []delivery.Head + for _, k := range slices.Sorted(func(yield func(string) bool) { + for k := range m.Approved { + if !yield(k) { + return + } + } + }) { + at := strings.LastIndex(k, "@") + if at < 0 { + continue + } + n, err := strconv.Atoi(k[at+1:]) + if err != nil { + continue + } + out = append(out, delivery.Head{Repository: k[:at], Number: n, Head: m.Approved[k]}) + } + return out +} + +// member sets the group and its approved members on a merged message's body (ADR 0633 §4); any other body, and a +// merge of no group, is left as it is. +func (g *grouper) member(ctx context.Context, data []byte) ([]byte, delivery.Forge, bool) { + f, err := delivery.DecodeForge(data) + if err != nil || f.Kind != delivery.ForgeMerged { + return data, f, false + } + m, _, ok := g.groupOf(ctx, f) + if !ok { + return data, f, false + } + f.Group, f.Members = m.Group, membersOf(m) + out, err := delivery.EncodeForge(f) + if err != nil { + g.logger.Printf("%s#%d: its merged message cannot name the group %q, written without: %v", f.Repository, f.Number, m.Group, err) + return data, f, false + } + return out, f, true +} + +// published keeps that a member's merged message is written, and says the group's end when it was the last. +func (g *grouper) published(ctx context.Context, f delivery.Forge) { + if err := g.done.Put(ctx, publishedKey(f.Repository, f.Number, f.Merge), 1); err != nil { + g.logger.Printf("%s#%d: that its merged message is written is not kept, so the group's end waits for the next act: %v", f.Repository, f.Number, err) + return + } + key, err := delivery.MergesKey(f.Branch) + if err != nil { + return + } + value, rev, found, err := g.merges(ctx, key) + if err != nil || !found { + return + } + g.ended(ctx, key, rev, value) +} + +// ended says the group's end (forge.group.merged) once every approved member of the gate's merge is merged and its +// merged message written, after the last of them; never on a refusal (ADR 0441 §3: a held group merges on from it). +// It is written once per group: `groups.` in performed says it was. again says it is written again by hand +// (bridge_group_end_again), with the time appended to the message id, so the duplicate window does not drop it. +func (g *grouper) ended(ctx context.Context, key string, rev uint64, value []byte) { + g.end(ctx, key, rev, value, "") +} + +func (g *grouper) end(ctx context.Context, key string, rev uint64, value []byte, again string) bool { + m, err := delivery.DecodeMerge(value) + if err != nil || len(m.Approved) < 2 { + return false + } + if again == "" { + if _, seen, err := g.done.Get(ctx, groupsKey(key)); err == nil && seen { + return false + } + } + end := delivery.ForgeGroupEnd{Group: m.Group} + for _, member := range membersOf(m) { + at := slices.IndexFunc(m.Pulls, func(p delivery.MergePull) bool { + return delivery.SameRepository(p.Repository, member.Repository) && p.Number == member.Number + }) + if at < 0 || m.Pulls[at].State != delivery.MergeMerged || m.Pulls[at].Commit == "" { + return false // not every member is merged: the group stays gathered + } + if _, ok, err := g.done.Get(ctx, publishedKey(member.Repository, member.Number, m.Pulls[at].Commit)); err != nil || !ok { + return false // its merged message is not written yet: said when it is (published) + } + end.Members = append(end.Members, delivery.GroupMerge{Repository: member.Repository, Number: member.Number, Merge: m.Pulls[at].Commit}) + } + // The members in the order the gate merged them, so the last commit names the end. + slices.SortStableFunc(end.Members, func(a, b delivery.GroupMerge) int { + return slices.IndexFunc(m.Pulls, func(p delivery.MergePull) bool { return p.Commit == a.Merge }) - + slices.IndexFunc(m.Pulls, func(p delivery.MergePull) bool { return p.Commit == b.Merge }) + }) + data, err := delivery.EncodeForgeGroupEnd(end) + if err != nil { + g.logger.Printf("the end of the group %q cannot be written: %v", m.Group, err) + return false + } + id := groupEndID(m.Group, end.Members[len(end.Members)-1].Merge) + if again != "" { + id += "." + again + } + if err := g.send(ctx, forgeSubject(delivery.ForgeGroupEndEvent), data, id); err != nil { + g.logger.Printf("the end of the group %q is not written on FORGE, written at the gate's next write or by bridge_group_end_again: %v", m.Group, err) + return false + } + if err := g.done.Put(ctx, groupsKey(key), rev); err != nil { + g.logger.Printf("the end of the group %q is written and not kept: the stream's duplicate window drops a second one: %v", m.Group, err) + } + g.logger.Printf("the end of the group %q is written: %d members merged, the last %s", m.Group, len(end.Members), end.Members[len(end.Members)-1].Merge) + return true +} diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups_test.go new file mode 100644 index 00000000..419c259e --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups_test.go @@ -0,0 +1,94 @@ +package main + +import ( + "bytes" + "context" + "log" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// idPub records the subjects, bodies and message ids it is handed. +type idPub struct { + subjects, ids []string + data [][]byte +} + +func (p *idPub) send(_ context.Context, subject string, data []byte, id string) error { + p.subjects, p.data, p.ids = append(p.subjects, subject), append(p.data, data), append(p.ids, id) + return nil +} + +func TestAGroupsMembersNameItAndItsEndIsSaidOnce(t *testing.T) { // 1585/T16, T17 (ADR 0633 §4) + ctx := context.Background() + h1, h2 := strings.Repeat("1", 40), strings.Repeat("2", 40) + c1, c2 := strings.Repeat("c", 40), strings.Repeat("d", 40) + merge := func(states ...delivery.MergePull) []byte { + v, err := delivery.EncodeMerge(delivery.Merge{Group: "feat/take-in", Ask: "ask", Pulls: states, + Approved: map[string]string{"novox/mesh-sdk@37": h1, "novox/mesh-delivery@9": h2}}) + if err != nil { + t.Fatal(err) + } + return v + } + sdk := func(state delivery.MergeState, commit, words string) delivery.MergePull { + return delivery.MergePull{Head: delivery.Head{Repository: "novox/mesh-sdk", Number: 37, Head: h1}, State: state, Commit: commit, Words: words} + } + dlv := func(state delivery.MergeState, commit string) delivery.MergePull { + return delivery.MergePull{Head: delivery.Head{Repository: "novox/mesh-delivery", Number: 9, Head: h2}, State: state, Commit: commit} + } + gate := map[string][]byte{} + var logs bytes.Buffer + pub, done := &idPub{}, memRevisions{} + g := &grouper{send: pub.send, done: done, logger: log.New(&logs, "", 0), + merges: func(_ context.Context, key string) ([]byte, uint64, bool, error) { + v, ok := gate[key] + return v, 7, ok, nil + }} + mergedOf := func(repo string, number int, head, commit string) []byte { + data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: repo, Branch: "feat/take-in", Number: number, Head: head, Merge: commit, Root: "r"}) + if err != nil { + t.Fatal(err) + } + return data + } + // A lone pull request's merged message carries no group. + if _, f, ok := g.member(ctx, mergedOf("novox/hq", 4, h1, c1)); ok || f.Group != "" { + t.Errorf("a lone pull request names the group %q", f.Group) + } + // The first member: its message names the group and both members; the group is not ended. + gate["feat/take-in"] = merge(sdk(delivery.MergeMerged, c1, ""), dlv(delivery.MergeMerging, "")) + data, f, ok := g.member(ctx, mergedOf("novox/mesh-sdk", 37, h1, c1)) + if back, err := delivery.DecodeForge(data); !ok || err != nil || back.Group != "feat/take-in" || len(back.Members) != 2 { + t.Fatalf("a member's message: %+v %v", back, err) + } + g.published(ctx, f) + // A refusal of the other member is no end (ADR 0441 §3: the group merges on from it later). + gate["feat/take-in"] = merge(sdk(delivery.MergeMerged, c1, ""), delivery.MergePull{Head: delivery.Head{Repository: "novox/mesh-delivery", Number: 9, Head: h2}, State: delivery.MergeRefused, Words: "405"}) + g.ended(ctx, "feat/take-in", 8, gate["feat/take-in"]) + // Merged on at the gate before its merged message is written: still no end; then its message ends the group. + gate["feat/take-in"] = merge(sdk(delivery.MergeMerged, c1, ""), dlv(delivery.MergeMerged, c2)) + g.ended(ctx, "feat/take-in", 9, gate["feat/take-in"]) + if len(pub.subjects) != 0 { + t.Fatalf("an end before every member's merged message is written: %v", pub.subjects) + } + _, f, _ = g.member(ctx, mergedOf("novox/mesh-delivery", 9, h2, c2)) + g.published(ctx, f) + g.ended(ctx, "feat/take-in", 10, gate["feat/take-in"]) // a later write of the gate says no second end + if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.group.merged" || pub.ids[0] != "group-end.feat/take-in."+c2 { + t.Fatalf("ends %v ids %v, want one forge.group.merged with id group-end.feat/take-in.%s", pub.subjects, pub.ids, c2) + } + end, err := delivery.DecodeForgeGroupEnd(pub.data[0]) + if err != nil || len(end.Members) != 2 || end.Members[0].Merge != c1 || end.Members[1].Merge != c2 { + t.Errorf("the end = %+v %v, want both members in their merge order", end, err) + } + if rev, ok := done[groupsKey("feat/take-in")]; !ok || rev != 7 { + t.Errorf("performed %v: want groups.feat/take-in kept", done) + } + // Again by hand: written once more, its id not the first's. + if !g.end(ctx, "feat/take-in", 10, gate["feat/take-in"], "20261104T2300") || pub.ids[1] == pub.ids[0] { + t.Errorf("the end again by hand: ids %v", pub.ids) + } +} diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go index c8bb3b00..c630e424 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go @@ -68,6 +68,12 @@ type publisher interface { // redelivered, and the log names its subject and why. A message it could not write stays unacknowledged: the server // delivers it again (the server's ack wait is the only clock). func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, kept bridge.Kept, logger *log.Logger) { + handleGrouped(ctx, m, pub, trees, files, kept, nil, logger) +} + +// handleGrouped is handle; a merged message of a delivery group's member names its group (g, ADR 0633 §4; nil names +// none), and the member's merged message written may end the group. +func handleGrouped(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, kept bridge.Kept, g *grouper, logger *log.Logger) { msg, err := bridge.ConvertKept(m.Subject(), m.Data(), trees, files, kept) switch { case errors.Is(err, bridge.ErrRetry): @@ -88,10 +94,18 @@ func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, if id := m.Headers().Get("x-event-id"); id != "" { opts = append(opts, jetstream.WithMsgID(id)) } + var member delivery.Forge + grouped := false + if g != nil { + msg.Data, member, grouped = g.member(ctx, msg.Data) + } if _, err := pub.Publish(ctx, forgeSubject(msg.Event), msg.Data, opts...); err != nil { logger.Printf("%s: %s could not be written on FORGE, left for redelivery: %v", m.Subject(), msg.Event, err) return } + if grouped { + g.published(ctx, member) + } if msg.Said != "" { logger.Printf("%s: %s", m.Subject(), msg.Said) } @@ -136,17 +150,25 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } perf := &performer{call: call, pub: rec, done: kvRevisions{performedKV}, logger: logger} + // A delivery group's member names its group, and its end is said once (ADR 0633 §4), from the gate's merges. + groups := &grouper{send: sendVia(js), done: kvRevisions{performedKV}, logger: logger} watching := make(chan error, 3) for _, w := range []struct { bucket string react func(ctx context.Context, key string, rev uint64, value []byte) - }{{delivery.OwedBucket, perf.owed}, {delivery.MergesBucket, perf.merges}} { + }{{delivery.OwedBucket, perf.owed}, {delivery.MergesBucket, func(ctx context.Context, key string, rev uint64, value []byte) { + perf.merges(ctx, key, rev, value) + groups.ended(ctx, key, rev, value) + }}} { // The gate's buckets come with the gate and its grant; the bridge does not wait for them (no timer), and is started // again by its delivery when the gate arrives. A bucket refused or not answered is the same (issue 1945). kv := process.OptionalBucket(ctx, js, w.bucket, "nothing of it is performed until the bridge starts again", logger.Printf) if kv == nil { continue } + if w.bucket == delivery.MergesBucket { + groups.merges = kvMerges(kv) + } go func() { watching <- watch(ctx, kv, func(key string, rev uint64, value []byte) { hctx, cancel := context.WithTimeout(ctx, syncDeadline) @@ -171,7 +193,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // Bounded: the turn is held, and no sync or other event goes on without it. hctx, cancel := context.WithTimeout(ctx, syncDeadline) defer cancel() - handle(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), inputs, logger) + handleGrouped(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), inputs, groups, logger) }) }, logger.Printf) if err != nil { diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go index 85df5828..6234a90d 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go @@ -107,11 +107,10 @@ func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5 if !strings.Contains(logs.String(), "mesh.mod.gitea.event.pull.updated") || !strings.Contains(logs.String(), "head") { t.Errorf("log %q names neither the subject nor why", logs.String()) } -} - -func TestUnwrittenMessageIsNotAcked(t *testing.T) { // 1235 (a failed write is redelivered by the server, not retried here) - var logs bytes.Buffer - m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{err: errors.New("no stream")} + // 1235: an unwritten message is not acked (a failed write is redelivered by the server, not retried here); merged here + // from a test of its own, for the test budget. + logs.Reset() + m, pub = &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{err: errors.New("no stream")} handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0)) if m.acked != 0 || m.termd != 0 { t.Errorf("acked %d terminated %d, want neither", m.acked, m.termd) From a762701e3dcfb1a5e6febf549271cd3dcc301936 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 23:11:53 +0100 Subject: [PATCH 10/18] =?UTF-8?q?forge=20bridge:=20a=20member's=20merged?= =?UTF-8?q?=20message=20names=20its=20group,=20and=20the=20group's=20end?= =?UTF-8?q?=20is=20said=20once=20(ADR=200633=20=C2=A74,=20issue=201585)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The group is read from the gate's merges state the bridge already reads; the end comes only once every approved member is merged and its merged message written, never on a refusal, with a message id the stream's duplicate window holds to one. Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn --- .../cmd/mesh-delivery-forge-bridge/groups.go | 215 ++++++++++++++++++ .../mesh-delivery-forge-bridge/groups_test.go | 94 ++++++++ .../cmd/mesh-delivery-forge-bridge/main.go | 26 ++- .../mesh-delivery-forge-bridge/main_test.go | 9 +- 4 files changed, 337 insertions(+), 7 deletions(-) create mode 100644 modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups.go create mode 100644 modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups_test.go diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups.go new file mode 100644 index 00000000..486990b9 --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups.go @@ -0,0 +1,215 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "log" + "slices" + "strconv" + "strings" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// A delivery group's merges reach the indexer together (novox/hq ADR 0633 §4): the bridge sets the group on each +// member's merged message, from the gate's merges state it reads (ADR 0437 §8), and once every approved member is +// merged and its merged message written on FORGE it says the group's end, forge.group.merged, once. +// +// What it keeps of it is in its state performed, beside the gate's keys it acted on: `published.` for each +// member's merged message written (the hash is the first 16 hex digits of the sha256 of `#@`), +// and `groups.` once the group's end is written, its value the revision of the gate's merges key it was +// worked out from. A group whose end is not written yet is open (bridge_groups reads them). + +// groupsKey is the key of performed that says a group's end is written. +func groupsKey(mergesKey string) string { return "groups." + mergesKey } + +// publishedKey is the key of performed that says a member's merged message is written. +func publishedKey(repository string, number int, merge string) string { + sum := sha256.Sum256([]byte(fmt.Sprintf("%s#%d@%s", strings.ToLower(repository), number, merge))) + return "published." + hex.EncodeToString(sum[:8]) +} + +// groupEndID is the message id of a group's end: the stream's duplicate window drops a second publish of one end. +func groupEndID(group, lastMerge string) string { return "group-end." + group + "." + lastMerge } + +// merges reads one key of the gate's merges state. +type mergesRead func(ctx context.Context, key string) (value []byte, rev uint64, found bool, err error) + +// grouper sets a member's group and says a group's end. +type grouper struct { + merges mergesRead + send func(ctx context.Context, subject string, data []byte, id string) error // writes on FORGE with the message id + done revisions + logger *log.Logger +} + +// sendVia writes on FORGE through pub, with the message id. +func sendVia(pub publisher) func(context.Context, string, []byte, string) error { + return func(ctx context.Context, subject string, data []byte, id string) error { + _, err := pub.Publish(ctx, subject, data, jetstream.WithMsgID(id)) + return err + } +} + +func kvMerges(kv jetstream.KeyValue) mergesRead { + return func(ctx context.Context, key string) ([]byte, uint64, bool, error) { + e, err := kv.Get(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, 0, false, nil + } + if err != nil { + return nil, 0, false, err + } + return e.Value(), e.Revision(), true, nil + } +} + +// groupOf is the gate's merge of the group a merged message's branch names, when the message's pull request is one of +// its approved members and the group has more than one: a lone pull request is no group. +func (g *grouper) groupOf(ctx context.Context, f delivery.Forge) (delivery.Merge, uint64, bool) { + if g == nil || g.merges == nil || f.Branch == "" { + return delivery.Merge{}, 0, false + } + key, err := delivery.MergesKey(f.Branch) + if err != nil { + return delivery.Merge{}, 0, false + } + value, rev, found, err := g.merges(ctx, key) + if err != nil { + g.logger.Printf("%s#%d: the gate's merges.%s cannot be read, so its merged message names no group: %v", f.Repository, f.Number, key, err) + return delivery.Merge{}, 0, false + } + if !found { + return delivery.Merge{}, 0, false + } + m, err := delivery.DecodeMerge(value) + if err != nil { + g.logger.Printf("%s#%d: the gate's merges.%s is not a merge, so its merged message names no group: %v", f.Repository, f.Number, key, err) + return delivery.Merge{}, 0, false + } + if _, ok := m.Approved[delivery.ApprovedKey(f.Repository, f.Number)]; !ok || len(m.Approved) < 2 { + return delivery.Merge{}, 0, false + } + return m, rev, true +} + +// membersOf are a merge's approved members with their heads, in the order of their keys. +func membersOf(m delivery.Merge) []delivery.Head { + var out []delivery.Head + for _, k := range slices.Sorted(func(yield func(string) bool) { + for k := range m.Approved { + if !yield(k) { + return + } + } + }) { + at := strings.LastIndex(k, "@") + if at < 0 { + continue + } + n, err := strconv.Atoi(k[at+1:]) + if err != nil { + continue + } + out = append(out, delivery.Head{Repository: k[:at], Number: n, Head: m.Approved[k]}) + } + return out +} + +// member sets the group and its approved members on a merged message's body (ADR 0633 §4); any other body, and a +// merge of no group, is left as it is. +func (g *grouper) member(ctx context.Context, data []byte) ([]byte, delivery.Forge, bool) { + f, err := delivery.DecodeForge(data) + if err != nil || f.Kind != delivery.ForgeMerged { + return data, f, false + } + m, _, ok := g.groupOf(ctx, f) + if !ok { + return data, f, false + } + f.Group, f.Members = m.Group, membersOf(m) + out, err := delivery.EncodeForge(f) + if err != nil { + g.logger.Printf("%s#%d: its merged message cannot name the group %q, written without: %v", f.Repository, f.Number, m.Group, err) + return data, f, false + } + return out, f, true +} + +// published keeps that a member's merged message is written, and says the group's end when it was the last. +func (g *grouper) published(ctx context.Context, f delivery.Forge) { + if err := g.done.Put(ctx, publishedKey(f.Repository, f.Number, f.Merge), 1); err != nil { + g.logger.Printf("%s#%d: that its merged message is written is not kept, so the group's end waits for the next act: %v", f.Repository, f.Number, err) + return + } + key, err := delivery.MergesKey(f.Branch) + if err != nil { + return + } + value, rev, found, err := g.merges(ctx, key) + if err != nil || !found { + return + } + g.ended(ctx, key, rev, value) +} + +// ended says the group's end (forge.group.merged) once every approved member of the gate's merge is merged and its +// merged message written, after the last of them; never on a refusal (ADR 0441 §3: a held group merges on from it). +// It is written once per group: `groups.` in performed says it was. again says it is written again by hand +// (bridge_group_end_again), with the time appended to the message id, so the duplicate window does not drop it. +func (g *grouper) ended(ctx context.Context, key string, rev uint64, value []byte) { + g.end(ctx, key, rev, value, "") +} + +func (g *grouper) end(ctx context.Context, key string, rev uint64, value []byte, again string) bool { + m, err := delivery.DecodeMerge(value) + if err != nil || len(m.Approved) < 2 { + return false + } + if again == "" { + if _, seen, err := g.done.Get(ctx, groupsKey(key)); err == nil && seen { + return false + } + } + end := delivery.ForgeGroupEnd{Group: m.Group} + for _, member := range membersOf(m) { + at := slices.IndexFunc(m.Pulls, func(p delivery.MergePull) bool { + return delivery.SameRepository(p.Repository, member.Repository) && p.Number == member.Number + }) + if at < 0 || m.Pulls[at].State != delivery.MergeMerged || m.Pulls[at].Commit == "" { + return false // not every member is merged: the group stays gathered + } + if _, ok, err := g.done.Get(ctx, publishedKey(member.Repository, member.Number, m.Pulls[at].Commit)); err != nil || !ok { + return false // its merged message is not written yet: said when it is (published) + } + end.Members = append(end.Members, delivery.GroupMerge{Repository: member.Repository, Number: member.Number, Merge: m.Pulls[at].Commit}) + } + // The members in the order the gate merged them, so the last commit names the end. + slices.SortStableFunc(end.Members, func(a, b delivery.GroupMerge) int { + return slices.IndexFunc(m.Pulls, func(p delivery.MergePull) bool { return p.Commit == a.Merge }) - + slices.IndexFunc(m.Pulls, func(p delivery.MergePull) bool { return p.Commit == b.Merge }) + }) + data, err := delivery.EncodeForgeGroupEnd(end) + if err != nil { + g.logger.Printf("the end of the group %q cannot be written: %v", m.Group, err) + return false + } + id := groupEndID(m.Group, end.Members[len(end.Members)-1].Merge) + if again != "" { + id += "." + again + } + if err := g.send(ctx, forgeSubject(delivery.ForgeGroupEndEvent), data, id); err != nil { + g.logger.Printf("the end of the group %q is not written on FORGE, written at the gate's next write or by bridge_group_end_again: %v", m.Group, err) + return false + } + if err := g.done.Put(ctx, groupsKey(key), rev); err != nil { + g.logger.Printf("the end of the group %q is written and not kept: the stream's duplicate window drops a second one: %v", m.Group, err) + } + g.logger.Printf("the end of the group %q is written: %d members merged, the last %s", m.Group, len(end.Members), end.Members[len(end.Members)-1].Merge) + return true +} diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups_test.go new file mode 100644 index 00000000..419c259e --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups_test.go @@ -0,0 +1,94 @@ +package main + +import ( + "bytes" + "context" + "log" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// idPub records the subjects, bodies and message ids it is handed. +type idPub struct { + subjects, ids []string + data [][]byte +} + +func (p *idPub) send(_ context.Context, subject string, data []byte, id string) error { + p.subjects, p.data, p.ids = append(p.subjects, subject), append(p.data, data), append(p.ids, id) + return nil +} + +func TestAGroupsMembersNameItAndItsEndIsSaidOnce(t *testing.T) { // 1585/T16, T17 (ADR 0633 §4) + ctx := context.Background() + h1, h2 := strings.Repeat("1", 40), strings.Repeat("2", 40) + c1, c2 := strings.Repeat("c", 40), strings.Repeat("d", 40) + merge := func(states ...delivery.MergePull) []byte { + v, err := delivery.EncodeMerge(delivery.Merge{Group: "feat/take-in", Ask: "ask", Pulls: states, + Approved: map[string]string{"novox/mesh-sdk@37": h1, "novox/mesh-delivery@9": h2}}) + if err != nil { + t.Fatal(err) + } + return v + } + sdk := func(state delivery.MergeState, commit, words string) delivery.MergePull { + return delivery.MergePull{Head: delivery.Head{Repository: "novox/mesh-sdk", Number: 37, Head: h1}, State: state, Commit: commit, Words: words} + } + dlv := func(state delivery.MergeState, commit string) delivery.MergePull { + return delivery.MergePull{Head: delivery.Head{Repository: "novox/mesh-delivery", Number: 9, Head: h2}, State: state, Commit: commit} + } + gate := map[string][]byte{} + var logs bytes.Buffer + pub, done := &idPub{}, memRevisions{} + g := &grouper{send: pub.send, done: done, logger: log.New(&logs, "", 0), + merges: func(_ context.Context, key string) ([]byte, uint64, bool, error) { + v, ok := gate[key] + return v, 7, ok, nil + }} + mergedOf := func(repo string, number int, head, commit string) []byte { + data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: repo, Branch: "feat/take-in", Number: number, Head: head, Merge: commit, Root: "r"}) + if err != nil { + t.Fatal(err) + } + return data + } + // A lone pull request's merged message carries no group. + if _, f, ok := g.member(ctx, mergedOf("novox/hq", 4, h1, c1)); ok || f.Group != "" { + t.Errorf("a lone pull request names the group %q", f.Group) + } + // The first member: its message names the group and both members; the group is not ended. + gate["feat/take-in"] = merge(sdk(delivery.MergeMerged, c1, ""), dlv(delivery.MergeMerging, "")) + data, f, ok := g.member(ctx, mergedOf("novox/mesh-sdk", 37, h1, c1)) + if back, err := delivery.DecodeForge(data); !ok || err != nil || back.Group != "feat/take-in" || len(back.Members) != 2 { + t.Fatalf("a member's message: %+v %v", back, err) + } + g.published(ctx, f) + // A refusal of the other member is no end (ADR 0441 §3: the group merges on from it later). + gate["feat/take-in"] = merge(sdk(delivery.MergeMerged, c1, ""), delivery.MergePull{Head: delivery.Head{Repository: "novox/mesh-delivery", Number: 9, Head: h2}, State: delivery.MergeRefused, Words: "405"}) + g.ended(ctx, "feat/take-in", 8, gate["feat/take-in"]) + // Merged on at the gate before its merged message is written: still no end; then its message ends the group. + gate["feat/take-in"] = merge(sdk(delivery.MergeMerged, c1, ""), dlv(delivery.MergeMerged, c2)) + g.ended(ctx, "feat/take-in", 9, gate["feat/take-in"]) + if len(pub.subjects) != 0 { + t.Fatalf("an end before every member's merged message is written: %v", pub.subjects) + } + _, f, _ = g.member(ctx, mergedOf("novox/mesh-delivery", 9, h2, c2)) + g.published(ctx, f) + g.ended(ctx, "feat/take-in", 10, gate["feat/take-in"]) // a later write of the gate says no second end + if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.group.merged" || pub.ids[0] != "group-end.feat/take-in."+c2 { + t.Fatalf("ends %v ids %v, want one forge.group.merged with id group-end.feat/take-in.%s", pub.subjects, pub.ids, c2) + } + end, err := delivery.DecodeForgeGroupEnd(pub.data[0]) + if err != nil || len(end.Members) != 2 || end.Members[0].Merge != c1 || end.Members[1].Merge != c2 { + t.Errorf("the end = %+v %v, want both members in their merge order", end, err) + } + if rev, ok := done[groupsKey("feat/take-in")]; !ok || rev != 7 { + t.Errorf("performed %v: want groups.feat/take-in kept", done) + } + // Again by hand: written once more, its id not the first's. + if !g.end(ctx, "feat/take-in", 10, gate["feat/take-in"], "20261104T2300") || pub.ids[1] == pub.ids[0] { + t.Errorf("the end again by hand: ids %v", pub.ids) + } +} diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go index c8bb3b00..c630e424 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go @@ -68,6 +68,12 @@ type publisher interface { // redelivered, and the log names its subject and why. A message it could not write stays unacknowledged: the server // delivers it again (the server's ack wait is the only clock). func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, kept bridge.Kept, logger *log.Logger) { + handleGrouped(ctx, m, pub, trees, files, kept, nil, logger) +} + +// handleGrouped is handle; a merged message of a delivery group's member names its group (g, ADR 0633 §4; nil names +// none), and the member's merged message written may end the group. +func handleGrouped(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, kept bridge.Kept, g *grouper, logger *log.Logger) { msg, err := bridge.ConvertKept(m.Subject(), m.Data(), trees, files, kept) switch { case errors.Is(err, bridge.ErrRetry): @@ -88,10 +94,18 @@ func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, if id := m.Headers().Get("x-event-id"); id != "" { opts = append(opts, jetstream.WithMsgID(id)) } + var member delivery.Forge + grouped := false + if g != nil { + msg.Data, member, grouped = g.member(ctx, msg.Data) + } if _, err := pub.Publish(ctx, forgeSubject(msg.Event), msg.Data, opts...); err != nil { logger.Printf("%s: %s could not be written on FORGE, left for redelivery: %v", m.Subject(), msg.Event, err) return } + if grouped { + g.published(ctx, member) + } if msg.Said != "" { logger.Printf("%s: %s", m.Subject(), msg.Said) } @@ -136,17 +150,25 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } perf := &performer{call: call, pub: rec, done: kvRevisions{performedKV}, logger: logger} + // A delivery group's member names its group, and its end is said once (ADR 0633 §4), from the gate's merges. + groups := &grouper{send: sendVia(js), done: kvRevisions{performedKV}, logger: logger} watching := make(chan error, 3) for _, w := range []struct { bucket string react func(ctx context.Context, key string, rev uint64, value []byte) - }{{delivery.OwedBucket, perf.owed}, {delivery.MergesBucket, perf.merges}} { + }{{delivery.OwedBucket, perf.owed}, {delivery.MergesBucket, func(ctx context.Context, key string, rev uint64, value []byte) { + perf.merges(ctx, key, rev, value) + groups.ended(ctx, key, rev, value) + }}} { // The gate's buckets come with the gate and its grant; the bridge does not wait for them (no timer), and is started // again by its delivery when the gate arrives. A bucket refused or not answered is the same (issue 1945). kv := process.OptionalBucket(ctx, js, w.bucket, "nothing of it is performed until the bridge starts again", logger.Printf) if kv == nil { continue } + if w.bucket == delivery.MergesBucket { + groups.merges = kvMerges(kv) + } go func() { watching <- watch(ctx, kv, func(key string, rev uint64, value []byte) { hctx, cancel := context.WithTimeout(ctx, syncDeadline) @@ -171,7 +193,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // Bounded: the turn is held, and no sync or other event goes on without it. hctx, cancel := context.WithTimeout(ctx, syncDeadline) defer cancel() - handle(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), inputs, logger) + handleGrouped(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), inputs, groups, logger) }) }, logger.Printf) if err != nil { diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go index 85df5828..6234a90d 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go @@ -107,11 +107,10 @@ func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5 if !strings.Contains(logs.String(), "mesh.mod.gitea.event.pull.updated") || !strings.Contains(logs.String(), "head") { t.Errorf("log %q names neither the subject nor why", logs.String()) } -} - -func TestUnwrittenMessageIsNotAcked(t *testing.T) { // 1235 (a failed write is redelivered by the server, not retried here) - var logs bytes.Buffer - m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{err: errors.New("no stream")} + // 1235: an unwritten message is not acked (a failed write is redelivered by the server, not retried here); merged here + // from a test of its own, for the test budget. + logs.Reset() + m, pub = &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{err: errors.New("no stream")} handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0)) if m.acked != 0 || m.termd != 0 { t.Errorf("acked %d terminated %d, want neither", m.acked, m.termd) From ad9a35348453a7e1a65fea0cb0f49ae06406326a Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 23:17:27 +0100 Subject: [PATCH 11/18] =?UTF-8?q?Forge=20bridge:=20bridge=5Fgroups=20and?= =?UTF-8?q?=20bridge=5Fgroup=5Fend=5Fagain,=20the=20end=20said=20again=20t?= =?UTF-8?q?hrough=20the=20state=20end-asks=20(issue=201585,=20ADR=200633?= =?UTF-8?q?=20=C2=A74)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn --- .../mesh-delivery-forge-bridge-tools/main.go | 174 ++++++++++++++++++ .../main_test.go | 109 +++++++++++ .../mesh-delivery-forge-bridge/groupask.go | 61 ++++++ .../groupask_test.go | 68 +++++++ .../cmd/mesh-delivery-forge-bridge/main.go | 11 ++ .../internal/groups/groups.go | 158 ++++++++++++++++ .../manifest_test.go | 30 +-- .../mesh-delivery-forge-bridge/module.json | 18 +- 8 files changed, 616 insertions(+), 13 deletions(-) create mode 100644 modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go create mode 100644 modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go create mode 100644 modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groupask.go create mode 100644 modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groupask_test.go create mode 100644 modules/mesh-delivery-forge-bridge/internal/groups/groups.go diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go new file mode 100644 index 00000000..f3638d1e --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go @@ -0,0 +1,174 @@ +// Command mesh-delivery-forge-bridge-tools serves the forge bridge's tools (novox/hq ADR 0633 §4, issue 1585): the runtime +// launches it for a call. bridge_groups reads the delivery groups the gate holds and what the bridge kept of their ends; +// bridge_group_end_again writes an ask into the bridge's state end-asks and waits for the bridge's answer there, since the +// bridge is the other program and serves no request subject (ADR 0437 §4). +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strconv" + "strings" + "sync" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" + + "mesh-delivery-forge-bridge/internal/groups" +) + +// Tool names. +const ( + GroupsTool = "bridge_groups" + EndAgainTool = "bridge_group_end_again" +) + +// asker writes an end-asks value and waits for the bridge's answer to it. +type asker func(key string, value []byte) (groups.EndAsk, error) + +// Tools are the bridge's tools: groupsRead reads the groups the gate holds with what the bridge kept of them. +func Tools(ask asker, now func() time.Time, groupsRead func() ([]groups.Row, []string, error)) []stdio.Tool { + str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } + return []stdio.Tool{ + {Name: GroupsTool, + Description: "The delivery groups the gate holds (more than one member), each with its members' merge state and whether each " + + "merged message is written on the forge stream, and whether the group's end (forge.group.merged) is written: ended, " + + "or open with why (gathered, or merged and its end not written yet). An open group with every member merged is " + + "what bridge_group_end_again says the end of. A group's name narrows it to that group; open lists only open ones.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "group": map[string]any{"type": "string", "description": "only this group, by the name the gate keys it"}, + "open": map[string]any{"type": "boolean", "description": "only the groups whose end is not written"}}}, + Run: func(args map[string]any) (any, error) { + rows, unread, err := groupsRead() + if err != nil { + return nil, err + } + only, _ := args["open"].(bool) + group := str(args, "group") + kept := rows[:0] + for _, r := range rows { + if (group != "" && r.Group != group) || (only && !r.Open) { + continue + } + kept = append(kept, r) + } + return map[string]any{"groups": kept, "unread": unread}, nil + }}, + {Name: EndAgainTool, + Description: "Say a delivery group's end (forge.group.merged) again, by hand, for a group whose end the indexer did not take " + + "in (a group bridge_groups names as merged with its end not written, or one the indexer waits for). The bridge " + + "writes it again from the gate's merges, with the time appended to the message id so the stream does not drop it as " + + "a duplicate; the indexer takes an end for a group it already took in without changing anything. Refused while " + + "a member is not merged or its merged message is not written. Answers whether it was written.", + Input: map[string]any{"type": "object", "required": []string{"group"}, "properties": map[string]any{ + "group": map[string]any{"type": "string", "description": "the group, by the name the gate keys it"}, + "why": map[string]any{"type": "string", "description": "why its end is said again, in plain words"}}}, + Run: func(args map[string]any) (any, error) { + key, value, err := groups.NewEndAsk(str(args, "group"), str(args, "why"), stdio.CallerOf(args), now()) + if err != nil { + return nil, err + } + a, err := ask(key, value) + if err != nil { + return nil, err + } + return map[string]any{"group": a.Group, "written": a.Written, "answer": a.Answer, "by": a.By, "why": a.Why}, nil + }}, + } +} + +// read is the groups the gate holds with what the bridge kept of them. +func read(merges, performed stdio.KeptState) ([]groups.Row, []string, error) { + keys, err := merges.Keys() + if err != nil { + return nil, nil, fmt.Errorf("the gate's merges cannot be listed: %w", err) + } + raws := map[string][]byte{} + for _, k := range keys { + e, err := merges.Get(k) + if err != nil || e == nil { + continue + } + raws[k] = e.Value + } + pkeys, err := performed.Keys() + if err != nil { + return nil, nil, fmt.Errorf("the bridge's performed state cannot be listed: %w", err) + } + published, ended := map[string]bool{}, map[string]uint64{} + for _, k := range pkeys { + switch { + case strings.HasPrefix(k, "published."): + published[k] = true + case strings.HasPrefix(k, "groups."): + if e, err := performed.Get(k); err == nil && e != nil { + rev, _ := strconv.ParseUint(strings.TrimSpace(string(e.Value)), 10, 64) + ended[strings.TrimPrefix(k, "groups.")] = rev + } + } + } + rows, unread := groups.Rows(raws, published, ended) + return rows, unread, nil +} + +// askEnd writes the end-asks value at key and waits, on a watch of the key, at most bound for the bridge's answer. +func askEnd(s stdio.KeptState, bound time.Duration) asker { + return func(key string, value []byte) (groups.EndAsk, error) { + answered := make(chan groups.EndAsk, 1) + var rev uint64 + var mu sync.Mutex + take := func(revision uint64, v []byte) { + mu.Lock() + mine := rev + mu.Unlock() + if mine == 0 || revision <= mine { + return + } + if a, err := groups.DecodeEndAsk(v); err == nil && a.Done() { + select { + case answered <- a: + default: + } + } + } + if err := s.Watch(key, func(c stdio.StateChange) error { + if c.Op == "put" { + take(c.Revision, c.Value) + } + return nil + }); err != nil { + return groups.EndAsk{}, fmt.Errorf("watching %s.%s: %w", groups.EndAsksState, key, err) + } + r, err := s.Put(key, json.RawMessage(value)) + if err != nil { + return groups.EndAsk{}, fmt.Errorf("the ask could not be written to %s: %w", groups.EndAsksBucket, err) + } + mu.Lock() + rev = r + mu.Unlock() + if e, err := s.Get(key); err == nil && e != nil { + take(e.Revision, e.Value) + } + wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8) + defer cancel() + select { + case a := <-answered: + return a, nil + case <-wait.Done(): + return groups.EndAsk{}, fmt.Errorf("the bridge did not answer within %s: its answer will be in %s.%s", bound, groups.EndAsksBucket, key) + } + } +} + +func main() { + merges, performed := stdio.State(delivery.GateModule+".merges"), stdio.State(groups.PerformedState) + tools := Tools(askEnd(stdio.State(groups.EndAsksState), 25*time.Second), time.Now, + func() ([]groups.Row, []string, error) { return read(merges, performed) }) + if err := stdio.Serve("", tools); err != nil { + fmt.Fprintf(os.Stderr, "[mesh-delivery-forge-bridge-tools] %v\n", err) + os.Exit(1) + } +} diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go new file mode 100644 index 00000000..cdfe4250 --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go @@ -0,0 +1,109 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" + + "mesh-delivery-forge-bridge/internal/groups" +) + +// 1585/T18, T19 (ADR 0633 §4): bridge_groups names each group the gate holds with its members' state and whether its end is +// written (open or ended, with why), narrowed by group and by open; bridge_group_end_again writes an ask naming the group, +// who asked and why, and answers what the bridge answered. +func TestTheBridgeToolsReadTheGroupsAndAskTheEndAgain(t *testing.T) { + h1, h2 := strings.Repeat("1", 40), strings.Repeat("2", 40) + c1, c2 := strings.Repeat("c", 40), strings.Repeat("d", 40) + enc := func(group string, state2 delivery.MergeState, c string) []byte { + v, err := delivery.EncodeMerge(delivery.Merge{Group: group, Ask: "ask", Approved: map[string]string{"novox/a@1": h1, "novox/b@2": h2}, + Pulls: []delivery.MergePull{{Head: delivery.Head{Repository: "novox/a", Number: 1, Head: h1}, State: delivery.MergeMerged, Commit: c1}, + {Head: delivery.Head{Repository: "novox/b", Number: 2, Head: h2}, State: state2, Commit: c}}}) + if err != nil { + t.Fatal(err) + } + return v + } + lone, _ := delivery.EncodeMerge(delivery.Merge{Group: "solo", Ask: "ask", Approved: map[string]string{"novox/a@5": h1}, + Pulls: []delivery.MergePull{{Head: delivery.Head{Repository: "novox/a", Number: 5, Head: h1}, State: delivery.MergeMerged, Commit: c1}}}) + merges := map[string][]byte{"ended": enc("ended", delivery.MergeMerged, c2), "ready": enc("ready", delivery.MergeMerged, c2), + "gathered": enc("gathered", delivery.MergeMerging, ""), "solo": lone, "bad": []byte("{")} + performed := map[string]bool{} + for _, c := range []string{c1, c2} { + performed[groups.PublishedKey("novox/a", 1, c)], performed[groups.PublishedKey("novox/b", 2, c)] = true, true + } + var asked groups.EndAsk + var askedKey string + ask := func(key string, value []byte) (groups.EndAsk, error) { + askedKey = key + a, err := groups.DecodeEndAsk(value) + a.Written, a.Answer, a.Answered = true, "the end of the group is written again", time.Now() + asked = a + return a, err + } + tools := Tools(ask, func() time.Time { return time.Date(2026, 11, 4, 23, 0, 0, 0, time.UTC) }, + func() ([]groups.Row, []string, error) { + rows, unread := groups.Rows(merges, performed, map[string]uint64{"ended": 7}) + return rows, unread, nil + }) + call := func(name string, args map[string]any) any { + for _, tl := range tools { + if tl.Name == name { + out, err := tl.Run(args) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + return out + } + } + t.Fatalf("no tool %s", name) + return nil + } + rowsOf := func(args map[string]any) map[string]groups.Row { + raw, _ := json.Marshal(call(GroupsTool, args)) + var out struct { + Groups []groups.Row `json:"groups"` + Unread []string `json:"unread"` + } + if err := json.Unmarshal(raw, &out); err != nil { + t.Fatal(err) + } + m := map[string]groups.Row{} + for _, r := range out.Groups { + m[r.Group] = r + } + if len(args) == 0 && (len(out.Unread) != 1 || out.Unread[0] != "bad") { + t.Errorf("unread = %v, want bad", out.Unread) + } + return m + } + all := rowsOf(map[string]any{}) + if len(all) != 3 || !all["ended"].Ended || all["ended"].EndedAt != 7 { + t.Fatalf("groups = %+v", all) + } + if r := all["ready"]; !r.Open || !strings.Contains(r.Words, "bridge_group_end_again") || !r.Members[1].Published { + t.Errorf("ready = %+v, want open, merged and published, its end the thing to say", r) + } + if r := all["gathered"]; !r.Open || !strings.Contains(r.Words, "gathered") || r.Members[1].State != string(delivery.MergeMerging) { + t.Errorf("gathered = %+v", r) + } + if open := rowsOf(map[string]any{"open": true}); len(open) != 2 || open["ended"].Group != "" { + t.Errorf("open = %+v, want ready and gathered", open) + } + if one := rowsOf(map[string]any{"group": "ready"}); len(one) != 1 { + t.Errorf("group ready = %+v", one) + } + out := call(EndAgainTool, map[string]any{"group": "ready", "why": "the indexer never took it in"}).(map[string]any) + if askedKey != "ready" || asked.Group != "ready" || asked.Why != "the indexer never took it in" || out["written"] != true { + t.Errorf("the ask %q %+v, answered %v", askedKey, asked, out) + } + for _, tl := range tools { + if tl.Name == EndAgainTool { + if _, err := tl.Run(map[string]any{}); err == nil { + t.Error("an ask naming no group was taken") + } + } + } +} diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groupask.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groupask.go new file mode 100644 index 00000000..3c0365b8 --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groupask.go @@ -0,0 +1,61 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "mesh-delivery-forge-bridge/internal/groups" +) + +// endAsked answers one value of the state end-asks (the tool bridge_group_end_again, ADR 0633 §4): a value not yet answered +// says the group's end again from the gate's merges, with the ask's time appended to the message id so the stream's duplicate +// window does not drop it, and is written back by compare-and-set at its revision with whether it was written. A value +// answered already (this bridge's own write) is left. +func (g *grouper) endAsked(ctx context.Context, kv jetstream.KeyValue, key string, rev uint64, value []byte) { + a, err := groups.DecodeEndAsk(value) + if err != nil { + g.logger.Printf("%s.%s holds what is not an end ask, left: %v", groups.EndAsksState, key, err) + return + } + if a.Done() { + return + } + a = g.answerEnd(ctx, key, a, time.Now()) + g.logger.Printf("the end of the group %q said again by %s (%s): %s", a.Group, a.By, a.Why, a.Answer) + data, err := json.Marshal(a) + if err != nil { + return + } + if _, err := kv.Update(ctx, key, data, rev); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) { + g.logger.Printf("%s.%s: the answer could not be written: %v", groups.EndAsksState, key, err) + } +} + +// readMerges reads one key of the gate's merges state. +func (g *grouper) readMerges(ctx context.Context, key string) ([]byte, uint64, bool, error) { + if g.merges == nil { + return nil, 0, false, errors.New("the bridge does not read the gate's merges (its bucket was not there at the start)") + } + value, rev, found, err := g.merges(ctx, key) + return value, rev, found, err +} + +// answerEnd says the group's end again for an ask and returns the ask with its answer. +func (g *grouper) answerEnd(ctx context.Context, key string, a groups.EndAsk, at time.Time) groups.EndAsk { + a.Answered = at.UTC() + switch merges, mrev, found, err := g.readMerges(ctx, key); { + case err != nil: + a.Answer = "the gate's merges " + key + " cannot be read: " + err.Error() + case !found: + a.Answer = "the gate holds no merge of the group " + a.Group + case g.end(ctx, key, mrev, merges, groups.AgainStamp(a.At)): + a.Written, a.Answer = true, "the end of the group is written again" + default: + a.Answer = "the end is not written: a member is not merged or its merged message is not written, or FORGE refused it (the bridge's journal says which)" + } + return a +} diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groupask_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groupask_test.go new file mode 100644 index 00000000..3d0bb3af --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groupask_test.go @@ -0,0 +1,68 @@ +package main + +import ( + "bytes" + "context" + "log" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" + + "mesh-delivery-forge-bridge/internal/groups" +) + +// 1585/T19 (ADR 0633 §4): an ask to say a group's end again is answered by the bridge from the gate's merges, with the ask's +// time on the message id; not written while a member is not merged, with the reason; not for a group the gate does not hold; +// and the keys of performed the tools read are the ones the process writes. +func TestAnEndAskSaysTheGroupsEndAgainOrWhyNot(t *testing.T) { + ctx := context.Background() + h1, h2 := strings.Repeat("1", 40), strings.Repeat("2", 40) + c1, c2 := strings.Repeat("c", 40), strings.Repeat("d", 40) + pull := func(repo string, n int, head string, state delivery.MergeState, commit string) delivery.MergePull { + return delivery.MergePull{Head: delivery.Head{Repository: repo, Number: n, Head: head}, State: state, Commit: commit} + } + merge := func(second delivery.MergeState, commit2 string) []byte { + v, err := delivery.EncodeMerge(delivery.Merge{Group: "feat/take-in", Ask: "ask", Approved: map[string]string{"novox/mesh-sdk@37": h1, "novox/mesh-delivery@9": h2}, + Pulls: []delivery.MergePull{pull("novox/mesh-sdk", 37, h1, delivery.MergeMerged, c1), pull("novox/mesh-delivery", 9, h2, second, commit2)}}) + if err != nil { + t.Fatal(err) + } + return v + } + ask := groups.EndAsk{V: 1, Group: "feat/take-in", By: "operator@shanks", Why: "the indexer never took it in", At: time.Date(2026, 11, 4, 23, 0, 0, 123, time.UTC)} + for _, tc := range []struct { + name string + gate map[string][]byte + published bool + written bool + answer string + }{ + {"merged and published: written again", map[string][]byte{"feat/take-in": merge(delivery.MergeMerged, c2)}, true, true, "written again"}, + {"a member's message not written", map[string][]byte{"feat/take-in": merge(delivery.MergeMerged, c2)}, false, false, "not written"}, + {"a member not merged", map[string][]byte{"feat/take-in": merge(delivery.MergeMerging, "")}, true, false, "not written"}, + {"no merge at the gate", map[string][]byte{}, true, false, "holds no merge"}, + } { + pub, done := &idPub{}, memRevisions{} + if tc.published { + done[publishedKey("novox/mesh-sdk", 37, c1)], done[publishedKey("novox/mesh-delivery", 9, c2)] = 1, 1 + } + g := &grouper{send: pub.send, done: done, logger: log.New(&bytes.Buffer{}, "", 0), + merges: func(_ context.Context, key string) ([]byte, uint64, bool, error) { + v, ok := tc.gate[key] + return v, 4, ok, nil + }} + got := g.answerEnd(ctx, "feat/take-in", ask, ask.At.Add(time.Second)) + if got.Written != tc.written || !strings.Contains(got.Answer, tc.answer) || !got.Done() || len(pub.ids) != map[bool]int{true: 1}[tc.written] { + t.Errorf("%s: %+v, ids %v", tc.name, got, pub.ids) + } + if tc.written && pub.ids[0] != groupEndID("feat/take-in", c2)+"."+groups.AgainStamp(ask.At) { + t.Errorf("%s: the id %s does not carry the ask's time", tc.name, pub.ids[0]) + } + } + if groupsKey("feat/take-in") != groups.Key("feat/take-in") || publishedKey("novox/mesh-sdk", 37, c1) != groups.PublishedKey("novox/mesh-sdk", 37, c1) || + groupEndID("g", c1) != groups.EndID("g", c1, "") { + t.Error("the tools bundle's keys of performed are not the process's") + } +} diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go index c630e424..26331490 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go @@ -24,6 +24,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" "mesh-delivery-forge-bridge/internal/bridge" + groupkeys "mesh-delivery-forge-bridge/internal/groups" ) // eventsStream is the shared stream every module's events are on; the module's consumer on it is the durable @@ -177,6 +178,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro }) }() } + // A person says a group's end again with bridge_group_end_again, which writes the state end-asks (ADR 0633 §4): watched, and + // answered there; a state not there at this start is answered from the next start. + if kv := process.OptionalBucket(ctx, js, groupkeys.EndAsksBucket, "bridge_group_end_again is not answered until the bridge starts again", logger.Printf); kv != nil { + go func() { + err := watch(ctx, kv, func(key string, rev uint64, value []byte) { groups.endAsked(ctx, kv, key, rev, value) }) + if err != nil && ctx.Err() == nil { + logger.Printf("the end asks are no longer watched, so bridge_group_end_again is not answered until the bridge starts again: %v", err) + } + }() + } if err := said(ctx); err != nil { return err } diff --git a/modules/mesh-delivery-forge-bridge/internal/groups/groups.go b/modules/mesh-delivery-forge-bridge/internal/groups/groups.go new file mode 100644 index 00000000..006e1703 --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/internal/groups/groups.go @@ -0,0 +1,158 @@ +// Package groups is what the forge bridge's process and its tools bundle share of a delivery group's end (novox/hq ADR +// 0633 §4, issue 1585): the keys of the bridge's state performed, and the asks of the state end-asks, through which the +// tool bridge_group_end_again asks the process to say a group's end again (the bridge serves no request subject, ADR 0437 +// §4, so a tool asks in a state the process watches, as indexer_take_in asks the indexer in takeins). +package groups + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// The bridge's states the tools read and write. +const ( + // PerformedState is the bridge's state of what it performed and published. + PerformedState = "performed" + // EndAsksState is the state of asks to say a group's end again, keyed by the gate's merges key of the group. + EndAsksState = "end-asks" + // EndAsksBucket is its bucket on the bus. + EndAsksBucket = delivery.ForgeBridgeModule + "_" + EndAsksState +) + +// Key is the key of performed that says a group's end is written: `groups.`, its value the revision of the +// gate's merges key it was worked out from. +func Key(mergesKey string) string { return "groups." + mergesKey } + +// PublishedKey is the key of performed that says a member's merged message is written: `published.<16 hex digits of the +// sha256 of #@>`. +func PublishedKey(repository string, number int, merge string) string { + sum := sha256.Sum256([]byte(fmt.Sprintf("%s#%d@%s", strings.ToLower(repository), number, merge))) + return "published." + hex.EncodeToString(sum[:8]) +} + +// EndID is the message id of a group's end, and of one said again by hand at a time: the stream's duplicate window drops a +// second publish of the same id. +func EndID(group, lastMerge, again string) string { + id := "group-end." + group + "." + lastMerge + if again != "" { + id += "." + again + } + return id +} + +// AgainStamp is the time a hand ask appends to a group end's message id. +func AgainStamp(at time.Time) string { return at.UTC().Format("20060102T150405.000000000Z") } + +// EndAsk is a value of the end-asks state: which group's end a person asked to be said again, why, who and when, and, once +// the bridge read it, its answer. +type EndAsk struct { + V int `json:"v"` + Group string `json:"group"` + Why string `json:"why,omitempty"` + By string `json:"by,omitempty"` + At time.Time `json:"at"` + Written bool `json:"written,omitempty"` + Answer string `json:"answer,omitempty"` + Answered time.Time `json:"answered,omitzero"` +} + +// Done says the ask is answered. +func (a EndAsk) Done() bool { return !a.Answered.IsZero() } + +// NewEndAsk is the value bridge_group_end_again writes for a group: its key (the gate's merges key) and the value. +func NewEndAsk(group, why, by string, at time.Time) (string, []byte, error) { + group = strings.TrimSpace(group) + if group == "" { + return "", nil, errors.New("name the group whose end is said again (group)") + } + key, err := delivery.MergesKey(group) + if err != nil { + return "", nil, err + } + data, err := json.Marshal(EndAsk{V: 1, Group: group, Why: strings.TrimSpace(why), By: by, At: at.UTC()}) + return key, data, err +} + +// DecodeEndAsk reads an end-asks value. +func DecodeEndAsk(data []byte) (EndAsk, error) { + var a EndAsk + if err := json.Unmarshal(data, &a); err != nil { + return a, err + } + if a.V != 1 || a.Group == "" { + return a, errors.New("an end-asks value of no version 1 naming no group") + } + return a, nil +} + +// Row is one group as bridge_groups answers it: the gate's merge of it and what the bridge kept of its end. +type Row struct { + Key string `json:"key"` + Group string `json:"group"` + Members []RowMember `json:"members"` + Ended bool `json:"ended"` + EndedAt uint64 `json:"ended-at-revision,omitempty"` + Open bool `json:"open"` + Words string `json:"words"` +} + +// RowMember is one approved member of a group: its merge's state at the gate and whether its merged message is written. +type RowMember struct { + Repository string `json:"repository"` + Number int `json:"number"` + State string `json:"state"` + Commit string `json:"commit,omitempty"` + Published bool `json:"published"` +} + +// Rows are the groups the gate's merges hold with more than one member, each with what the bridge kept: ended (its end +// written, the revision it was worked out from) or open (some member is not merged or its merged message not written, or its +// end not written). merges is the gate's merges by key, performed the bridge's performed keys by key; unread are the gate's +// keys that do not read. +func Rows(merges map[string][]byte, performed map[string]bool, endedAt map[string]uint64) (rows []Row, unread []string) { + for key, raw := range merges { + m, err := delivery.DecodeMerge(raw) + if err != nil { + unread = append(unread, key) + continue + } + if len(m.Approved) < 2 { + continue + } + r := Row{Key: key, Group: m.Group} + allMerged := true + for _, p := range m.Pulls { + pub := p.Commit != "" && performed[PublishedKey(p.Repository, p.Number, p.Commit)] + r.Members = append(r.Members, RowMember{Repository: p.Repository, Number: p.Number, State: string(p.State), Commit: p.Commit, Published: pub}) + if p.State != delivery.MergeMerged || !pub { + allMerged = false + } + } + if len(m.Pulls) < len(m.Approved) { + allMerged = false + } + r.EndedAt, r.Ended = endedAt[key] + r.Open = !r.Ended + switch { + case r.Ended: + r.Words = "its end is written" + case allMerged: + r.Words = "every member is merged and its message written, and its end is not written: bridge_group_end_again says it" + default: + r.Words = "gathered: not every member is merged and its merged message written" + } + rows = append(rows, r) + } + sortRows(rows) + return rows, unread +} + +func sortRows(rows []Row) { sort.Slice(rows, func(i, j int) bool { return rows[i].Key < rows[j].Key }) } diff --git a/modules/mesh-delivery-forge-bridge/manifest_test.go b/modules/mesh-delivery-forge-bridge/manifest_test.go index 19686ac1..f702fa2f 100644 --- a/modules/mesh-delivery-forge-bridge/manifest_test.go +++ b/modules/mesh-delivery-forge-bridge/manifest_test.go @@ -28,12 +28,13 @@ type state struct { } type artifact struct { - Name string `json:"name"` - Kind string `json:"kind"` - Language string `json:"language"` - System string `json:"system"` - From string `json:"from"` - Binary string `json:"binary"` + Name string `json:"name"` + Kind string `json:"kind"` + Language string `json:"language"` + System string `json:"system"` + From string `json:"from"` + Binary string `json:"binary"` + Loads []string `json:"loads"` } type manifest struct { @@ -44,6 +45,7 @@ type manifest struct { Streams []stream `json:"streams"` State []state `json:"state"` Reads []string `json:"reads"` + Tools []string `json:"tools"` Invokes []string `json:"invokes"` OwnSecrets map[string]string `json:"own-secrets"` SecretsOwner string `json:"secrets-owner"` @@ -107,11 +109,15 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1 // §5), the revision of each key of the gate's buckets the bridge performed, so a key is performed once per // revision (ADR 0437 §8; issue 1341), and what a merge read of each module directory by its tree, so a merge reads // the forge only for the directories it changed (issue 1126). - if !reflect.DeepEqual(m.State, []state{{Name: "heads", History: 1}, {Name: "performed", History: 1}, {Name: "inputs", History: 1}}) { // 1279/AC6, 1341, 1126 - t.Errorf("state = %+v, want the buckets heads, performed and inputs", m.State) + if !reflect.DeepEqual(m.State, []state{{Name: "heads", History: 1}, {Name: "performed", History: 1}, {Name: "inputs", History: 1}, {Name: "end-asks", History: 1}}) { // 1279/AC6, 1341, 1126, 1585 + t.Errorf("state = %+v, want the buckets heads, performed, inputs and end-asks", m.State) } - // No other state, no bus, no seat, no tool, no event (ADR 0437 §4; ADR 0521: no check.status). - for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats"} { + // 1585/T18: the tools read the groups and ask the bridge to say a group's end again, through its state end-asks. + if want := []string{"bridge_group_end_again", "bridge_groups"}; !reflect.DeepEqual(sorted(m.Tools), want) { + t.Errorf("tools = %v, want %v", m.Tools, want) + } + // No other state, no bus, no seat, no event (ADR 0437 §4; ADR 0521: no check.status). + for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "seat", "seats"} { if _, ok := top[k]; ok { t.Errorf("the manifest declares %q: the bridge has none", k) } @@ -143,8 +149,8 @@ func TestManifestProcess(t *testing.T) { // 1235/AC1 if _, bus := p["bus"]; bus { t.Errorf("the process declares a bus: it runs on its own broker credential") } - if len(m.Build.Artifacts) != 1 { - t.Fatalf("artifacts = %+v", m.Build.Artifacts) + if len(m.Build.Artifacts) != 2 || m.Build.Artifacts[1].Name != "tools" || m.Build.Artifacts[1].From != "cmd/mesh-delivery-forge-bridge-tools" { + t.Fatalf("artifacts = %+v, want the program and the tools bundle", m.Build.Artifacts) } a := m.Build.Artifacts[0] if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-forge-bridge" || !strings.HasSuffix(a.From, a.Binary) { diff --git a/modules/mesh-delivery-forge-bridge/module.json b/modules/mesh-delivery-forge-bridge/module.json index f152cad0..338d9cdc 100644 --- a/modules/mesh-delivery-forge-bridge/module.json +++ b/modules/mesh-delivery-forge-bridge/module.json @@ -13,7 +13,12 @@ "state": [ { "name": "heads", "history": 1 }, { "name": "performed", "history": 1 }, - { "name": "inputs", "history": 1 } + { "name": "inputs", "history": 1 }, + { "name": "end-asks", "history": 1 } + ], + "tools": [ + "bridge_groups", + "bridge_group_end_again" ], "reads": [ "mesh-delivery-gate.merges", @@ -75,6 +80,17 @@ "system": "arch", "from": "cmd/mesh-delivery-forge-bridge", "binary": "mesh-delivery-forge-bridge" + }, + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-delivery-forge-bridge-tools", + "binary": "mesh-delivery-forge-bridge-tools", + "loads": [ + "mesh-delivery-forge-bridge-tools" + ] } ] } From 9c6e70e555aa3ae1250cc2d3198f3aed7051271c Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 23:19:07 +0100 Subject: [PATCH 12/18] indexer: before the pending state is on the bus, take a merge in at once and start without asking the own build (ADR 0633, issue 1585) TestIndexerWhole found the start waiting for ever on a build no node-engine answered, so the forge consumer never started, though nothing can be pending. Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn --- .../internal/index/pending.go | 19 +++++++++++++++---- .../internal/index/pending_test.go | 10 ++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index e88b20c0..6e660acc 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -192,6 +192,14 @@ func revisionWait(kind string, ready []PendingEntry, waitsFor string) delivery.T WaitsFor: waitsFor, Since: sinceOf(ready)} } +// pendingOnBus says whether the pending state is on the bus: a state this delivery brings is bound when it appears +// (issue 1790), and until then every merge is taken in at once, so the build this program runs is not needed. A read +// that fails otherwise counts as on the bus: the step that follows answers the error. +func (ix *Indexer) pendingOnBus(ctx context.Context) bool { + _, _, _, err := ix.Pending.Get(ctx, delivery.PendingKey(0)) + return !errors.Is(err, process.ErrNotYet) +} + // ownBuildNow is the build this program runs: the one learnt, else asked of the node-engine now. func (ix *Indexer) ownBuildNow(ctx context.Context) string { ix.ownMu.Lock() @@ -246,7 +254,9 @@ func (ix *Indexer) markSaid(ctx context.Context, entries []PendingEntry, t *deli // indexer it names. It answers what was taken in, and a line saying what waits. A message is never written while the // build this program runs is not known (ErrOwnBuildUnknown). func (ix *Indexer) Arrive(ctx context.Context, seq uint64, f delivery.Forge) (Applied, string, error) { - if ix.Pending == nil || ix.Modules == nil { + if ix.Pending == nil || ix.Modules == nil || !ix.pendingOnBus(ctx) { + // The state this delivery brings is not on the bus yet (issue 1790): the merge is taken in at once, as before, + // whether or not the build this program runs is known. a, err := ix.ApplyMessage(ctx, f) return a, "", err } @@ -295,7 +305,7 @@ func (ix *Indexer) Arrive(ctx context.Context, seq uint64, f delivery.Forge) (Ap // the member arrived. An end of a group taken in already (every member's merge at or behind its repository's taken // merge) changes nothing and is said: it is never delivered again for ever. func (ix *Indexer) EndGroup(ctx context.Context, end delivery.ForgeGroupEnd) (Applied, string, error) { - if ix.Pending == nil { + if ix.Pending == nil || !ix.pendingOnBus(ctx) { return Applied{}, "no pending state: the group's merges were taken in one by one", nil } own := ix.ownBuildNow(ctx) @@ -388,8 +398,9 @@ func (ix *Indexer) takenAlready(ctx context.Context, m delivery.GroupMerge) (boo // pending, taking in the whole revisions that name this build; a group still being gathered stays, and a revision // naming yet another build is held for that one. A claim left by an indexer that ran before is void here. func (ix *Indexer) TakeInAtStart(ctx context.Context, wait func(context.Context) bool) (Applied, string, error) { - if ix.Pending == nil { - return Applied{}, "", nil + if ix.Pending == nil || !ix.pendingOnBus(ctx) { + // Nothing can wait in a state not on the bus: the start asks no build and does not wait for one. + return Applied{}, "the pending state is not on the bus yet: nothing waits", nil } own := ix.ownBuildNow(ctx) for own == "" { diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index 21e6870a..cca2d25e 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -10,6 +10,7 @@ import ( "testing" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/process" ) // takeInBed is an indexer holding the catalogue's avahi and the indexer's own module, both with a fingerprint, its @@ -112,6 +113,15 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) if !errors.Is(err, ErrOwnBuildUnknown) || len(b.keys()) != 0 || b.commit(t, "avahi") == sha('b') { t.Errorf("err %v, pending %v, avahi at %s: want ErrOwnBuildUnknown and nothing written", err, b.keys(), b.commit(t, "avahi")) } + // Before the pending state is on the bus nothing waits: the start asks no build, and the merge is taken in at once, + // as before ADR 0633, though the build is unknown (TestIndexerWhole found the start waiting on it for ever). + b.ix.Pending = &memStore{unbound: process.ErrNotYet} + if _, _, err := b.ix.TakeInAtStart(ctx, func(context.Context) bool { t.Fatal("the start waits for the build"); return false }); err != nil { + t.Fatal(err) + } + if _, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", "")); err != nil || b.commit(t, "avahi") != sha('b') { + t.Errorf("no pending state: err %v, avahi at %s: want the merge taken in at once", err, b.commit(t, "avahi")) + } }) t.Run("T7 and T3 a merge alone, an indexer change, a merge behind it and a revert", func(t *testing.T) { b := newTakeInBed(t) From ef13e63268e46797fbd4a2ed205222ceb958da51 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 23:22:35 +0100 Subject: [PATCH 13/18] indexer: TestIndexerWhole runs the take-in step whole, a restart and a group (ADR 0633, issue 1585 I8a, I8b) A revision changing the indexer writes only what the named build needs and is taken in when that build starts; a gathered group stays across the restart and its end makes one take-in. Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn --- .../mesh-delivery-indexer/integration_test.go | 259 +++++++++++++++++- 1 file changed, 250 insertions(+), 9 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index a28b8db5..e8b23054 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -9,7 +9,9 @@ import ( "io" "os" "path/filepath" + "slices" "strings" + "sync/atomic" "testing" "time" @@ -20,6 +22,10 @@ import ( "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" + + "git.novox.be/novox/mesh-delivery/sdk/graph" + + "mesh-delivery-indexer/internal/index" ) // busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the @@ -42,11 +48,58 @@ accounts { // bus with the stream FORGE and the indexer's durable _ on it, the buckets the controller makes from the // manifest's state (mesh-delivery-indexer_candidates, history 10) and mesh-controller_builds holding build records, and // no mesh-delivery-gate_source bucket. -func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10 - js, ctx, candidates, _ := startIndexer(t, func(ctx context.Context, builds buildLog) { +// +// And the take-in step (ADR 0633, issue 1585 I8a, I8b): the indexer's states pending, verdicts, graph, taken and packages +// and its stream take-in are there, mesh-build-go reported its toolchain, and the node-engine's composition answers the +// build of the indexer that runs. A merge changing the indexer's own tree writes only the indexer's entry, its manifest, +// its verdict and its key of the graph (and the packages it publishes); the rest is pending, said take-in.pending and +// named by indexer_pending with the build it waits for. A group's first member is gathered meanwhile. Started again as +// that build, the indexer takes the revision in and leaves the group gathered; the group's second member and its end +// then make one take-in, said take-in.taken under the group's key. +func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 + var running atomic.Pointer[string] + first := "fp-the-indexer-that-runs" + running.Store(&first) + js, ctx, candidates, _, restart := startIndexerRun(t, func(ctx context.Context, builds buildLog) { // The builder's log entries of two builds' ends (ADR 0571 clause 1), as the builder appends them. builds.end(t, "novox/mesh-delivery", "a", "modules/a") builds.end(t, "novox/mesh-catalog", "c", "modules/c") + }, func(ctx context.Context, js jetstream.JetStream) { + // The states and the stream the controller makes from the indexer's manifest (old sections, ADR 0595 §11). + for bucket, history := range map[string]uint8{index.PendingBucket: 1, delivery.VerdictsBucket: delivery.VerdictsHistory, + graph.Bucket: 1, delivery.TakenBucket: 1, delivery.PackagesBucket: 1} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucket, History: history}); err != nil { + t.Fatal(err) + } + } + for name, subject := range map[string]string{"TAKE_IN": "mesh.stream.mesh-delivery-indexer.event.take-in.>", + "INDEXER_EVENTS": "mesh.mod.mesh-delivery-indexer.event.>"} { + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: name, Subjects: []string{subject}}); err != nil { + t.Fatal(err) + } + } + tc, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ToolchainsBucket("mesh-build-go"), History: 1}) + if err != nil { + t.Fatal(err) + } + key, _ := delivery.ToolchainKey("n1", "go") + value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "go", Version: "go1.27.1", Recipe: "1", At: time.Now()}) + if err != nil { + t.Fatal(err) + } + if _, err := tc.Put(ctx, key, value); err != nil { + t.Fatal(err) + } + // The node-engine's composition on n1 (process.Conn.Call's subjects): the build of the indexer that runs there. + answer := func(m *nats.Msg) { + raw, _ := json.Marshal(map[string]any{"result": map[string]any{"runs": map[string]string{"fp": *running.Load()}}, "node": "n1"}) + _ = m.Respond(raw) + } + for _, subject := range []string{"mesh.seat.node-engine.call.composition.n1.>", "mesh.seat.node-engine.tool.composition.n1"} { + if _, err := js.Conn().Subscribe(subject, answer); err != nil { + t.Fatal(err) + } + } }) h1, h2 := "0123456789abcdef0123456789abcdef01234567", "89abcdef0123456789abcdef0123456789abcdef" for _, f := range []delivery.Forge{ @@ -83,11 +136,186 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10 t.Errorf("%s#%d modules = %v, want [%s]", p.Repository, p.Number, p.Modules, want) } } + + // I8a: pull request 12 merged, changing the indexer's own tree and a's. + manifest, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + const ix = delivery.IndexerModule + m1 := delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: h1, + Merge: "1111111111111111111111111111111111111111", Root: "r0", + Trees: map[string]string{"modules/" + ix: "tree-indexer-2", "modules/a": "tree-a-2"}, + Manifests: map[string]string{"modules/" + ix: string(manifest), "modules/a": `{"module":"a","version":"1"}`}, + Locks: map[string]map[string]string{"modules/" + ix: {"go.sum": "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"}}} + merged := func(f delivery.Forge) uint64 { + t.Helper() + data, err := delivery.EncodeForge(f) + if err != nil { + t.Fatal(err) + } + event, _ := delivery.ForgeEvent(f.Repository, delivery.ForgeMerged) + ack, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event."+event, data) + if err != nil { + t.Fatal(err) + } + return ack.Sequence + } + m1Key := delivery.PendingKey(merged(m1)) + takeIns, err := js.Stream(ctx, "TAKE_IN") + if err != nil { + t.Fatal(err) + } + // said is the last take-in event under kind and key, within 20 s. + said := func(kind, key string) (delivery.TakeIn, bool) { + event, _ := delivery.TakeInEvent(kind, key) + for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { + if msg, err := takeIns.GetLastMsgForSubject(ctx, process.ModuleStreamSubject(ix, event)); err == nil { + if ti, err := delivery.DecodeTakeIn(msg.Data); err == nil { + return ti, true + } + } + } + return delivery.TakeIn{}, false + } + held, ok := said(delivery.TakeInPending, m1Key) + if !ok || held.What != delivery.TakeInRevision || held.WaitsFor == "" || held.WaitsFor == first || len(held.Merges) != 1 { + t.Fatalf("take-in.pending.%s = %+v (said %v) within 20 s, want the revision waiting for another build than %s", m1Key, held, ok, first) + } + named := held.WaitsFor + get := func(bucket, key string) ([]byte, bool) { + t.Helper() + kv, err := js.KeyValue(ctx, bucket) + if err != nil { + t.Fatal(err) + } + e, err := kv.Get(ctx, key) + if err != nil { + return nil, false + } + return e.Value(), true + } + // Only what building and moving the named indexer needs is written. + raw, ok := get(delivery.ModulesBucket, ix) + own, err := delivery.DecodeModule(raw) + if !ok || err != nil || own.FP2 != named || own.Commit != m1.Merge { + t.Fatalf("modules.%s = %+v (%v), want the named build %s at the merge", ix, own, err, named) + } + if _, ok := get(delivery.ManifestsBucket, own.Manifest); !ok { + t.Errorf("manifests.%s is not written", own.Manifest) + } + if _, ok := get(delivery.VerdictsBucket, own.Manifest); !ok { + t.Errorf("the verdict on the indexer's manifest %s is not written", own.Manifest) + } + if gk, _ := graph.Key(ix); !func() bool { _, ok := get(graph.Bucket, gk); return ok }() { + t.Errorf("the indexer's key %s of the module graph is not written", gk) + } + repoKey, _ := delivery.RepositoryKey(m1.Repository) + if _, ok := get(delivery.ModulesBucket, "a"); ok { + t.Errorf("modules.a is written while the revision is pending") + } + if _, ok := get(delivery.TakenBucket, repoKey); ok { + t.Errorf("taken.%s is written while the revision is pending", repoKey) + } + if e, err := candidates.Get(ctx, "feat/x"); err != nil { + t.Errorf("the candidate feat/x is gone while the merge of #12 is pending: %v", err) + } else if c, err := delivery.DecodeCandidate(e.Value()); err != nil || len(c.Pulls) != 2 { + t.Errorf("the candidate feat/x = %+v while the merge of #12 is pending, want its two pull requests", c) + } + + // A group's first member arrives meanwhile; its end is still to come (the knowledge base's remark: it stays across the restart). + members := []delivery.Head{{Repository: "novox/mesh-catalog", Number: 40, Head: "4040404040404040404040404040404040404040"}, + {Repository: "novox/mesh-controller", Number: 41, Head: "4141414141414141414141414141414141414141"}} + g1 := delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/g", Number: 40, Head: members[0].Head, + Merge: "4000000000000000000000000000000000000000", Root: "r0", Trees: map[string]string{"modules/c": "tree-c-2"}, + Manifests: map[string]string{"modules/c": `{"module":"c","version":"1"}`}, Group: "feat/g", Members: members} + g2 := delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-controller", Branch: "feat/g", Number: 41, Head: members[1].Head, + Merge: "4100000000000000000000000000000000000000", Root: "r0", Trees: map[string]string{"modules/k": "tree-k-1"}, + Manifests: map[string]string{"modules/k": `{"module":"k","version":"1"}`}, Group: "feat/g", Members: members} + g1Key, groupKey := delivery.PendingKey(merged(g1)), delivery.GroupTakeInKey("feat/g") + if gathered, ok := said(delivery.TakeInPending, groupKey); !ok || gathered.What != delivery.TakeInGroup || len(gathered.Merges) != 1 { + t.Fatalf("take-in.pending.%s = %+v (said %v) within 20 s, want the group with its first member", groupKey, gathered, ok) + } + // indexer_pending, the tool over the same state, names both waits. + pendingKV, err := js.KeyValue(ctx, index.PendingBucket) + if err != nil { + t.Fatal(err) + } + waits := func() []delivery.PendingWait { + t.Helper() + answer, err := index.PendingTools(kvStore{pendingKV}, nil, time.Now)[0].Run(map[string]any{}) + if err != nil { + t.Fatal(err) + } + return answer.(delivery.PendingAnswer).Waits + } + var w []delivery.PendingWait + for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { + if w = waits(); len(w) == 2 && w[0].WaitsFor == named && w[1].Said != "" { + break + } + } + if len(w) != 2 || w[0].Key != m1Key || w[0].WaitsFor != named || !slices.Contains(w[0].Holds, "a") || !slices.Contains(w[0].Holds, ix) || + w[1].Key != groupKey || w[1].What != delivery.TakeInGroup || len(w[1].Merges) != 1 || !strings.HasSuffix(w[1].Said, "pending."+groupKey) { + t.Fatalf("indexer_pending = %+v, want the revision %s waiting for %s holding a and the indexer, then the group gathered", w, m1Key, named) + } + + // The named indexer starts (the node-engine now runs that build): it takes the revision in, and leaves the group. + running.Store(&named) + restart() + if taken, ok := said(delivery.TakeInTaken, m1Key); !ok || len(taken.Merges) != 1 || taken.Merges[0].Merge != m1.Merge { + t.Fatalf("take-in.taken.%s = %+v (said %v) within 20 s of the start of %s", m1Key, taken, ok, named) + } + for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { + if w = waits(); len(w) == 1 { + break + } + } + if raw, ok := get(delivery.ModulesBucket, "a"); !ok || !strings.Contains(string(raw), m1.Merge) { + t.Errorf("modules.a = %s after the take-in at the start, want it at the merge %s", raw, m1.Merge) + } + if _, ok := get(delivery.TakenBucket, repoKey); !ok { + t.Errorf("taken.%s is not written by the take-in at the start", repoKey) + } + if len(w) != 1 || w[0].Key != groupKey || len(w[0].Keys) != 1 || w[0].Keys[0] != g1Key { + t.Fatalf("indexer_pending after the start = %+v, want only the group gathered, its first member %s", w, g1Key) + } + if _, ok := get(delivery.ModulesBucket, "c"); ok { + t.Errorf("modules.c is written while its group is gathered") + } + + // I8b: the group's second member and its end: one take-in of both, said under the group's key. + merged(g2) + end, err := delivery.EncodeForgeGroupEnd(delivery.ForgeGroupEnd{Group: "feat/g", Members: []delivery.GroupMerge{ + {Repository: g1.Repository, Number: g1.Number, Merge: g1.Merge}, {Repository: g2.Repository, Number: g2.Number, Merge: g2.Merge}}}) + if err != nil { + t.Fatal(err) + } + if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event."+delivery.ForgeGroupEndEvent, end); err != nil { + t.Fatal(err) + } + if taken, ok := said(delivery.TakeInTaken, groupKey); !ok || taken.What != delivery.TakeInGroup || len(taken.Merges) != 2 { + t.Fatalf("take-in.taken.%s = %+v (said %v) within 20 s of the group's end, want both members", groupKey, taken, ok) + } + var hasC, hasK bool + for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && !(hasC && hasK && len(w) == 0); time.Sleep(50 * time.Millisecond) { + _, hasC = get(delivery.ModulesBucket, "c") + _, hasK = get(delivery.ModulesBucket, "k") + w = waits() + } + if !hasC || !hasK || len(w) != 0 { + t.Errorf("after the group's end: modules.c %v, modules.k %v, indexer_pending %+v: want both taken in and nothing waiting", hasC, hasK, w) + } + // One take-in: neither member was said as a revision of its own. + single, _ := delivery.TakeInEvent(delivery.TakeInPending, g1Key) + if _, err := takeIns.GetLastMsgForSubject(ctx, process.ModuleStreamSubject(ix, single)); err == nil { + t.Errorf("the group's first member %s was said pending as a revision of its own", g1Key) + } } -// startIndexer starts the bus, the stream FORGE with the indexer's durable consumer, the buckets, lets put fill -// mesh-controller_builds, and runs the program. mesh-delivery-gate_source is never created. -func startIndexer(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, jetstream.KeyValue) { +// startIndexerRun starts the bus, the stream FORGE with the indexer's durable consumer, the buckets, lets put fill +// mesh-controller_builds, and runs the program; restart starts it again. mesh-delivery-gate_source is never created. +func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, jetstream.KeyValue, func()) { dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i") if err != nil { t.Fatal(err) @@ -166,11 +394,24 @@ func startIndexer(t *testing.T, put func(context.Context, buildLog), before ...f t.Fatal(err) } env := map[string]string{"MESH_BROKER_FILE": broker} - runCtx, stop := context.WithCancel(ctx) - done := make(chan error, 1) - go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() - t.Cleanup(func() { stop(); <-done }) + var stop func() + start := func() { + runCtx, cancel := context.WithCancel(ctx) + done := make(chan error, 1) + go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() + stop = func() { cancel(); <-done } + } + start() + t.Cleanup(func() { stop() }) + // restart stops the program and starts it again on the same bus, as the node-engine does with a new build. + restart := func() { stop(); start() } + return js, ctx, candidates, modules, restart +} + +// startIndexer is startIndexerRun for a test that never starts the program again. +func startIndexer(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, jetstream.KeyValue) { + js, ctx, candidates, modules, _ := startIndexerRun(t, put, before...) return js, ctx, candidates, modules } From 1040c8ec46152e808cd30a19b0a268380a05728d Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 23:24:05 +0100 Subject: [PATCH 14/18] Forge bridge: the process takes the keys of performed from internal/groups, one copy (issue 1585) Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn --- .../groupask_test.go | 8 ++---- .../cmd/mesh-delivery-forge-bridge/groups.go | 27 +++++-------------- .../mesh-delivery-forge-bridge/groups_test.go | 4 ++- 3 files changed, 12 insertions(+), 27 deletions(-) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groupask_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groupask_test.go index 3d0bb3af..de6ded88 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groupask_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groupask_test.go @@ -46,7 +46,7 @@ func TestAnEndAskSaysTheGroupsEndAgainOrWhyNot(t *testing.T) { } { pub, done := &idPub{}, memRevisions{} if tc.published { - done[publishedKey("novox/mesh-sdk", 37, c1)], done[publishedKey("novox/mesh-delivery", 9, c2)] = 1, 1 + done[groups.PublishedKey("novox/mesh-sdk", 37, c1)], done[groups.PublishedKey("novox/mesh-delivery", 9, c2)] = 1, 1 } g := &grouper{send: pub.send, done: done, logger: log.New(&bytes.Buffer{}, "", 0), merges: func(_ context.Context, key string) ([]byte, uint64, bool, error) { @@ -57,12 +57,8 @@ func TestAnEndAskSaysTheGroupsEndAgainOrWhyNot(t *testing.T) { if got.Written != tc.written || !strings.Contains(got.Answer, tc.answer) || !got.Done() || len(pub.ids) != map[bool]int{true: 1}[tc.written] { t.Errorf("%s: %+v, ids %v", tc.name, got, pub.ids) } - if tc.written && pub.ids[0] != groupEndID("feat/take-in", c2)+"."+groups.AgainStamp(ask.At) { + if tc.written && pub.ids[0] != groups.EndID("feat/take-in", c2, groups.AgainStamp(ask.At)) { t.Errorf("%s: the id %s does not carry the ask's time", tc.name, pub.ids[0]) } } - if groupsKey("feat/take-in") != groups.Key("feat/take-in") || publishedKey("novox/mesh-sdk", 37, c1) != groups.PublishedKey("novox/mesh-sdk", 37, c1) || - groupEndID("g", c1) != groups.EndID("g", c1, "") { - t.Error("the tools bundle's keys of performed are not the process's") - } } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups.go index 486990b9..824a6a7b 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups.go @@ -2,10 +2,7 @@ package main import ( "context" - "crypto/sha256" - "encoding/hex" "errors" - "fmt" "log" "slices" "strconv" @@ -14,6 +11,8 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/delivery" + + groupkeys "mesh-delivery-forge-bridge/internal/groups" ) // A delivery group's merges reach the indexer together (novox/hq ADR 0633 §4): the bridge sets the group on each @@ -25,18 +24,6 @@ import ( // and `groups.` once the group's end is written, its value the revision of the gate's merges key it was // worked out from. A group whose end is not written yet is open (bridge_groups reads them). -// groupsKey is the key of performed that says a group's end is written. -func groupsKey(mergesKey string) string { return "groups." + mergesKey } - -// publishedKey is the key of performed that says a member's merged message is written. -func publishedKey(repository string, number int, merge string) string { - sum := sha256.Sum256([]byte(fmt.Sprintf("%s#%d@%s", strings.ToLower(repository), number, merge))) - return "published." + hex.EncodeToString(sum[:8]) -} - -// groupEndID is the message id of a group's end: the stream's duplicate window drops a second publish of one end. -func groupEndID(group, lastMerge string) string { return "group-end." + group + "." + lastMerge } - // merges reads one key of the gate's merges state. type mergesRead func(ctx context.Context, key string) (value []byte, rev uint64, found bool, err error) @@ -143,7 +130,7 @@ func (g *grouper) member(ctx context.Context, data []byte) ([]byte, delivery.For // published keeps that a member's merged message is written, and says the group's end when it was the last. func (g *grouper) published(ctx context.Context, f delivery.Forge) { - if err := g.done.Put(ctx, publishedKey(f.Repository, f.Number, f.Merge), 1); err != nil { + if err := g.done.Put(ctx, groupkeys.PublishedKey(f.Repository, f.Number, f.Merge), 1); err != nil { g.logger.Printf("%s#%d: that its merged message is written is not kept, so the group's end waits for the next act: %v", f.Repository, f.Number, err) return } @@ -172,7 +159,7 @@ func (g *grouper) end(ctx context.Context, key string, rev uint64, value []byte, return false } if again == "" { - if _, seen, err := g.done.Get(ctx, groupsKey(key)); err == nil && seen { + if _, seen, err := g.done.Get(ctx, groupkeys.Key(key)); err == nil && seen { return false } } @@ -184,7 +171,7 @@ func (g *grouper) end(ctx context.Context, key string, rev uint64, value []byte, if at < 0 || m.Pulls[at].State != delivery.MergeMerged || m.Pulls[at].Commit == "" { return false // not every member is merged: the group stays gathered } - if _, ok, err := g.done.Get(ctx, publishedKey(member.Repository, member.Number, m.Pulls[at].Commit)); err != nil || !ok { + if _, ok, err := g.done.Get(ctx, groupkeys.PublishedKey(member.Repository, member.Number, m.Pulls[at].Commit)); err != nil || !ok { return false // its merged message is not written yet: said when it is (published) } end.Members = append(end.Members, delivery.GroupMerge{Repository: member.Repository, Number: member.Number, Merge: m.Pulls[at].Commit}) @@ -199,7 +186,7 @@ func (g *grouper) end(ctx context.Context, key string, rev uint64, value []byte, g.logger.Printf("the end of the group %q cannot be written: %v", m.Group, err) return false } - id := groupEndID(m.Group, end.Members[len(end.Members)-1].Merge) + id := groupkeys.EndID(m.Group, end.Members[len(end.Members)-1].Merge, "") if again != "" { id += "." + again } @@ -207,7 +194,7 @@ func (g *grouper) end(ctx context.Context, key string, rev uint64, value []byte, g.logger.Printf("the end of the group %q is not written on FORGE, written at the gate's next write or by bridge_group_end_again: %v", m.Group, err) return false } - if err := g.done.Put(ctx, groupsKey(key), rev); err != nil { + if err := g.done.Put(ctx, groupkeys.Key(key), rev); err != nil { g.logger.Printf("the end of the group %q is written and not kept: the stream's duplicate window drops a second one: %v", m.Group, err) } g.logger.Printf("the end of the group %q is written: %d members merged, the last %s", m.Group, len(end.Members), end.Members[len(end.Members)-1].Merge) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups_test.go index 419c259e..0f792c32 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups_test.go @@ -8,6 +8,8 @@ import ( "testing" "git.novox.be/novox/mesh-sdk/go/delivery" + + groupkeys "mesh-delivery-forge-bridge/internal/groups" ) // idPub records the subjects, bodies and message ids it is handed. @@ -84,7 +86,7 @@ func TestAGroupsMembersNameItAndItsEndIsSaidOnce(t *testing.T) { // 1585/T16, T1 if err != nil || len(end.Members) != 2 || end.Members[0].Merge != c1 || end.Members[1].Merge != c2 { t.Errorf("the end = %+v %v, want both members in their merge order", end, err) } - if rev, ok := done[groupsKey("feat/take-in")]; !ok || rev != 7 { + if rev, ok := done[groupkeys.Key("feat/take-in")]; !ok || rev != 7 { t.Errorf("performed %v: want groups.feat/take-in kept", done) } // Again by hand: written once more, its id not the first's. From a395b49be2fd1cb980f3f7ee81da5f36c98be369 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 23:24:26 +0100 Subject: [PATCH 15/18] WIP (ADR 0633 declined): watchman pinned to the group's mesh-sdk head; the other pins, the gate's const swap and the full checks not done Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn --- modules/mesh-delivery-watchman/go.mod | 2 +- modules/mesh-delivery-watchman/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/mesh-delivery-watchman/go.mod b/modules/mesh-delivery-watchman/go.mod index 9a0d188f..33a7e8dc 100644 --- a/modules/mesh-delivery-watchman/go.mod +++ b/modules/mesh-delivery-watchman/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-watchman go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104221203-10c91fba4a28 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-watchman/go.sum b/modules/mesh-delivery-watchman/go.sum index c0603c2b..5928f081 100644 --- a/modules/mesh-delivery-watchman/go.sum +++ b/modules/mesh-delivery-watchman/go.sum @@ -1,5 +1,7 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104221203-10c91fba4a28 h1:Zoi+myMvKaRYdM8w/96W6GnoAiTvgwvEF1QPA6HelaU= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104221203-10c91fba4a28/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= From 4e4601a1eb90150bb8d9d08ced48235a2a0ad93f Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 5 Nov 2026 00:05:01 +0100 Subject: [PATCH 16/18] Take-in and forge contracts live with their owners: the indexer's takein package, the forge bridge's forgemsg package; mesh-sdk pinned at main (issue 1585, ADR 0633) The operator, 2026-11-04: logic one module owns lives inside that module as a package of its own, so the take-in step changes no mesh-sdk. TakeIn, its keys and event names, PendingWait and PendingAnswer move to modules/mesh-delivery-indexer/takein (its merges a takein.Merge of its own); Forge whole, with its group fields, ForgeGroupEnd and GroupMerge, moves to modules/mesh-delivery-forge-bridge/forgemsg. Both are nested Go modules that their readers take by replace, as buildrun is. Every importer moves in this commit: the indexer, the bridge, the watchman and the view. Forge's private checks (the repository name, the version) are copied small; mesh-sdk main still holds an unused Forge until its own fold removes it. The indexer, bridge, watchman, view and checker pin mesh-sdk 5a38a12 (origin/main), no feat/take-in commit. The bridge's integration bus now grants the state end-asks that bridge_group_end_again added: without it every test of the package failed on a permissions violation. Test budget: two new test functions (TestForge, TestTakeInEvents) and the two the join added are paid by merging four pairs in internal/bridge; the repository stays at 806 unit tests. Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn --- modules/mesh-delivery-checker/go.mod | 2 +- modules/mesh-delivery-checker/go.sum | 4 +- .../mesh-delivery-forge-bridge/gate_test.go | 9 +- .../cmd/mesh-delivery-forge-bridge/groups.go | 23 +- .../mesh-delivery-forge-bridge/groups_test.go | 7 +- .../integration_test.go | 33 +- .../cmd/mesh-delivery-forge-bridge/main.go | 3 +- .../mesh-delivery-forge-bridge/main_test.go | 6 +- .../cmd/mesh-delivery-forge-bridge/open.go | 6 +- .../mesh-delivery-forge-bridge/open_test.go | 22 +- .../forgemsg/forge.go | 413 ++++++++++++++++++ .../forgemsg/forge_test.go | 193 ++++++++ .../forgemsg/go.mod | 5 + .../forgemsg/go.sum | 14 + .../forgemsg/group.go | 78 ++++ modules/mesh-delivery-forge-bridge/go.mod | 5 +- modules/mesh-delivery-forge-bridge/go.sum | 4 +- .../internal/bridge/acts.go | 5 +- .../internal/bridge/acts_test.go | 31 +- .../internal/bridge/bridge.go | 50 +-- .../internal/bridge/bridge_test.go | 104 +++-- .../internal/bridge/files_test.go | 14 +- .../internal/bridge/paths_test.go | 4 +- .../mesh-delivery-indexer/integration_test.go | 72 +-- .../cmd/mesh-delivery-indexer/main.go | 11 +- .../cmd/mesh-delivery-indexer/main_test.go | 11 +- .../cmd/mesh-delivery-indexer/named.go | 11 +- .../packages_integration_test.go | 15 +- .../cmd/mesh-delivery-indexer/verdicts.go | 9 +- modules/mesh-delivery-indexer/go.mod | 12 +- modules/mesh-delivery-indexer/go.sum | 4 +- .../index/candidate_manifests_test.go | 7 +- .../internal/index/candidates.go | 30 +- .../internal/index/control_records.go | 9 +- .../internal/index/control_records_test.go | 3 +- .../internal/index/index_test.go | 23 +- .../internal/index/manifests.go | 3 +- .../internal/index/manifests_test.go | 7 +- .../internal/index/modules.go | 17 +- .../internal/index/modules_test.go | 25 +- .../internal/index/moves.go | 17 +- .../internal/index/order.go | 3 +- .../internal/index/own.go | 3 +- .../internal/index/pending.go | 96 ++-- .../internal/index/pending_test.go | 48 +- .../internal/index/pending_tools.go | 4 +- .../internal/index/previews_test.go | 3 +- .../internal/index/registered.go | 5 +- .../internal/index/taken.go | 7 +- .../internal/index/toolchains_test.go | 5 +- .../internal/index/verdicts_test.go | 3 +- .../mesh-delivery-indexer/manifest_test.go | 3 +- modules/mesh-delivery-indexer/takein/go.mod | 5 + modules/mesh-delivery-indexer/takein/go.sum | 14 + .../mesh-delivery-indexer/takein/takein.go | 251 +++++++++++ .../takein/takein_test.go | 95 ++++ .../cmd/mesh-delivery-view-tools/main.go | 7 +- .../tiers_rows_test.go | 11 +- modules/mesh-delivery-view/go.mod | 9 +- modules/mesh-delivery-view/go.sum | 4 +- .../mesh-delivery-view/internal/view/read.go | 3 +- .../mesh-delivery-view/internal/view/tiers.go | 3 +- .../cmd/mesh-delivery-watchman/main.go | 5 +- modules/mesh-delivery-watchman/go.mod | 5 +- modules/mesh-delivery-watchman/go.sum | 6 +- .../internal/watch/findings_test.go | 43 +- .../internal/watch/takein.go | 10 +- 67 files changed, 1547 insertions(+), 425 deletions(-) create mode 100644 modules/mesh-delivery-forge-bridge/forgemsg/forge.go create mode 100644 modules/mesh-delivery-forge-bridge/forgemsg/forge_test.go create mode 100644 modules/mesh-delivery-forge-bridge/forgemsg/go.mod create mode 100644 modules/mesh-delivery-forge-bridge/forgemsg/go.sum create mode 100644 modules/mesh-delivery-forge-bridge/forgemsg/group.go create mode 100644 modules/mesh-delivery-indexer/takein/go.mod create mode 100644 modules/mesh-delivery-indexer/takein/go.sum create mode 100644 modules/mesh-delivery-indexer/takein/takein.go create mode 100644 modules/mesh-delivery-indexer/takein/takein_test.go diff --git a/modules/mesh-delivery-checker/go.mod b/modules/mesh-delivery-checker/go.mod index bf775228..8c2c7fe6 100644 --- a/modules/mesh-delivery-checker/go.mod +++ b/modules/mesh-delivery-checker/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-checker go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-checker/go.sum b/modules/mesh-delivery-checker/go.sum index 7ff42872..46655fec 100644 --- a/modules/mesh-delivery-checker/go.sum +++ b/modules/mesh-delivery-checker/go.sum @@ -1,7 +1,7 @@ git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a h1:oxgksCiq272/afXbUION/dxtAS5MPGnmyq6+AZQxmXg= git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a/go.mod h1:kpsX4Pgss5EeuBvel1zWa2ogj7MkR3jZso6oa/J0YUQ= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659 h1:tK7MzxVm/ECNg1Yblhh1ynVEthbZCg2mSPW0wQZC0b0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go index 608685e4..fae4fb07 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go @@ -9,6 +9,7 @@ import ( "strings" "testing" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -120,8 +121,8 @@ func TestARefusedActIsWrittenOnForgeOnceAndNotRetried(t *testing.T) { // 1341/AC if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.refused" { t.Fatalf("published on %v", pub.subjects) } - m, err := delivery.DecodeForge(pub.data[0]) - if err != nil || m.Kind != delivery.ForgeRefused || m.Act != "comment.1" || m.Number != 12 || m.Head != headA || m.Words != "the pull request is locked" { + m, err := forgemsg.DecodeForge(pub.data[0]) + if err != nil || m.Kind != forgemsg.ForgeRefused || m.Act != "comment.1" || m.Number != 12 || m.Head != headA || m.Words != "the pull request is locked" { t.Errorf("message %+v err %v", m, err) } f.refuse = nil @@ -188,8 +189,8 @@ func TestARefusedMergeIsWrittenOnForgeOnceAndNotRetried(t *testing.T) { // 1342/ if len(f.calls) != 1 || len(pub.data) != 1 { t.Fatalf("calls %d messages %d, want one each", len(f.calls), len(pub.data)) } - m, err := delivery.DecodeForge(pub.data[0]) - if err != nil || m.Kind != delivery.ForgeRefused || m.Repository != "novox/mesh-delivery" || m.Number != 12 || m.Head != headA || + m, err := forgemsg.DecodeForge(pub.data[0]) + if err != nil || m.Kind != forgemsg.ForgeRefused || m.Repository != "novox/mesh-delivery" || m.Number != 12 || m.Head != headA || m.Act != "merge" || m.Words != "novox/mesh-delivery#12 is not merged: its head moved" { t.Errorf("message %+v err %v", m, err) } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups.go index 824a6a7b..f00c6688 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups.go @@ -10,6 +10,7 @@ import ( "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/delivery" groupkeys "mesh-delivery-forge-bridge/internal/groups" @@ -58,7 +59,7 @@ func kvMerges(kv jetstream.KeyValue) mergesRead { // groupOf is the gate's merge of the group a merged message's branch names, when the message's pull request is one of // its approved members and the group has more than one: a lone pull request is no group. -func (g *grouper) groupOf(ctx context.Context, f delivery.Forge) (delivery.Merge, uint64, bool) { +func (g *grouper) groupOf(ctx context.Context, f forgemsg.Forge) (delivery.Merge, uint64, bool) { if g == nil || g.merges == nil || f.Branch == "" { return delivery.Merge{}, 0, false } @@ -110,9 +111,9 @@ func membersOf(m delivery.Merge) []delivery.Head { // member sets the group and its approved members on a merged message's body (ADR 0633 §4); any other body, and a // merge of no group, is left as it is. -func (g *grouper) member(ctx context.Context, data []byte) ([]byte, delivery.Forge, bool) { - f, err := delivery.DecodeForge(data) - if err != nil || f.Kind != delivery.ForgeMerged { +func (g *grouper) member(ctx context.Context, data []byte) ([]byte, forgemsg.Forge, bool) { + f, err := forgemsg.DecodeForge(data) + if err != nil || f.Kind != forgemsg.ForgeMerged { return data, f, false } m, _, ok := g.groupOf(ctx, f) @@ -120,7 +121,7 @@ func (g *grouper) member(ctx context.Context, data []byte) ([]byte, delivery.For return data, f, false } f.Group, f.Members = m.Group, membersOf(m) - out, err := delivery.EncodeForge(f) + out, err := forgemsg.EncodeForge(f) if err != nil { g.logger.Printf("%s#%d: its merged message cannot name the group %q, written without: %v", f.Repository, f.Number, m.Group, err) return data, f, false @@ -129,7 +130,7 @@ func (g *grouper) member(ctx context.Context, data []byte) ([]byte, delivery.For } // published keeps that a member's merged message is written, and says the group's end when it was the last. -func (g *grouper) published(ctx context.Context, f delivery.Forge) { +func (g *grouper) published(ctx context.Context, f forgemsg.Forge) { if err := g.done.Put(ctx, groupkeys.PublishedKey(f.Repository, f.Number, f.Merge), 1); err != nil { g.logger.Printf("%s#%d: that its merged message is written is not kept, so the group's end waits for the next act: %v", f.Repository, f.Number, err) return @@ -163,7 +164,7 @@ func (g *grouper) end(ctx context.Context, key string, rev uint64, value []byte, return false } } - end := delivery.ForgeGroupEnd{Group: m.Group} + end := forgemsg.ForgeGroupEnd{Group: m.Group} for _, member := range membersOf(m) { at := slices.IndexFunc(m.Pulls, func(p delivery.MergePull) bool { return delivery.SameRepository(p.Repository, member.Repository) && p.Number == member.Number @@ -174,14 +175,14 @@ func (g *grouper) end(ctx context.Context, key string, rev uint64, value []byte, if _, ok, err := g.done.Get(ctx, groupkeys.PublishedKey(member.Repository, member.Number, m.Pulls[at].Commit)); err != nil || !ok { return false // its merged message is not written yet: said when it is (published) } - end.Members = append(end.Members, delivery.GroupMerge{Repository: member.Repository, Number: member.Number, Merge: m.Pulls[at].Commit}) + end.Members = append(end.Members, forgemsg.GroupMerge{Repository: member.Repository, Number: member.Number, Merge: m.Pulls[at].Commit}) } // The members in the order the gate merged them, so the last commit names the end. - slices.SortStableFunc(end.Members, func(a, b delivery.GroupMerge) int { + slices.SortStableFunc(end.Members, func(a, b forgemsg.GroupMerge) int { return slices.IndexFunc(m.Pulls, func(p delivery.MergePull) bool { return p.Commit == a.Merge }) - slices.IndexFunc(m.Pulls, func(p delivery.MergePull) bool { return p.Commit == b.Merge }) }) - data, err := delivery.EncodeForgeGroupEnd(end) + data, err := forgemsg.EncodeForgeGroupEnd(end) if err != nil { g.logger.Printf("the end of the group %q cannot be written: %v", m.Group, err) return false @@ -190,7 +191,7 @@ func (g *grouper) end(ctx context.Context, key string, rev uint64, value []byte, if again != "" { id += "." + again } - if err := g.send(ctx, forgeSubject(delivery.ForgeGroupEndEvent), data, id); err != nil { + if err := g.send(ctx, forgeSubject(forgemsg.ForgeGroupEndEvent), data, id); err != nil { g.logger.Printf("the end of the group %q is not written on FORGE, written at the gate's next write or by bridge_group_end_again: %v", m.Group, err) return false } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups_test.go index 0f792c32..17c57b22 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups_test.go @@ -7,6 +7,7 @@ import ( "strings" "testing" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/delivery" groupkeys "mesh-delivery-forge-bridge/internal/groups" @@ -50,7 +51,7 @@ func TestAGroupsMembersNameItAndItsEndIsSaidOnce(t *testing.T) { // 1585/T16, T1 return v, 7, ok, nil }} mergedOf := func(repo string, number int, head, commit string) []byte { - data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: repo, Branch: "feat/take-in", Number: number, Head: head, Merge: commit, Root: "r"}) + data, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: repo, Branch: "feat/take-in", Number: number, Head: head, Merge: commit, Root: "r"}) if err != nil { t.Fatal(err) } @@ -63,7 +64,7 @@ func TestAGroupsMembersNameItAndItsEndIsSaidOnce(t *testing.T) { // 1585/T16, T1 // The first member: its message names the group and both members; the group is not ended. gate["feat/take-in"] = merge(sdk(delivery.MergeMerged, c1, ""), dlv(delivery.MergeMerging, "")) data, f, ok := g.member(ctx, mergedOf("novox/mesh-sdk", 37, h1, c1)) - if back, err := delivery.DecodeForge(data); !ok || err != nil || back.Group != "feat/take-in" || len(back.Members) != 2 { + if back, err := forgemsg.DecodeForge(data); !ok || err != nil || back.Group != "feat/take-in" || len(back.Members) != 2 { t.Fatalf("a member's message: %+v %v", back, err) } g.published(ctx, f) @@ -82,7 +83,7 @@ func TestAGroupsMembersNameItAndItsEndIsSaidOnce(t *testing.T) { // 1585/T16, T1 if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.group.merged" || pub.ids[0] != "group-end.feat/take-in."+c2 { t.Fatalf("ends %v ids %v, want one forge.group.merged with id group-end.feat/take-in.%s", pub.subjects, pub.ids, c2) } - end, err := delivery.DecodeForgeGroupEnd(pub.data[0]) + end, err := forgemsg.DecodeForgeGroupEnd(pub.data[0]) if err != nil || len(end.Members) != 2 || end.Members[0].Merge != c1 || end.Members[1].Merge != c2 { t.Errorf("the end = %+v %v, want both members in their merge order", end, err) } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go index 84644d6f..b45ca285 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go @@ -20,6 +20,7 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" ) @@ -36,7 +37,7 @@ import ( // // The lists are what mesh-controller's broker.PermissionsFor answers (internal/broker/nats.go:577) for a module principal // with the manifest of module.json: consumes the three gitea.pull events, invokes gitea.gitea_open_pulls, -// gitea.gitea_commit_trees and gitea.gitea_get_file, state heads, and the stream forge; `**` is written `>` on the wire (modstreams.go +// gitea.gitea_commit_trees and gitea.gitea_get_file, state heads, performed, inputs and end-asks (ADR 0633 §5's hand end), and the stream forge; `**` is written `>` on the wire (modstreams.go // WirePattern). The consumer's durable is _ on EVENTS (nats.go consumerDurable). The controller's own // user keeps all rights, as it has. const busConf = ` @@ -83,6 +84,12 @@ accounts { "$JS.FC.KV_mesh-delivery-forge-bridge_inputs.>" "$KV.mesh-delivery-forge-bridge_performed.>" "$KV.mesh-delivery-forge-bridge_inputs.>" + "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-forge-bridge_end-asks.>" + "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-forge-bridge_end-asks.>" + "$JS.API.DIRECT.GET.KV_mesh-delivery-forge-bridge_end-asks.>" + "$JS.API.STREAM.INFO.KV_mesh-delivery-forge-bridge_end-asks" + "$JS.FC.KV_mesh-delivery-forge-bridge_end-asks.>" + "$KV.mesh-delivery-forge-bridge_end-asks.>" "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-gate_merges.>" "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-gate_merges.>" "$JS.API.DIRECT.GET.KV_mesh-delivery-gate_merges.>" @@ -321,13 +328,13 @@ func TestBridgeWritesPullMessages(t *testing.T) { // 1235/AC2 if err != nil { t.Fatal(err) } - f, err := delivery.DecodeForge(raw.Data) + f, err := forgemsg.DecodeForge(raw.Data) if err != nil { t.Fatal(err) } - want := delivery.Forge{V: delivery.Version, Kind: "pull", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, + want := forgemsg.Forge{V: delivery.Version, Kind: "pull", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: "updated", Dirs: []string{"modules/a"}, DirsKnown: true, Manifests: map[string]string{"modules/a": "{}"}} - if !delivery.SameForge(f, want) { + if !forgemsg.SameForge(f, want) { t.Errorf("message = %+v, want %+v", f, want) } @@ -416,16 +423,16 @@ func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4, 1448/AC5 if err != nil { t.Fatal(err) } - f, err := delivery.DecodeForge(raw.Data) + f, err := forgemsg.DecodeForge(raw.Data) if err != nil { t.Fatal(err) } - want := delivery.Forge{V: delivery.Version, Kind: "merged", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, + want := forgemsg.Forge{V: delivery.Version, Kind: "merged", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Merge: merge, Root: "r0", Trees: map[string]string{"": "t0", "modules/a": "t1"}, Manifests: map[string]string{"": string(manifest), "modules/a": string(manifest)}, // No mesh package: only the indexer computes what a fingerprint takes (ADR 0593 §2). Locks: map[string]map[string]string{"modules/a": {"go.sum": hex.EncodeToString(sha256Sum(sum))}}} - if !delivery.SameForge(f, want) { + if !forgemsg.SameForge(f, want) { t.Errorf("message = %+v, want %+v", f, want) } select { @@ -478,7 +485,7 @@ func TestBridgeSaysTheOpenPullsAtStart(t *testing.T) { // 1237/AC1 if err != nil { t.Fatal(err) } - if f, err := delivery.DecodeForge(raw.Data); err != nil || f.Action != "updated" || f.DirsKnown || f.Branch != "feat/x" { + if f, err := forgemsg.DecodeForge(raw.Data); err != nil || f.Action != "updated" || f.DirsKnown || f.Branch != "feat/x" { t.Errorf("%s: %+v err %v", repo, f, err) } } @@ -548,20 +555,20 @@ func TestBridgeReadsTheForgeAtStartAndAfterAReconnectAndHearsNoOlderEvent(t *tes t.Fatalf("FORGE holds %d messages (%v), want %d", info.State.Msgs, err, want) } } - named := func(seq uint64) []delivery.OpenPull { + named := func(seq uint64) []forgemsg.OpenPull { t.Helper() raw, err := forge.GetMsg(b.ctx, seq) if err != nil || raw.Subject != openSubject { t.Fatalf("message %d: %v %v, want the open message", seq, raw, err) } - f, err := delivery.DecodeForge(raw.Data) + f, err := forgemsg.DecodeForge(raw.Data) if err != nil { t.Fatal(err) } return f.Open } - bothOpen := []delivery.OpenPull{{Repository: "novox/mesh-catalog", Number: 300}, {Repository: "novox/mesh-delivery", Number: 12}} - same := func(got []delivery.OpenPull) bool { + bothOpen := []forgemsg.OpenPull{{Repository: "novox/mesh-catalog", Number: 300}, {Repository: "novox/mesh-delivery", Number: 12}} + same := func(got []forgemsg.OpenPull) bool { return len(got) == 2 && got[0] == bothOpen[0] && got[1] == bothOpen[1] } @@ -713,7 +720,7 @@ func forgeMerged(t *testing.T, b *bed) []string { if err != nil { continue } - if f, err := delivery.DecodeForge(raw.Data); err == nil && f.Kind == delivery.ForgeMerged { + if f, err := forgemsg.DecodeForge(raw.Data); err == nil && f.Kind == forgemsg.ForgeMerged { out = append(out, f.Repository) } } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go index 26331490..25e4b970 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go @@ -21,6 +21,7 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/delivery" "mesh-delivery-forge-bridge/internal/bridge" @@ -95,7 +96,7 @@ func handleGrouped(ctx context.Context, m delivered, pub publisher, trees bridge if id := m.Headers().Get("x-event-id"); id != "" { opts = append(opts, jetstream.WithMsgID(id)) } - var member delivery.Forge + var member forgemsg.Forge grouped := false if g != nil { msg.Data, member, grouped = g.member(ctx, msg.Data) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go index 6234a90d..ede623fd 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go @@ -18,7 +18,7 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "mesh-delivery-forge-bridge/internal/bridge" ) @@ -85,7 +85,7 @@ func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull" { t.Fatalf("published on %v", pub.subjects) } - f, err := delivery.DecodeForge(pub.data[0]) + f, err := forgemsg.DecodeForge(pub.data[0]) if err != nil || f.Number != 12 || f.Branch != "feat/x" || f.Head != head || f.Action != "updated" || !f.DirsKnown { t.Errorf("message %+v err %v", f, err) } @@ -172,7 +172,7 @@ func TestMergeCallsTheToolOnceAndWritesTheMergedMessage(t *testing.T) { // 1236/ if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.merged" { t.Fatalf("published on %v", pub.subjects) } - f, err := delivery.DecodeForge(pub.data[0]) + f, err := forgemsg.DecodeForge(pub.data[0]) if err != nil || f.Root != "r0" || f.Trees[""] != "t0" || f.Trees["modules/a"] != "t1" || f.Merge == "" { t.Errorf("message %+v err %v", f, err) } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/open.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/open.go index f09913f9..a93543dd 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/open.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/open.go @@ -10,7 +10,7 @@ import ( "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "mesh-delivery-forge-bridge/internal/bridge" ) @@ -41,7 +41,7 @@ func (r *recorder) Publish(ctx context.Context, subject string, payload []byte, if err != nil { return ack, err } - f, derr := delivery.DecodeForge(payload) + f, derr := forgemsg.DecodeForge(payload) if derr != nil { return ack, nil } @@ -101,7 +101,7 @@ func syncOpen(ctx context.Context, call caller, pub publisher, heads headStore, } var differing []bridge.Message for _, m := range msgs { - f, err := delivery.DecodeForge(m.Data) + f, err := forgemsg.DecodeForge(m.Data) if err != nil { differing = append(differing, m) continue diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/open_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/open_test.go index c1009b3b..a9caade0 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/open_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/open_test.go @@ -13,7 +13,7 @@ import ( "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" ) // openAnswer is gitea_open_pulls's answer as mesh-catalog modules/gitea/client.ts:369-381 builds its entries inside {pulls}. @@ -43,7 +43,7 @@ func TestSyncOpenWritesOneUpdatedMessagePerOpenPull(t *testing.T) { // 1237/AC1, } want := []string{"mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull", "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-catalog.pull"} for i, w := range want { - f, err := delivery.DecodeForge(pub.data[i]) + f, err := forgemsg.DecodeForge(pub.data[i]) if pub.subjects[i] != w || err != nil || f.Action != "updated" || f.DirsKnown || f.Branch != "feat/x" { t.Errorf("message %d on %s = %+v err %v", i, pub.subjects[i], f, err) } @@ -112,8 +112,8 @@ func TestSyncOpenWritesPullsForNewHeadsThenOneOpenMessageNamingAll(t *testing.T) if pub.subjects[1] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.open" { t.Fatalf("last message on %s, want the open message after the pull messages", pub.subjects[1]) } - f, err := delivery.DecodeForge(pub.data[1]) - want := []delivery.OpenPull{{Repository: "novox/mesh-catalog", Number: 300}, {Repository: "novox/mesh-delivery", Number: 12}} + f, err := forgemsg.DecodeForge(pub.data[1]) + want := []forgemsg.OpenPull{{Repository: "novox/mesh-catalog", Number: 300}, {Repository: "novox/mesh-delivery", Number: 12}} if err != nil || len(f.Open) != 2 || f.Open[0] != want[0] || f.Open[1] != want[1] { t.Errorf("open message = %+v err %v, want it to name %v, the pull request whose head did not differ among them", f, err, want) } @@ -140,10 +140,10 @@ func TestWhatIsWrittenOnFORGEIsKeptInTheBucket(t *testing.T) { // 1279/AC6 pub := &fakePub{} heads := newHeads() r := &recorder{pub: pub, heads: heads, logger: log.New(&bytes.Buffer{}, "", 0)} - write := func(f delivery.Forge) { + write := func(f forgemsg.Forge) { t.Helper() - ev, _ := delivery.ForgeEvent(f.Repository, f.Kind) - raw, err := delivery.EncodeForge(f) + ev, _ := forgemsg.ForgeEvent(f.Repository, f.Kind) + raw, err := forgemsg.EncodeForge(f) if err != nil { t.Fatal(err) } @@ -151,19 +151,19 @@ func TestWhatIsWrittenOnFORGEIsKeptInTheBucket(t *testing.T) { // 1279/AC6 t.Fatal(err) } } - p := delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: delivery.PullUpdated} + p := forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: forgemsg.PullUpdated} write(p) if heads.get("novox.mesh-delivery.12") != head { t.Fatalf("bucket %v", heads.vals) } - p.Action = delivery.PullClosed + p.Action = forgemsg.PullClosed write(p) if _, kept := heads.vals["novox.mesh-delivery.12"]; kept { t.Errorf("a closed pull request is still kept: %v", heads.vals) } - p.Action = delivery.PullUpdated + p.Action = forgemsg.PullUpdated write(p) - write(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Merge: "m", Root: "r", Trees: map[string]string{"": "t"}}) + write(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Merge: "m", Root: "r", Trees: map[string]string{"": "t"}}) if _, kept := heads.vals["novox.mesh-delivery.12"]; kept { t.Errorf("a merged pull request is still kept: %v", heads.vals) } diff --git a/modules/mesh-delivery-forge-bridge/forgemsg/forge.go b/modules/mesh-delivery-forge-bridge/forgemsg/forge.go new file mode 100644 index 00000000..16f632dd --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/forgemsg/forge.go @@ -0,0 +1,413 @@ +// Package forgemsg is the forge bridge's contract on its own stream `forge` (novox/hq ADR 0633 §4): its messages (Forge, +// one per pull request, merge, refusal or list of open pull requests) and a delivery group's end (ForgeGroupEnd). The +// forge bridge writes them; the indexer and the gate read them. It is the forge bridge's own package: one module owns +// these types, so they live with it and not in mesh-sdk (the operator, 2026-11-04). +package forgemsg + +import ( + "encoding/json" + "fmt" + "reflect" + "regexp" + "strings" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Version is the version of every value this package writes and the only one it reads: mesh-sdk's delivery values'. +const Version = delivery.Version + +// ForgeStream is the forge bridge's stream as its manifest declares it; the broker derives the real names. +const ForgeStream = "forge" + +// The kinds of forge message, the last part of a forge event's name. +const ( + ForgePull = "pull" + ForgeMerged = "merged" + ForgeRefused = "refused" + ForgeOpen = "open" +) + +// OpenPull names an open pull request into the trunk: the repository as `/` and the number. +type OpenPull struct { + Repository string `json:"repository"` + Number int `json:"number"` +} + +// OpenEvent is the name of an open message: `forge.open`. It names no repository: it says every open pull request of +// every repository the forge bridge reads (novox/hq ADR 0443 §5). +func OpenEvent() string { return "forge." + ForgeOpen } + +// The actions of a pull message. +const ( + PullOpened = "opened" + PullUpdated = "updated" + PullClosed = "closed" +) + +// checkRepository refuses a repository that is not `/`: exactly one slash, each part a valid name. +func checkRepository(repository string) error { + owner, name, ok := strings.Cut(repository, "/") + if !ok || strings.Contains(name, "/") { + return fmt.Errorf("delivery: the repository %q is not /, with exactly one slash", repository) + } + if err := checkName("repository owner", owner); err != nil { + return fmt.Errorf("delivery: the repository %q: %w", repository, err) + } + if err := checkName("repository name", name); err != nil { + return fmt.Errorf("delivery: the repository %q: %w", repository, err) + } + if strings.Contains(owner, "/") { + return fmt.Errorf("delivery: the repository %q is not /, with exactly one slash", repository) + } + return nil +} + +func checkForgeKind(kind string) error { + switch kind { + case ForgePull, ForgeMerged, ForgeRefused, ForgeOpen: + return nil + } + return fmt.Errorf("delivery: the forge kind %q is not pull, merged, refused or open", kind) +} + +// ForgeEvent is the name of a forge message of the given kind: `forge...`. +func ForgeEvent(repository, kind string) (string, error) { + if err := checkForgeKind(kind); err != nil { + return "", err + } + if kind == ForgeOpen { + return "", fmt.Errorf("delivery: an open message names no repository, its event is %s", OpenEvent()) + } + if err := checkRepository(repository); err != nil { + return "", err + } + owner, name, _ := strings.Cut(repository, "/") + return "forge." + owner + "." + name + "." + kind, nil +} + +// ParseForgeEvent is the inverse of ForgeEvent. +func ParseForgeEvent(event string) (repository, kind string, err error) { + if event == OpenEvent() { + return "", ForgeOpen, nil + } + parts := strings.Split(event, ".") + if len(parts) != 4 || parts[0] != "forge" { + return "", "", fmt.Errorf("delivery: %q is not a forge event, forge...", event) + } + repository = parts[1] + "/" + parts[2] + if _, err := ForgeEvent(repository, parts[3]); err != nil { + return "", "", fmt.Errorf("delivery: %q is not a forge event: %w", event, err) + } + return repository, parts[3], nil +} + +// Forge is a message of the forge bridge's stream, trimmed to repository, branch, pull request and head. A pull +// message says Action; a merged message carries the merge commit, the tree of every directory holding a +// module.json and the repository's root tree; a refused message names the owed act it answers and the forge's own +// words. A pull message may also carry the module directories its head changes (Dirs, in the order the forge said) and +// DirsKnown, true when the forge could say them: known and empty is DirsKnown with no Dirs, not known is neither +// (the forge's module_dirs and module_dirs_said, mesh-catalog modules/gitea/trunk.ts). Other messages carry neither. +type Forge struct { + V int `json:"v"` + Kind string `json:"kind"` + Repository string `json:"repository"` + Branch string `json:"branch,omitempty"` + Number int `json:"number,omitempty"` + Head string `json:"head,omitempty"` + Action string `json:"action,omitempty"` + Dirs []string `json:"dirs,omitempty"` + DirsKnown bool `json:"dirs-known,omitempty"` + Merge string `json:"merge,omitempty"` + Trees map[string]string `json:"trees,omitempty"` + Root string `json:"root,omitempty"` + // Manifests are the module.json bytes, exact, of each module directory: of the merge's trees on a merged message, + // of Dirs at the head on a pull message. Locks are, of each such directory, its lock files' hex sha256 by path + // relative to it (novox/hq ADR 0465 clause 1). Older writers send neither. + Manifests map[string]string `json:"manifests,omitempty"` + Locks map[string]map[string]string `json:"locks,omitempty"` + // Paths are, of each module directory whose manifest names paths on every artifact, each named path's git object + // id at the same commit as its manifest, by path relative to the directory (novox/hq ADR 0499 clause 3). The + // indexer names the paths; a message without them gives such a module none. Older writers send none. + Paths map[string]map[string]string `json:"paths,omitempty"` + // Packages are, of each module directory, the mesh packages its lock file names (MeshPackagesOf; novox/hq ADR 0573 + // §2): the indexer keeps them in the module's inputs and takes each one's latest published version into its + // fingerprint. Absent for a directory that uses none, and from an older writer. + Packages map[string][]string `json:"packages,omitempty"` + Act string `json:"act,omitempty"` + Words string `json:"words,omitempty"` + Open []OpenPull `json:"open,omitempty"` + // RetiredPlacements is the placement files a bridge before ADR 0567 carried (ADR 0549 §2, ADR 0559 §2): placement is + // the seat mesh-placement's state now and no repository holds it (ADR 0567 §11), so a message carrying them is + // refused, loudly, rather than read with them dropped. + RetiredPlacements json.RawMessage `json:"placements,omitempty"` + RetiredRead bool `json:"placements-read,omitempty"` + // Predefined is the catalogue's predefined.json, exact, at the merge commit of a merged message (novox/hq ADR 0565 + // §2): what the indexer keeps as its state `predefined`. PredefinedRead says the bridge read the merge and it holds + // no predefined.json. Carried on a merged message only; an older bridge sends neither, and the indexer keeps what it + // held. + Predefined json.RawMessage `json:"predefined,omitempty"` + PredefinedRead bool `json:"predefined-read,omitempty"` + // Group is the delivery group a merged message's pull request was merged in, and Members every approved member of + // that group with its approved head, this one included (novox/hq ADR 0633 §4): the forge bridge sets both from the + // gate's merges state, and the indexer gathers the group's merges until the group's end (ForgeGroupEnd). A merge + // with no group carries neither, and an older bridge sends neither; an older reader ignores both. + Group string `json:"group,omitempty"` + Members []delivery.Head `json:"members,omitempty"` +} + +// checkGroup refuses a group on anything but a merged message, a group without its members or members without a +// group, a member that is not a repository's pull request at a head, a member named twice, and members that leave out +// the message's own pull request. +func (f Forge) checkGroup() error { + if f.Group == "" && len(f.Members) == 0 { + return nil + } + if f.Kind != ForgeMerged { + return fmt.Errorf("delivery: only a merged message names its delivery group, not a %s message", f.Kind) + } + if f.Group == "" || len(f.Members) == 0 { + return fmt.Errorf("delivery: a merged message names its group and the group's members together, or neither") + } + self := false + seen := map[string]bool{} + for _, m := range f.Members { + if err := checkRepository(m.Repository); err != nil { + return err + } + if m.Number <= 0 || m.Head == "" { + return fmt.Errorf("delivery: the member %s of the group %q needs its pull request number and head", m.Repository, f.Group) + } + k := fmt.Sprintf("%s#%d", m.Repository, m.Number) + if seen[k] { + return fmt.Errorf("delivery: the group %q names its member %s twice", f.Group, k) + } + seen[k] = true + self = self || (delivery.SameRepository(m.Repository, f.Repository) && m.Number == f.Number) + } + if !self { + return fmt.Errorf("delivery: the members of the group %q leave out the merged pull request %s#%d", f.Group, f.Repository, f.Number) + } + return nil +} + +// checkPredefined refuses the predefined list on anything but a merged message, a list that is not one JSON object, and +// a list said both present and absent. +func (f Forge) checkPredefined() error { + if len(f.Predefined) == 0 && !f.PredefinedRead { + return nil + } + if f.Kind != ForgeMerged { + return fmt.Errorf("delivery: only a merged message carries the predefined list, not a %s message", f.Kind) + } + if len(f.Predefined) > 0 && f.PredefinedRead { + return fmt.Errorf("delivery: a merged message carries the predefined list and says the merge holds none (predefined-read)") + } + var list map[string]json.RawMessage + if len(f.Predefined) > 0 && json.Unmarshal(f.Predefined, &list) != nil { + return fmt.Errorf("delivery: the predefined list a merged message carries is not one JSON object") + } + return nil +} + +func (f Forge) check() error { + if err := checkForgeKind(f.Kind); err != nil { + return err + } + if f.Kind == ForgeOpen { + return f.checkOpen() + } + if len(f.Open) > 0 { + return fmt.Errorf("delivery: only an open message names open pull requests") + } + if len(f.RetiredPlacements) > 0 || f.RetiredRead { + return fmt.Errorf("delivery: a forge message carries placement files, which no repository holds since ADR 0567: " + + "placement is the seat mesh-placement's state") + } + if err := f.checkPredefined(); err != nil { + return err + } + if err := f.checkGroup(); err != nil { + return err + } + if err := checkRepository(f.Repository); err != nil { + return err + } + switch f.Kind { + case ForgePull: + if f.Number <= 0 { + return fmt.Errorf("delivery: a pull message needs its pull request number") + } + if f.Head == "" { + return fmt.Errorf("delivery: a pull message needs its head") + } + switch f.Action { + case PullOpened, PullUpdated, PullClosed: + default: + return fmt.Errorf("delivery: the pull action %q is not opened, updated or closed", f.Action) + } + if len(f.Dirs) > 0 && !f.DirsKnown { + return fmt.Errorf("delivery: a pull message with module directories must say it knows them (dirs-known)") + } + dirs := map[string]bool{} + for _, d := range f.Dirs { + dirs[d] = true + } + if err := f.checkInputs(func(d string) bool { return dirs[d] }, "its module directories"); err != nil { + return err + } + case ForgeMerged: + if len(f.Dirs) > 0 || f.DirsKnown { + return fmt.Errorf("delivery: a merged message carries no module directories") + } + if f.Number <= 0 { + return fmt.Errorf("delivery: a merged message needs its pull request number") + } + if f.Head == "" { + return fmt.Errorf("delivery: a merged message needs its head") + } + if f.Merge == "" { + return fmt.Errorf("delivery: a merged message needs its merge commit") + } + if f.Root == "" { + return fmt.Errorf("delivery: a merged message needs the repository's root tree") + } + // No trees is a merge that leaves the repository with no module.json anywhere (a documents repository, or one + // whose last module was removed): it is written all the same, so the indexer takes the merge in and drops the + // repository's modules (novox/hq issue 1126). An older writer always sent at least one tree, so none is unambiguous; + // such a message carries no manifests, locks or paths (checkInputs below: each names a directory of its trees). + for dir, tree := range f.Trees { + if tree == "" { + return fmt.Errorf("delivery: a merged message's tree of the directory %q is empty", dir) + } + } + if err := f.checkInputs(func(d string) bool { _, ok := f.Trees[d]; return ok }, "its trees"); err != nil { + return err + } + case ForgeRefused: + if len(f.Dirs) > 0 || f.DirsKnown { + return fmt.Errorf("delivery: a refused message carries no module directories") + } + if len(f.Manifests) > 0 || len(f.Locks) > 0 || len(f.Paths) > 0 { + return fmt.Errorf("delivery: a refused message carries no manifests, lock hashes or named paths") + } + if f.Act == "" { + return fmt.Errorf("delivery: a refused message needs the act it answers") + } + if f.Words == "" { + return fmt.Errorf("delivery: a refused message needs the forge's words") + } + } + return nil +} + +// checkInputs refuses a manifest or a lock hash of a directory that known does not know (of is what it knows them +// by), an empty manifest and an empty lock path or hash, naming the directory. +func (f Forge) checkInputs(known func(dir string) bool, of string) error { + for dir, m := range f.Manifests { + if !known(dir) { + return fmt.Errorf("delivery: the manifest of the directory %q is not of %s", dir, of) + } + if m == "" { + return fmt.Errorf("delivery: the manifest of the directory %q is empty", dir) + } + } + for dir, locks := range f.Locks { + if !known(dir) { + return fmt.Errorf("delivery: the lock hashes of the directory %q are not of %s", dir, of) + } + for path, hash := range locks { + if path == "" || hash == "" { + return fmt.Errorf("delivery: the lock %q of the directory %q has an empty path or hash", path, dir) + } + } + } + for dir, paths := range f.Paths { + if !known(dir) { + return fmt.Errorf("delivery: the named paths of the directory %q are not of %s", dir, of) + } + for path, object := range paths { + if path == "" || object == "" { + return fmt.Errorf("delivery: the named path %q of the directory %q has an empty path or object", path, dir) + } + } + } + return nil +} + +// checkOpen refuses an open message that names a repository, a branch, a head or anything of the other kinds, or +// an open pull request that is not `/` and a number. +func (f Forge) checkOpen() error { + if f.Repository != "" || f.Branch != "" || f.Number != 0 || f.Head != "" || f.Action != "" || len(f.Dirs) > 0 || f.DirsKnown || + f.Merge != "" || len(f.Trees) > 0 || len(f.Manifests) > 0 || len(f.Locks) > 0 || len(f.Paths) > 0 || f.Root != "" || f.Act != "" || f.Words != "" || f.Group != "" || len(f.Members) > 0 { + return fmt.Errorf("delivery: an open message carries only the open pull requests it names") + } + for _, p := range f.Open { + if err := checkRepository(p.Repository); err != nil { + return err + } + if p.Number <= 0 { + return fmt.Errorf("delivery: the open pull request of %s has no number", p.Repository) + } + } + return nil +} + +// EncodeForge is f as JSON at Version; it refuses a message missing what its kind needs. +func EncodeForge(f Forge) ([]byte, error) { + if err := f.check(); err != nil { + return nil, err + } + f.V = Version + return json.Marshal(f) +} + +// DecodeForge reads a forge message; it refuses an unknown version (ErrUnknownVersion) and what EncodeForge refuses. +func DecodeForge(data []byte) (Forge, error) { + var f Forge + if err := decode(data, &f); err != nil { + return Forge{}, err + } + if err := f.check(); err != nil { + return Forge{}, err + } + return f, nil +} + +// SameForge says whether two forge messages are the same value. +func SameForge(a, b Forge) bool { return reflect.DeepEqual(a, b) } + +var nameOK = regexp.MustCompile(`^[-/_=a-zA-Z0-9]+$`) + +// checkName refuses a name that is empty or holds a dot, a wildcard, whitespace or a character a bus key refuses +// (mesh-sdk delivery's rule for a name). +func checkName(kind, name string) error { + if name == "" { + return fmt.Errorf("delivery: the %s name is empty", kind) + } + if !nameOK.MatchString(name) { + return fmt.Errorf("delivery: the %s name %q holds a dot, a wildcard, whitespace or a character a key refuses", kind, name) + } + return nil +} + +// decode reads a value of Version, refusing another or none with delivery.ErrUnknownVersion, as mesh-sdk delivery's +// values are read. +func decode(data []byte, into any) error { + var probe struct { + V *int `json:"v"` + } + if err := json.Unmarshal(data, &probe); err != nil { + return fmt.Errorf("delivery: value is not JSON: %w", err) + } + if probe.V == nil { + return fmt.Errorf("%w: no version in the value, this reader reads %d", delivery.ErrUnknownVersion, Version) + } + if *probe.V != Version { + return fmt.Errorf("%w: the value is version %d, this reader reads %d", delivery.ErrUnknownVersion, *probe.V, Version) + } + if err := json.Unmarshal(data, into); err != nil { + return fmt.Errorf("delivery: value does not fit its shape: %w", err) + } + return nil +} diff --git a/modules/mesh-delivery-forge-bridge/forgemsg/forge_test.go b/modules/mesh-delivery-forge-bridge/forgemsg/forge_test.go new file mode 100644 index 00000000..1ee2382a --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/forgemsg/forge_test.go @@ -0,0 +1,193 @@ +package forgemsg + +import ( + "encoding/json" + "errors" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +const oddManifest = "{\n\t\"module\": \"a\",\n \"z\": 1, \"a\": [ \"x\",\"y\" ],\n \"why\": \"é — & \\n\"\n}\n\n" + +func sampleForges() []Forge { + return []Forge{ + {Kind: ForgePull, Repository: "novox/mesh-sdk", Branch: "feat/x", Number: 36, Head: "aaaa", Action: "opened"}, + {Kind: ForgeMerged, Repository: "novox/mesh-sdk", Number: 36, Head: "aaaa", Merge: "cccc", + Trees: map[string]string{"modules/a": "t1"}, Root: "t0"}, + {Kind: ForgeRefused, Repository: "novox/mesh-sdk", Act: "novox/mesh-sdk.36.aaaa.view", Words: "403 forbidden"}, + {Kind: ForgeOpen, Open: []OpenPull{{Repository: "novox/mesh-sdk", Number: 36}, {Repository: "novox/mesh-control", Number: 7}}}, + // 1585/T1: a group member's merged message names its group and every approved member (ADR 0633 §4). + {Kind: ForgeMerged, Repository: "novox/mesh-sdk", Number: 37, Head: "dddd", Merge: "eeee", Root: "t0", + Group: "feat/take-in", Members: []delivery.Head{{Repository: "novox/mesh-sdk", Number: 37, Head: "dddd"}, {Repository: "novox/mesh-delivery", Number: 9, Head: "ffff"}}}, + } +} + +func mergedWithInputs() Forge { + return Forge{Kind: ForgeMerged, Repository: "novox/mesh-sdk", Number: 36, Head: "aaaa", Merge: "cccc", Root: "t0", + Trees: map[string]string{"a": "t1", "b": "t2"}, + Manifests: map[string]string{"a": oddManifest}, + Locks: map[string]map[string]string{"a": {"go.sum": strings.Repeat("ab", 32)}}} +} + +// TestForge holds the forge bridge's contract, moved whole from mesh-sdk go/delivery with its tests (1200/AC1, AC3, +// AC4; 1279/AC5; 1442/AC1-AC4; 1585/T1). +func TestForge(t *testing.T) { + t.Run("events", func(t *testing.T) { + ev, err := ForgeEvent("novox/mesh-sdk", ForgeMerged) + if err != nil || ev != "forge.novox.mesh-sdk.merged" { + t.Fatalf("forge event = %q, %v", ev, err) + } + if repo, kind, err := ParseForgeEvent(ev); err != nil || repo != "novox/mesh-sdk" || kind != ForgeMerged { + t.Fatalf("parse forge event = %q %q %v", repo, kind, err) + } + for _, bad := range []struct{ repo, kind string }{ + {"", "pull"}, {"novox", "pull"}, {"a/b/c", "pull"}, {"/b", "pull"}, {"a/", "pull"}, + {"a.b/c", "pull"}, {"a/b c", "pull"}, {"a/b", "closed"}, {"a/b", ""}, {"a/b", ForgeOpen}, + } { + if _, err := ForgeEvent(bad.repo, bad.kind); err == nil { + t.Errorf("forge event %q %q accepted", bad.repo, bad.kind) + } + } + for _, bad := range []string{"", "forge.a.b", "forge.a.b.closed", "x.a.b.pull", "forge.a.b.pull.x", "forge..b.pull", ForgeGroupEndEvent} { + if _, _, err := ParseForgeEvent(bad); err == nil { + t.Errorf("parse forge event %q accepted", bad) + } + } + if repo, kind, err := ParseForgeEvent(OpenEvent()); OpenEvent() != "forge.open" || err != nil || repo != "" || kind != ForgeOpen { + t.Errorf("open event %q = %q %q %v", OpenEvent(), repo, kind, err) + } + }) + t.Run("round trip", func(t *testing.T) { + fs := append(sampleForges(), mergedWithInputs()) + for _, f := range fs { + data, err := EncodeForge(f) + if err != nil { + t.Fatalf("%s: %v", f.Kind, err) + } + var probe map[string]any + if err := json.Unmarshal(data, &probe); err != nil || probe["v"] != float64(1) { + t.Fatalf("%s: json %s does not carry v:1", f.Kind, data) + } + f.V = Version + if got, err := DecodeForge(data); err != nil || !SameForge(got, f) { + t.Errorf("forge %s round trip: %+v %v", f.Kind, got, err) + } + } + // An empty list is an open message too: nothing is open. + if data, err := EncodeForge(Forge{Kind: ForgeOpen, Open: []OpenPull{}}); err != nil { + t.Errorf("empty open: %v", err) + } else if f, err := DecodeForge(data); err != nil || f.Kind != ForgeOpen || len(f.Open) != 0 { + t.Errorf("empty open round trip: %+v %v", f, err) + } + ge := ForgeGroupEnd{Group: "feat/take-in", Members: []GroupMerge{{Repository: "novox/mesh-sdk", Number: 37, Merge: "eeee"}}} + data, err := EncodeForgeGroupEnd(ge) + ge.V = Version + if got, err2 := DecodeForgeGroupEnd(data); err != nil || err2 != nil || !SameForgeGroupEnd(got, ge) { + t.Errorf("group end round trip: %+v %v %v", got, err, err2) + } + // An older bridge's message reads, and is written back byte for byte. + old := `{"v":1,"kind":"merged","repository":"novox/mesh-sdk","number":36,"head":"aaaa","merge":"cccc","trees":{"a":"t1"},"root":"t0"}` + if f, err := DecodeForge([]byte(old)); err != nil || f.Manifests != nil { + t.Fatalf("%+v, %v", f, err) + } else if data, err := EncodeForge(f); err != nil || string(data) != old { + t.Errorf("%s, %v, want exactly %s", data, err, old) + } + // A merge leaving the repository with no module.json is a message too (issue 1126). + f := sampleForges()[1] + f.Trees = nil + if data, err := EncodeForge(f); err != nil { + t.Errorf("a merged message with no trees is refused: %v", err) + } else if got, err := DecodeForge(data); err != nil || len(got.Trees) != 0 { + t.Errorf("a merged message with no trees comes back as %+v, %v", got, err) + } + for name, dec := range map[string]func([]byte) error{ + "forge": func(b []byte) error { _, err := DecodeForge(b); return err }, + "group end": func(b []byte) error { _, err := DecodeForgeGroupEnd(b); return err }, + } { + for _, in := range []string{`{"group":"g"}`, `{"v":2,"group":"g"}`} { + if err := dec([]byte(in)); !errors.Is(err, delivery.ErrUnknownVersion) { + t.Errorf("%s decode of %s = %v, want delivery.ErrUnknownVersion", name, in, err) + } + } + } + }) + t.Run("refused", func(t *testing.T) { + at := func(i int, mut func(*Forge)) Forge { f := sampleForges()[i]; mut(&f); return f } + in := func(mut func(*Forge)) Forge { f := mergedWithInputs(); mut(&f); return f } + pullIn := func(mut func(*Forge)) Forge { + f := Forge{Kind: ForgePull, Repository: "novox/mesh-sdk", Number: 1, Head: "h", Action: PullOpened, + Dirs: []string{"a"}, DirsKnown: true, Manifests: map[string]string{"a": oddManifest}, + Locks: map[string]map[string]string{"a": {"go.sum": strings.Repeat("0", 64)}}} + mut(&f) + return f + } + for _, c := range []struct { + name, want string + f Forge + }{ + {"merged without merge commit", "merge", at(1, func(f *Forge) { f.Merge = "" })}, + {"merged without root", "root", at(1, func(f *Forge) { f.Root = "" })}, + {"merged with a manifest and no trees", "modules/a", at(1, func(f *Forge) { f.Trees = nil; f.Manifests = map[string]string{"modules/a": "{}"} })}, + {"merged with an empty tree", "modules/a", at(1, func(f *Forge) { f.Trees = map[string]string{"modules/a": ""} })}, + {"pull with an unknown action", "reopened", at(0, func(f *Forge) { f.Action = "reopened" })}, + {"pull without head", "head", at(0, func(f *Forge) { f.Head = "" })}, + {"refused without words", "words", at(2, func(f *Forge) { f.Words = "" })}, + {"refused without act", "act", at(2, func(f *Forge) { f.Act = "" })}, + {"unknown kind", "closed", Forge{Kind: "closed", Repository: "a/b"}}, + {"bad repository", "nope", Forge{Kind: ForgePull, Repository: "nope", Number: 1, Head: "h", Action: "opened"}}, + {"open with a repository", "only the open", Forge{Kind: ForgeOpen, Repository: "a/b", Open: []OpenPull{{Repository: "a/b", Number: 1}}}}, + {"open with a bad entry", "", Forge{Kind: ForgeOpen, Open: []OpenPull{{Repository: "a", Number: 1}}}}, + {"open with a zero number", "no number", Forge{Kind: ForgeOpen, Open: []OpenPull{{Repository: "a/b"}}}}, + {"pull naming open pull requests", "only an open message", Forge{Kind: ForgePull, Repository: "a/b", Number: 1, Head: "h", Action: "updated", Open: []OpenPull{{Repository: "a/b", Number: 1}}}}, + {"manifest outside trees", `"zz"`, in(func(f *Forge) { f.Manifests["zz"] = "{}" })}, + {"locks outside trees", `"yy"`, in(func(f *Forge) { f.Locks["yy"] = map[string]string{"go.sum": "h"} })}, + {"empty manifest", `"b"`, in(func(f *Forge) { f.Manifests["b"] = "" })}, + {"empty lock hash", `"a"`, in(func(f *Forge) { f.Locks["a"]["go.sum"] = "" })}, + {"named paths outside trees", `"other"`, in(func(f *Forge) { f.Paths = map[string]map[string]string{"other": {"src": "o"}} })}, + {"a named path with no object", `"src"`, in(func(f *Forge) { f.Paths = map[string]map[string]string{"a": {"src": ""}} })}, + {"placement files", "mesh-placement", in(func(f *Forge) { f.RetiredPlacements = []byte(`{"anchor":"{}"}`) })}, + {"placements read", "mesh-placement", in(func(f *Forge) { f.RetiredRead = true })}, + {"predefined not an object", "one JSON object", in(func(f *Forge) { f.Predefined = []byte(`["domain"]`) })}, + {"predefined both", "says the merge holds none", in(func(f *Forge) { f.Predefined = []byte(`{}`); f.PredefinedRead = true })}, + {"pull with the predefined list", "only a merged message", Forge{Kind: ForgePull, Repository: "novox/mesh-sdk", Number: 1, Head: "h", Action: PullOpened, PredefinedRead: true}}, + {"pull manifest outside dirs", `"b"`, pullIn(func(f *Forge) { f.Manifests["b"] = "{}" })}, + {"pull locks outside dirs", `"c"`, pullIn(func(f *Forge) { f.Locks["c"] = map[string]string{"go.sum": "h"} })}, + {"refused with manifests", "no manifests", Forge{Kind: ForgeRefused, Repository: "a/b", Act: "x", Words: "no", Manifests: map[string]string{"a": "{}"}}}, + {"open with locks", "only the open", Forge{Kind: ForgeOpen, Locks: map[string]map[string]string{"a": {"go.sum": "h"}}}}, + // 1585/T1: a group only on a merged message, with its members, the message's own pull request among them. + {"pull with a group", "only a merged message names its delivery group", at(0, func(f *Forge) { + f.Group, f.Members = "g", []delivery.Head{{Repository: "novox/mesh-sdk", Number: 36, Head: "aaaa"}} + })}, + {"merged with a group and no members", "together", at(4, func(f *Forge) { f.Members = nil })}, + {"merged with members and no group", "together", at(4, func(f *Forge) { f.Group = "" })}, + {"merged with members leaving itself out", "leave out", at(4, func(f *Forge) { f.Members = f.Members[1:] })}, + {"merged with a member twice", "twice", at(4, func(f *Forge) { f.Members = append(f.Members, f.Members[0]) })}, + {"open with a group", "only the open pull requests", Forge{Kind: ForgeOpen, Group: "g"}}, + } { + if _, err := EncodeForge(c.f); err == nil || !strings.Contains(err.Error(), c.want) { + t.Errorf("%s: %v, want a refusal naming %q", c.name, err, c.want) + } + } + // A member's repository in another case still names the merged pull request (10c91fba). + f := sampleForges()[4] + f.Members[0].Repository = "Novox/Mesh-SDK" + if _, err := EncodeForge(f); err != nil { + t.Errorf("a member in another case: %v", err) + } + for _, c := range []struct { + name, want string + e ForgeGroupEnd + }{ + {"group end without members", "no member", ForgeGroupEnd{Group: "g"}}, + {"group end without a group", "names its group", ForgeGroupEnd{Members: []GroupMerge{{Repository: "a/b", Number: 1, Merge: "m"}}}}, + {"group end with a member without its merge", "merge commit", ForgeGroupEnd{Group: "g", Members: []GroupMerge{{Repository: "a/b", Number: 1}}}}, + {"group end with a member twice", "twice", ForgeGroupEnd{Group: "g", Members: []GroupMerge{{Repository: "a/b", Number: 1, Merge: "m"}, {Repository: "a/b", Number: 1, Merge: "m"}}}}, + } { + if _, err := EncodeForgeGroupEnd(c.e); err == nil || !strings.Contains(err.Error(), c.want) { + t.Errorf("%s: %v, want a refusal naming %q", c.name, err, c.want) + } + } + }) +} diff --git a/modules/mesh-delivery-forge-bridge/forgemsg/go.mod b/modules/mesh-delivery-forge-bridge/forgemsg/go.mod new file mode 100644 index 00000000..f54fcd87 --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/forgemsg/go.mod @@ -0,0 +1,5 @@ +module git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg + +go 1.26.0 + +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659 diff --git a/modules/mesh-delivery-forge-bridge/forgemsg/go.sum b/modules/mesh-delivery-forge-bridge/forgemsg/go.sum new file mode 100644 index 00000000..5e18d943 --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/forgemsg/go.sum @@ -0,0 +1,14 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659 h1:tK7MzxVm/ECNg1Yblhh1ynVEthbZCg2mSPW0wQZC0b0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= diff --git a/modules/mesh-delivery-forge-bridge/forgemsg/group.go b/modules/mesh-delivery-forge-bridge/forgemsg/group.go new file mode 100644 index 00000000..a36702c5 --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/forgemsg/group.go @@ -0,0 +1,78 @@ +package forgemsg + +import ( + "encoding/json" + "fmt" + "reflect" +) + +// ForgeGroupEndEvent is the name of a delivery group's end on the forge bridge's stream (novox/hq ADR 0633 §4): the +// bridge publishes it once every approved member of the group is merged, after the last member's merged message. It +// has three parts, so no repository's forge event (`forge...`) is ever this name. +const ForgeGroupEndEvent = "forge.group.merged" + +// GroupMerge is one merged member of a delivery group: the repository as `/`, its pull request and the +// merge commit the forge made of it. +type GroupMerge struct { + Repository string `json:"repository"` + Number int `json:"number"` + Merge string `json:"merge"` +} + +// ForgeGroupEnd is the body of ForgeGroupEndEvent: the group and every member merged in it, with its merge commit. +// The indexer takes the members it holds in as one source revision once it hears it (ADR 0633 §4). +type ForgeGroupEnd struct { + V int `json:"v"` + Group string `json:"group"` + Members []GroupMerge `json:"members"` +} + +func (e ForgeGroupEnd) check() error { + if e.Group == "" { + return fmt.Errorf("delivery: a group's end names its group") + } + if len(e.Members) == 0 { + return fmt.Errorf("delivery: the end of the group %q names no member merged in it", e.Group) + } + seen := map[string]bool{} + for _, m := range e.Members { + if err := checkRepository(m.Repository); err != nil { + return err + } + if m.Number <= 0 || m.Merge == "" { + return fmt.Errorf("delivery: the member %s of the group %q's end needs its pull request number and merge commit", m.Repository, e.Group) + } + k := fmt.Sprintf("%s#%d", m.Repository, m.Number) + if seen[k] { + return fmt.Errorf("delivery: the end of the group %q names its member %s twice", e.Group, k) + } + seen[k] = true + } + return nil +} + +// EncodeForgeGroupEnd is e as JSON at Version; it refuses an end with no group or no member, and a member without its +// pull request or merge commit. +func EncodeForgeGroupEnd(e ForgeGroupEnd) ([]byte, error) { + if err := e.check(); err != nil { + return nil, err + } + e.V = Version + return json.Marshal(e) +} + +// DecodeForgeGroupEnd reads a group's end; it refuses an unknown version (ErrUnknownVersion) and what +// EncodeForgeGroupEnd refuses. +func DecodeForgeGroupEnd(data []byte) (ForgeGroupEnd, error) { + var e ForgeGroupEnd + if err := decode(data, &e); err != nil { + return ForgeGroupEnd{}, err + } + if err := e.check(); err != nil { + return ForgeGroupEnd{}, err + } + return e, nil +} + +// SameForgeGroupEnd says whether two group ends are the same value. +func SameForgeGroupEnd(a, b ForgeGroupEnd) bool { return reflect.DeepEqual(a, b) } diff --git a/modules/mesh-delivery-forge-bridge/go.mod b/modules/mesh-delivery-forge-bridge/go.mod index 16f053bc..1a4b02ee 100644 --- a/modules/mesh-delivery-forge-bridge/go.mod +++ b/modules/mesh-delivery-forge-bridge/go.mod @@ -3,12 +3,13 @@ module mesh-delivery-forge-bridge go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) require ( + git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg v0.0.0-00010101000000-000000000000 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect github.com/google/go-tpm v0.9.8 // indirect github.com/klauspost/compress v1.20.0 // indirect @@ -20,3 +21,5 @@ require ( golang.org/x/sys v0.48.0 // indirect golang.org/x/time v0.15.0 // indirect ) + +replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg => ./forgemsg diff --git a/modules/mesh-delivery-forge-bridge/go.sum b/modules/mesh-delivery-forge-bridge/go.sum index 7679a5dc..1c3af991 100644 --- a/modules/mesh-delivery-forge-bridge/go.sum +++ b/modules/mesh-delivery-forge-bridge/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659 h1:tK7MzxVm/ECNg1Yblhh1ynVEthbZCg2mSPW0wQZC0b0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/acts.go b/modules/mesh-delivery-forge-bridge/internal/bridge/acts.go index f35ce988..32aae566 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/acts.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/acts.go @@ -5,6 +5,7 @@ import ( "slices" "strings" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -145,11 +146,11 @@ func MergeActs(value []byte) ([]Act, error) { // RefusedMessage is the refused message the bridge writes on FORGE when the forge refuses an act: the act, the pull // request and head it was on, and the forge's own words. func RefusedMessage(a Act, words string) (Message, error) { - event, err := delivery.ForgeEvent(a.Repository, delivery.ForgeRefused) + event, err := forgemsg.ForgeEvent(a.Repository, forgemsg.ForgeRefused) if err != nil { return Message{}, err } - body, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeRefused, Repository: a.Repository, Number: a.Number, + body, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeRefused, Repository: a.Repository, Number: a.Number, Head: a.Head, Act: a.Name, Words: words}) if err != nil { return Message{}, err diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/acts_test.go b/modules/mesh-delivery-forge-bridge/internal/bridge/acts_test.go index 4883012f..26fa4f41 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/acts_test.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/acts_test.go @@ -5,6 +5,7 @@ import ( "strings" "testing" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -111,18 +112,12 @@ func TestOwedActWithoutAPullRequestNeedsOneOnlyWhereTheToolDoes(t *testing.T) { } } -func TestOwedActOfADottedRepositoryIsUnescaped(t *testing.T) { - key, v := owedValue(t, "novox/my.repo", 3, "comment.1", delivery.Owed{Act: delivery.OwedComment, Text: "x"}) - a, err := OwedAct(key, v) - if err != nil { - t.Fatal(err) - } - if a.Call.Args["repo"] != "my.repo" || a.Repository != "novox/my.repo" { - t.Errorf("act = %+v", a) - } -} - func TestOwedActThatDisagreesWithItsKeyIsRefused(t *testing.T) { + // A dotted repository's key is unescaped, and its act names the repository whole. + dk, dv := owedValue(t, "novox/my.repo", 3, "comment.1", delivery.Owed{Act: delivery.OwedComment, Text: "x"}) + if a, err := OwedAct(dk, dv); err != nil || a.Call.Args["repo"] != "my.repo" || a.Repository != "novox/my.repo" { + t.Errorf("a dotted repository's act = %+v, %v", a, err) + } key, _ := owedValue(t, "novox/mesh-delivery", 12, "view", delivery.Owed{Act: delivery.OwedView, Text: "v"}) v, _ := delivery.EncodeOwed(delivery.Owed{Act: delivery.OwedComment, Text: "v"}) if _, err := OwedAct(key, v); err == nil { @@ -166,13 +161,11 @@ func TestMergeActsHandOnlyTheMemberHandedWithEveryApprovedHead(t *testing.T) { / if acts[0].Name != "merge" || acts[0].Number != 12 || acts[0].Head != headA { t.Errorf("act = %+v", acts[0]) } -} - -func TestMergeActsOfAMergeWithoutApprovedHeadsAreRefused(t *testing.T) { - v := mergeValue(t, delivery.Merge{Group: "g", Ask: "a", Pulls: []delivery.MergePull{ + // A merge with no approved heads is refused: the forge's tool would refuse it with nothing to say which. + v = mergeValue(t, delivery.Merge{Group: "g", Ask: "a", Pulls: []delivery.MergePull{ {Head: delivery.Head{Repository: "novox/mesh-delivery", Number: 12, Head: headA}, State: delivery.MergeMerging}}}) if _, err := MergeActs(v); err == nil { - t.Error("a merge with no approved heads was taken: the forge's tool would refuse it with nothing to say which") + t.Error("a merge with no approved heads was taken") } } @@ -185,13 +178,13 @@ func TestRefusedMessageNamesTheActAndTheForgesWords(t *testing.T) { // 1341/AC3, if m.Event != "forge.novox.mesh-delivery.refused" { t.Errorf("event = %q", m.Event) } - f, err := delivery.DecodeForge(m.Data) + f, err := forgemsg.DecodeForge(m.Data) if err != nil { t.Fatal(err) } - want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeRefused, Repository: "novox/mesh-delivery", Number: 12, Head: headA, + want := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeRefused, Repository: "novox/mesh-delivery", Number: 12, Head: headA, Act: "merge", Words: "novox/mesh-delivery#12 is not merged: its head moved"} - if !delivery.SameForge(f, want) { + if !forgemsg.SameForge(f, want) { t.Errorf("message = %+v", f) } } diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go index 774bc6b1..bfc80440 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go @@ -15,7 +15,7 @@ import ( "strings" "sync" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" ) // ErrUnreadable is wrapped by every error that says an event cannot be turned into a message; it is never @@ -110,11 +110,11 @@ func ConvertKept(subject string, data []byte, trees Trees, files Files, kept Kep var action string switch subject { case UpdatedSubject: - action = delivery.PullUpdated + action = forgemsg.PullUpdated case ClosedSubject: - action = delivery.PullClosed + action = forgemsg.PullClosed case MergedSubject: - action = delivery.PullClosed // unless it merged into the trunk, below + action = forgemsg.PullClosed // unless it merged into the trunk, below default: return Message{}, fmt.Errorf("%w: %s", ErrNotHandled, subject) } @@ -136,29 +136,29 @@ func ConvertKept(subject string, data []byte, trees Trees, files Files, kept Kep return merged(subject, p, trees, files, kept) } said := "" - if action == delivery.PullUpdated && p.Base != "" && p.Base != Trunk { + if action == forgemsg.PullUpdated && p.Base != "" && p.Base != Trunk { // The checker judges pull requests into main only, and source.main names main's head only (novox/hq ADR 0484 §11): a // pull request into another base is no candidate, as it is none at the start (OpenPulls) and at its merge. It is // written closed, so a candidate an older head made is taken out, and said (issue 1422). - action = delivery.PullClosed + action = forgemsg.PullClosed said = fmt.Sprintf("%s/%s#%d at %s is into %s, not %s: written closed, so it is no candidate; the checker judges pull requests into %s only (novox/hq ADR 0484 §11)", p.Owner, p.Repo, p.Number, p.HeadSHA[:12], p.Base, Trunk, Trunk) } - f := delivery.Forge{Kind: delivery.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number, + f := forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number, Head: p.HeadSHA, Action: action} // The directories are the forge's word only when it said it could read them (module_dirs_said). - if action == delivery.PullUpdated && p.ModuleDirsSaid { + if action == forgemsg.PullUpdated && p.ModuleDirsSaid { f.Dirs, f.DirsKnown = p.ModuleDirs, true var err error if f.Manifests, f.Locks, _, err = inputsRead(p.Owner, p.Repo, p.HeadSHA, p.ModuleDirs, files); err != nil { return Message{}, fmt.Errorf("%w: %s: the manifests of %s/%s at the head %s: %v", ErrRetry, subject, p.Owner, p.Repo, p.HeadSHA, err) } } - event, err := delivery.ForgeEvent(f.Repository, delivery.ForgePull) + event, err := forgemsg.ForgeEvent(f.Repository, forgemsg.ForgePull) if err != nil { return Message{}, unreadable(subject, "%v", err) } - body, err := delivery.EncodeForge(f) + body, err := forgemsg.EncodeForge(f) if err != nil { return Message{}, unreadable(subject, "%v", err) } @@ -313,11 +313,11 @@ func merged(subject string, p pull, trees Trees, files Files, kept Kept) (Messag if len(locks) == 0 { locks = nil } - event, err := delivery.ForgeEvent(repository, delivery.ForgeMerged) + event, err := forgemsg.ForgeEvent(repository, forgemsg.ForgeMerged) if err != nil { return Message{}, unreadable(subject, "%v", err) } - f := delivery.Forge{Kind: delivery.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number, + f := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number, Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs, Manifests: manifests, Locks: locks} if repository == CatalogRepository { // The catalogue's predefined list at the merge (novox/hq ADR 0565 §2): the indexer keeps it for mesh-config. @@ -331,7 +331,7 @@ func merged(subject string, p pull, trees Trees, files Files, kept Kept) (Messag f.Predefined = json.RawMessage(raw) } } - body, err := delivery.EncodeForge(f) + body, err := forgemsg.EncodeForge(f) if err != nil { return Message{}, unreadable(subject, "%v", err) } @@ -373,12 +373,12 @@ func OpenPulls(answer []byte) (msgs []Message, skipped []string, err error) { skipped = append(skipped, name+": it names no owner, repo, number, branch or 40-hex head") continue } - f := delivery.Forge{Kind: delivery.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number, - Head: p.HeadSHA, Action: delivery.PullUpdated} - event, err := delivery.ForgeEvent(f.Repository, delivery.ForgePull) + f := forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number, + Head: p.HeadSHA, Action: forgemsg.PullUpdated} + event, err := forgemsg.ForgeEvent(f.Repository, forgemsg.ForgePull) var body []byte if err == nil { - body, err = delivery.EncodeForge(f) + body, err = forgemsg.EncodeForge(f) } if err != nil { skipped = append(skipped, name+": "+err.Error()) @@ -392,12 +392,12 @@ func OpenPulls(answer []byte) (msgs []Message, skipped []string, err error) { // Track says what a message the bridge wrote changes in the head it keeps for each pull request (the bucket heads, // novox/hq ADR 0443 §5): the key `..`, and the head to keep under it, or "" to take the key away // (a closed pull request, a merged one). ok is false for a message that keeps no head. -func Track(f delivery.Forge) (key, head string, ok bool) { +func Track(f forgemsg.Forge) (key, head string, ok bool) { key = strings.ReplaceAll(f.Repository, "/", ".") + "." + strconv.Itoa(f.Number) switch { - case f.Kind == delivery.ForgePull && f.Action != delivery.PullClosed: + case f.Kind == forgemsg.ForgePull && f.Action != forgemsg.PullClosed: return key, f.Head, true - case f.Kind == delivery.ForgePull || f.Kind == delivery.ForgeMerged: + case f.Kind == forgemsg.ForgePull || f.Kind == forgemsg.ForgeMerged: return key, "", true } return "", "", false @@ -413,22 +413,22 @@ func OpenMessage(answer []byte) (Message, error) { if err := json.Unmarshal(answer, &a); err != nil || a.Pulls == nil { return Message{}, fmt.Errorf("the answer of gitea_open_pulls is not {pulls: [...]}: %v", err) } - open := []delivery.OpenPull{} + open := []forgemsg.OpenPull{} for _, p := range *a.Pulls { if p.Base != Trunk || p.Owner == "" || p.Repo == "" || p.Number <= 0 { continue } - open = append(open, delivery.OpenPull{Repository: p.Owner + "/" + p.Repo, Number: p.Number}) + open = append(open, forgemsg.OpenPull{Repository: p.Owner + "/" + p.Repo, Number: p.Number}) } - slices.SortFunc(open, func(a, b delivery.OpenPull) int { + slices.SortFunc(open, func(a, b forgemsg.OpenPull) int { if c := strings.Compare(a.Repository, b.Repository); c != 0 { return c } return a.Number - b.Number }) - body, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeOpen, Open: open}) + body, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeOpen, Open: open}) if err != nil { return Message{}, fmt.Errorf("the open pull requests cannot be said: %w", err) } - return Message{Event: delivery.OpenEvent(), Data: body}, nil + return Message{Event: forgemsg.OpenEvent(), Data: body}, nil } diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge_test.go b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge_test.go index 3bd1116f..68f5a0c1 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge_test.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge_test.go @@ -6,6 +6,7 @@ import ( "strings" "testing" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -54,13 +55,13 @@ func TestUpdatedBecomesAPullMessage(t *testing.T) { // 1235/AC2 (the conversion; if msg.Event != "forge.novox.mesh-delivery.pull" { t.Errorf("event = %q", msg.Event) } - f, err := delivery.DecodeForge(msg.Data) + f, err := forgemsg.DecodeForge(msg.Data) if err != nil { t.Fatal(err) } - want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, - Head: head, Action: delivery.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true, Manifests: map[string]string{"modules/a": "{}"}} - if !delivery.SameForge(f, want) { + want := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, + Head: head, Action: forgemsg.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true, Manifests: map[string]string{"modules/a": "{}"}} + if !forgemsg.SameForge(f, want) { t.Errorf("message = %+v, want %+v", f, want) } } @@ -70,7 +71,7 @@ func TestDirsNotSaidAreNotKnown(t *testing.T) { // 1235/AC3 if err != nil { t.Fatal(err) } - f, _ := delivery.DecodeForge(msg.Data) + f, _ := forgemsg.DecodeForge(msg.Data) if f.DirsKnown || len(f.Dirs) != 0 { t.Errorf("dirs = %v known = %v, want not known", f.Dirs, f.DirsKnown) } @@ -79,7 +80,7 @@ func TestDirsNotSaidAreNotKnown(t *testing.T) { // 1235/AC3 if err != nil { t.Fatal(err) } - f, _ = delivery.DecodeForge(msg.Data) + f, _ = forgemsg.DecodeForge(msg.Data) if !f.DirsKnown || len(f.Dirs) != 0 { t.Errorf("dirs = %v known = %v, want known and empty", f.Dirs, f.DirsKnown) } @@ -88,7 +89,7 @@ func TestDirsNotSaidAreNotKnown(t *testing.T) { // 1235/AC3 if err != nil { t.Fatal(err) } - if f, _ = delivery.DecodeForge(msg.Data); f.DirsKnown || len(f.Dirs) != 0 { + if f, _ = forgemsg.DecodeForge(msg.Data); f.DirsKnown || len(f.Dirs) != 0 { t.Errorf("dirs %v known %v without module_dirs_said", f.Dirs, f.DirsKnown) } } @@ -101,13 +102,13 @@ func TestClosedBecomesAClosedPullMessage(t *testing.T) { // 1235/AC4 if msg.Event != "forge.novox.mesh-delivery.pull" { t.Errorf("event = %q", msg.Event) } - f, err := delivery.DecodeForge(msg.Data) + f, err := forgemsg.DecodeForge(msg.Data) if err != nil { t.Fatal(err) } - want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, - Head: head, Action: delivery.PullClosed} - if !delivery.SameForge(f, want) { + want := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, + Head: head, Action: forgemsg.PullClosed} + if !forgemsg.SameForge(f, want) { t.Errorf("message = %+v, want %+v", f, want) } // Issue 1422, ADR 0484 §11: a head pushed to a pull request into another base is written closed, read nothing for, and said; @@ -116,12 +117,12 @@ func TestClosedBecomesAClosedPullMessage(t *testing.T) { // 1235/AC4 if err != nil { t.Fatal(err) } - if f, err = delivery.DecodeForge(msg.Data); err != nil || !delivery.SameForge(f, want) || !strings.Contains(msg.Said, "is into nox-mesh, not main") { + if f, err = forgemsg.DecodeForge(msg.Data); err != nil || !forgemsg.SameForge(f, want) || !strings.Contains(msg.Said, "is into nox-mesh, not main") { t.Errorf("an update into nox-mesh is %+v (%v), said %q; want it closed and said", f, err, msg.Said) } for _, base := range []any{"main", nil} { msg, err = Convert(updatedSubject, updated(map[string]any{"base": base}), nil, anyFiles) - if f, _ = delivery.DecodeForge(msg.Data); err != nil || f.Action != delivery.PullUpdated || msg.Said != "" { + if f, _ = forgemsg.DecodeForge(msg.Data); err != nil || f.Action != forgemsg.PullUpdated || msg.Said != "" { t.Errorf("an update with base %v is %+v (%v), said %q; want it updated", base, f, err, msg.Said) } } @@ -170,12 +171,18 @@ func mustDrop(body []byte, key string) []byte { return b } -func TestOtherSubjectsAreNotHandled(t *testing.T) { // 1235 +func TestOtherSubjectsAreNotHandled(t *testing.T) { // 1235, 1236 for _, s := range []string{"mesh.mod.gitea.event.something.else", "mesh.mod.other.event.pull.updated"} { if _, err := Convert(s, updated(nil), nil, anyFiles); !errors.Is(err, ErrNotHandled) { t.Errorf("%s: err = %v, want ErrNotHandled", s, err) } } + // 1236: a merged event without a full merge commit is unreadable, and the forge is not asked. + var calls []call + _, err := Convert(mergedSubject, mergedBody(map[string]any{"merge_commit_sha": "abc"}), treesOf(&calls, "r0", map[string]string{"a": "t"}, nil), anyFiles) + if !errors.Is(err, ErrUnreadable) || len(calls) != 0 { + t.Errorf("merged without a merge commit: err = %v calls = %v", err, calls) + } } const merge = "89abcdef0123456789abcdef0123456789abcdef" @@ -222,13 +229,13 @@ func TestMergeIntoMainBecomesAMergedMessage(t *testing.T) { // 1236/AC1 if msg.Event != "forge.novox.mesh-delivery.merged" { t.Errorf("event = %q", msg.Event) } - f, err := delivery.DecodeForge(msg.Data) + f, err := forgemsg.DecodeForge(msg.Data) if err != nil { t.Fatal(err) } - want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, + want := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Merge: merge, Root: "r0", Trees: answer, Manifests: map[string]string{"modules/a": "{}", "modules/b": "{}"}} - if !delivery.SameForge(f, want) { + if !forgemsg.SameForge(f, want) { t.Errorf("message = %+v, want %+v", f, want) } if len(calls) != 1 || calls[0] != (call{"novox", "mesh-delivery", merge}) { @@ -242,13 +249,13 @@ func TestMergeIntoAnotherBranchIsAClosedPull(t *testing.T) { // 1236/AC2 if err != nil { t.Fatal(err) } - f, err := delivery.DecodeForge(msg.Data) + f, err := forgemsg.DecodeForge(msg.Data) if err != nil { t.Fatal(err) } - want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, - Head: head, Action: delivery.PullClosed} - if msg.Event != "forge.novox.mesh-delivery.pull" || !delivery.SameForge(f, want) { + want := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, + Head: head, Action: forgemsg.PullClosed} + if msg.Event != "forge.novox.mesh-delivery.pull" || !forgemsg.SameForge(f, want) { t.Errorf("event %q message %+v, want a closed pull message", msg.Event, f) } if len(calls) != 0 { @@ -277,22 +284,14 @@ func TestNoModuleJSONAnywhereWritesAMergedMessageWithNoTrees(t *testing.T) { if err != nil { t.Fatal(err) } - f, err := delivery.DecodeForge(msg.Data) - want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, + f, err := forgemsg.DecodeForge(msg.Data) + want := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Merge: merge, Root: "r0"} - if err != nil || msg.Event != "forge.novox.mesh-delivery.merged" || !delivery.SameForge(f, want) || !strings.Contains(msg.Said, "no module.json") { + if err != nil || msg.Event != "forge.novox.mesh-delivery.merged" || !forgemsg.SameForge(f, want) || !strings.Contains(msg.Said, "no module.json") { t.Errorf("event %q message %+v said %q, %v; want a merged message with no trees", msg.Event, f, msg.Said, err) } } -func TestMergedWithoutAMergeCommitIsUnreadable(t *testing.T) { // 1236 - var calls []call - _, err := Convert(mergedSubject, mergedBody(map[string]any{"merge_commit_sha": "abc"}), treesOf(&calls, "r0", map[string]string{"a": "t"}, nil), anyFiles) - if !errors.Is(err, ErrUnreadable) || len(calls) != 0 { - t.Errorf("err = %v calls = %v", err, calls) - } -} - // openAnswer is gitea_open_pulls's answer as mesh-catalog modules/gitea/client.ts:369-381 builds each entry // (owner, repo, number, base, head, head_sha, html_url, required, statuses, checks, labels) inside {pulls}. func openAnswer(pulls ...map[string]any) []byte { @@ -311,12 +310,12 @@ func TestOpenPullsAreUpdatedPullMessagesWithUnknownDirs(t *testing.T) { // 1237/ if err != nil || len(skipped) != 0 || len(msgs) != 2 { t.Fatalf("msgs %d skipped %v err %v", len(msgs), skipped, err) } - for i, want := range []delivery.Forge{ - {V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: delivery.PullUpdated}, - {V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 300, Head: h2, Action: delivery.PullUpdated}, + for i, want := range []forgemsg.Forge{ + {V: delivery.Version, Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: forgemsg.PullUpdated}, + {V: delivery.Version, Kind: forgemsg.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 300, Head: h2, Action: forgemsg.PullUpdated}, } { - f, err := delivery.DecodeForge(msgs[i].Data) - if err != nil || !delivery.SameForge(f, want) { + f, err := forgemsg.DecodeForge(msgs[i].Data) + if err != nil || !forgemsg.SameForge(f, want) { t.Errorf("message %d = %+v err %v, want %+v", i, f, err, want) } if f.DirsKnown { @@ -328,14 +327,11 @@ func TestOpenPullsAreUpdatedPullMessagesWithUnknownDirs(t *testing.T) { // 1237/ } } -func TestOpenPullIntoAnotherBranchGetsNoMessage(t *testing.T) { // 1237/AC4 - msgs, skipped, err := OpenPulls(openAnswer(openEntry("mesh-delivery", 12, "feat/y", "feat/x", head))) - if err != nil || len(msgs) != 0 || len(skipped) != 0 { - t.Errorf("msgs %d skipped %v err %v, want none", len(msgs), skipped, err) +func TestOpenPullsUnreadableEntryIsNamedAndTheRestGo(t *testing.T) { // 1237, 1237/AC4 + // 1237/AC4: an open pull request into another branch gets no message. + if msgs, skipped, err := OpenPulls(openAnswer(openEntry("mesh-delivery", 12, "feat/y", "feat/x", head))); err != nil || len(msgs) != 0 || len(skipped) != 0 { + t.Errorf("into another branch: msgs %d skipped %v err %v, want none", len(msgs), skipped, err) } -} - -func TestOpenPullsUnreadableEntryIsNamedAndTheRestGo(t *testing.T) { // 1237 msgs, skipped, err := OpenPulls(openAnswer(openEntry("a", 1, "main", "b", "short"), openEntry("mesh-delivery", 12, "main", "feat/x", head))) if err != nil || len(msgs) != 1 || len(skipped) != 1 || !strings.Contains(skipped[0], "novox/a#1") { t.Errorf("msgs %d skipped %v err %v", len(msgs), skipped, err) @@ -355,15 +351,15 @@ func TestOpenMessageNamesEveryOpenPullIntoTheTrunk(t *testing.T) { // 1279/AC6 if err != nil || m.Event != "forge.open" { t.Fatalf("message %+v err %v", m, err) } - f, err := delivery.DecodeForge(m.Data) + f, err := forgemsg.DecodeForge(m.Data) // Every one into the trunk, whether its head reads or not (it is open); none into another branch; in a fixed order. - want := []delivery.OpenPull{{Repository: "novox/a", Number: 1}, {Repository: "novox/mesh-catalog", Number: 300}, {Repository: "novox/mesh-delivery", Number: 12}} - if err != nil || f.Kind != delivery.ForgeOpen || len(f.Open) != 3 || f.Open[0] != want[0] || f.Open[1] != want[1] || f.Open[2] != want[2] { + want := []forgemsg.OpenPull{{Repository: "novox/a", Number: 1}, {Repository: "novox/mesh-catalog", Number: 300}, {Repository: "novox/mesh-delivery", Number: 12}} + if err != nil || f.Kind != forgemsg.ForgeOpen || len(f.Open) != 3 || f.Open[0] != want[0] || f.Open[1] != want[1] || f.Open[2] != want[2] { t.Errorf("open message = %+v err %v, want %v", f, err, want) } // Nothing open is a message too: the indexer then drops every candidate. m, err = OpenMessage([]byte(`{"pulls":[]}`)) - if f, derr := delivery.DecodeForge(m.Data); err != nil || derr != nil || f.Kind != delivery.ForgeOpen || len(f.Open) != 0 { + if f, derr := forgemsg.DecodeForge(m.Data); err != nil || derr != nil || f.Kind != forgemsg.ForgeOpen || len(f.Open) != 0 { t.Errorf("empty answer: %+v %v %v", f, err, derr) } for _, bad := range []string{`{"pulls":null}`, `{}`, `x`} { @@ -374,24 +370,24 @@ func TestOpenMessageNamesEveryOpenPullIntoTheTrunk(t *testing.T) { // 1279/AC6 } func TestTrackSaysWhichHeadOfWhichPullRequestWasWritten(t *testing.T) { // 1279/AC6 - pullMsg := func(action string) delivery.Forge { - return delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: action} + pullMsg := func(action string) forgemsg.Forge { + return forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: action} } - for _, a := range []string{delivery.PullOpened, delivery.PullUpdated} { + for _, a := range []string{forgemsg.PullOpened, forgemsg.PullUpdated} { if key, h, ok := Track(pullMsg(a)); key != "novox.mesh-delivery.12" || h != head || !ok { t.Errorf("%s: Track = %q %q %v, want the head kept under a bucket key", a, key, h, ok) } } // A closed pull request and a merged one are no longer kept: their key goes. - if key, h, ok := Track(pullMsg(delivery.PullClosed)); key != "novox.mesh-delivery.12" || h != "" || !ok { + if key, h, ok := Track(pullMsg(forgemsg.PullClosed)); key != "novox.mesh-delivery.12" || h != "" || !ok { t.Errorf("closed: Track = %q %q %v", key, h, ok) } - merged := delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Merge: "m", Root: "r", Trees: map[string]string{"": "t"}} + merged := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Merge: "m", Root: "r", Trees: map[string]string{"": "t"}} if key, h, ok := Track(merged); key != "novox.mesh-delivery.12" || h != "" || !ok { t.Errorf("merged: Track = %q %q %v", key, h, ok) } // The open message and the rest hold no head. - if _, _, ok := Track(delivery.Forge{Kind: delivery.ForgeOpen}); ok { + if _, _, ok := Track(forgemsg.Forge{Kind: forgemsg.ForgeOpen}); ok { t.Error("an open message was tracked") } } diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/files_test.go b/modules/mesh-delivery-forge-bridge/internal/bridge/files_test.go index ce1f5458..602b3e7b 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/files_test.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/files_test.go @@ -13,7 +13,7 @@ import ( "testing" "time" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" ) // The bytes the forge answers in these tests are two real files of this repository, read from the working tree: the @@ -69,7 +69,7 @@ func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1 if err != nil { t.Fatal(err) } - f, err := delivery.DecodeForge(msg.Data) + f, err := forgemsg.DecodeForge(msg.Data) if err != nil { t.Fatal(err) } @@ -90,7 +90,7 @@ func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1 if err != nil { t.Fatal(err) } - if f, err = delivery.DecodeForge(msg.Data); err != nil { + if f, err = forgemsg.DecodeForge(msg.Data); err != nil { t.Fatal(err) } if len(f.RetiredPlacements) > 0 || f.RetiredRead || strings.Contains(string(msg.Data), `"placements`) { @@ -108,7 +108,7 @@ func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1 } forge.have[PredefinedFile] = []byte(`{"font-size":{"type":"integer","value":13}}`) msg, _ = Convert(mergedSubject, mergedBody(map[string]any{"repo": "mesh-catalog"}), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get) - if f, err = delivery.DecodeForge(msg.Data); err != nil || string(f.Predefined) != string(forge.have[PredefinedFile]) || f.PredefinedRead { + if f, err = forgemsg.DecodeForge(msg.Data); err != nil || string(f.Predefined) != string(forge.have[PredefinedFile]) || f.PredefinedRead { t.Errorf("predefined = %s (%v)", f.Predefined, err) } forge.calls = nil @@ -133,7 +133,7 @@ func TestPullWithKnownDirsCarriesTheSameAtTheHead(t *testing.T) { // 1448/AC2 if err != nil { t.Fatal(err) } - f, _ := delivery.DecodeForge(msg.Data) + f, _ := forgemsg.DecodeForge(msg.Data) if !reflect.DeepEqual(f.Manifests, map[string]string{"modules/a": string(manifest)}) || len(f.Locks) != 1 || len(f.Packages) != 0 { t.Errorf("manifests %v locks %v packages %v", f.Manifests, f.Locks, f.Packages) } @@ -146,7 +146,7 @@ func TestPullWithKnownDirsCarriesTheSameAtTheHead(t *testing.T) { // 1448/AC2 if err != nil { t.Fatal(err) } - f, _ = delivery.DecodeForge(msg.Data) + f, _ = forgemsg.DecodeForge(msg.Data) if f.Manifests != nil || f.Locks != nil || len(forge.calls) != 0 { t.Errorf("manifests %v locks %v calls %v", f.Manifests, f.Locks, forge.calls) } @@ -175,7 +175,7 @@ func TestAForgeErrorOrAMissingManifestIsRetryAndALockNotFoundIsLeftOut(t *testin if err != nil { t.Fatal(err) } - if f, _ := delivery.DecodeForge(msg.Data); f.Locks != nil { + if f, _ := forgemsg.DecodeForge(msg.Data); f.Locks != nil { t.Errorf("locks = %v, want none for lock files the forge does not have", f.Locks) } } diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/paths_test.go b/modules/mesh-delivery-forge-bridge/internal/bridge/paths_test.go index 57c06bc4..c4ddac59 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/paths_test.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/paths_test.go @@ -4,7 +4,7 @@ import ( "os" "testing" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" ) // ADR 0495 clause 1 with the lead's decision L8 of R4e: the forge bridge is no reader of a manifest's `paths`. A merge of a @@ -26,7 +26,7 @@ func TestAMergedMessageCarriesNoNamedPathsTheBridgeDerived(t *testing.T) { if err != nil { t.Fatal(err) } - f, err := delivery.DecodeForge(msg.Data) + f, err := forgemsg.DecodeForge(msg.Data) if err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index e8b23054..db89bb59 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -19,6 +19,8 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" @@ -102,15 +104,15 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 } }) h1, h2 := "0123456789abcdef0123456789abcdef01234567", "89abcdef0123456789abcdef0123456789abcdef" - for _, f := range []delivery.Forge{ - {Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: h1, Action: delivery.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true}, - {Kind: delivery.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 300, Head: h2, Action: delivery.PullUpdated, Dirs: []string{"modules/c"}, DirsKnown: true}, + for _, f := range []forgemsg.Forge{ + {Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: h1, Action: forgemsg.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true}, + {Kind: forgemsg.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 300, Head: h2, Action: forgemsg.PullUpdated, Dirs: []string{"modules/c"}, DirsKnown: true}, } { - data, err := delivery.EncodeForge(f) + data, err := forgemsg.EncodeForge(f) if err != nil { t.Fatal(err) } - event, _ := delivery.ForgeEvent(f.Repository, delivery.ForgePull) + event, _ := forgemsg.ForgeEvent(f.Repository, forgemsg.ForgePull) if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event."+event, data); err != nil { t.Fatal(err) } @@ -143,43 +145,43 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 t.Fatal(err) } const ix = delivery.IndexerModule - m1 := delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: h1, + m1 := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: h1, Merge: "1111111111111111111111111111111111111111", Root: "r0", Trees: map[string]string{"modules/" + ix: "tree-indexer-2", "modules/a": "tree-a-2"}, Manifests: map[string]string{"modules/" + ix: string(manifest), "modules/a": `{"module":"a","version":"1"}`}, Locks: map[string]map[string]string{"modules/" + ix: {"go.sum": "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"}}} - merged := func(f delivery.Forge) uint64 { + merged := func(f forgemsg.Forge) uint64 { t.Helper() - data, err := delivery.EncodeForge(f) + data, err := forgemsg.EncodeForge(f) if err != nil { t.Fatal(err) } - event, _ := delivery.ForgeEvent(f.Repository, delivery.ForgeMerged) + event, _ := forgemsg.ForgeEvent(f.Repository, forgemsg.ForgeMerged) ack, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event."+event, data) if err != nil { t.Fatal(err) } return ack.Sequence } - m1Key := delivery.PendingKey(merged(m1)) + m1Key := takein.PendingKey(merged(m1)) takeIns, err := js.Stream(ctx, "TAKE_IN") if err != nil { t.Fatal(err) } // said is the last take-in event under kind and key, within 20 s. - said := func(kind, key string) (delivery.TakeIn, bool) { - event, _ := delivery.TakeInEvent(kind, key) + said := func(kind, key string) (takein.TakeIn, bool) { + event, _ := takein.TakeInEvent(kind, key) for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { if msg, err := takeIns.GetLastMsgForSubject(ctx, process.ModuleStreamSubject(ix, event)); err == nil { - if ti, err := delivery.DecodeTakeIn(msg.Data); err == nil { + if ti, err := takein.DecodeTakeIn(msg.Data); err == nil { return ti, true } } } - return delivery.TakeIn{}, false + return takein.TakeIn{}, false } - held, ok := said(delivery.TakeInPending, m1Key) - if !ok || held.What != delivery.TakeInRevision || held.WaitsFor == "" || held.WaitsFor == first || len(held.Merges) != 1 { + held, ok := said(takein.TakeInPending, m1Key) + if !ok || held.What != takein.TakeInRevision || held.WaitsFor == "" || held.WaitsFor == first || len(held.Merges) != 1 { t.Fatalf("take-in.pending.%s = %+v (said %v) within 20 s, want the revision waiting for another build than %s", m1Key, held, ok, first) } named := held.WaitsFor @@ -226,14 +228,14 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 // A group's first member arrives meanwhile; its end is still to come (the knowledge base's remark: it stays across the restart). members := []delivery.Head{{Repository: "novox/mesh-catalog", Number: 40, Head: "4040404040404040404040404040404040404040"}, {Repository: "novox/mesh-controller", Number: 41, Head: "4141414141414141414141414141414141414141"}} - g1 := delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/g", Number: 40, Head: members[0].Head, + g1 := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/g", Number: 40, Head: members[0].Head, Merge: "4000000000000000000000000000000000000000", Root: "r0", Trees: map[string]string{"modules/c": "tree-c-2"}, Manifests: map[string]string{"modules/c": `{"module":"c","version":"1"}`}, Group: "feat/g", Members: members} - g2 := delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-controller", Branch: "feat/g", Number: 41, Head: members[1].Head, + g2 := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-controller", Branch: "feat/g", Number: 41, Head: members[1].Head, Merge: "4100000000000000000000000000000000000000", Root: "r0", Trees: map[string]string{"modules/k": "tree-k-1"}, Manifests: map[string]string{"modules/k": `{"module":"k","version":"1"}`}, Group: "feat/g", Members: members} - g1Key, groupKey := delivery.PendingKey(merged(g1)), delivery.GroupTakeInKey("feat/g") - if gathered, ok := said(delivery.TakeInPending, groupKey); !ok || gathered.What != delivery.TakeInGroup || len(gathered.Merges) != 1 { + g1Key, groupKey := takein.PendingKey(merged(g1)), takein.GroupTakeInKey("feat/g") + if gathered, ok := said(takein.TakeInPending, groupKey); !ok || gathered.What != takein.TakeInGroup || len(gathered.Merges) != 1 { t.Fatalf("take-in.pending.%s = %+v (said %v) within 20 s, want the group with its first member", groupKey, gathered, ok) } // indexer_pending, the tool over the same state, names both waits. @@ -241,29 +243,29 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 if err != nil { t.Fatal(err) } - waits := func() []delivery.PendingWait { + waits := func() []takein.PendingWait { t.Helper() answer, err := index.PendingTools(kvStore{pendingKV}, nil, time.Now)[0].Run(map[string]any{}) if err != nil { t.Fatal(err) } - return answer.(delivery.PendingAnswer).Waits + return answer.(takein.PendingAnswer).Waits } - var w []delivery.PendingWait + var w []takein.PendingWait for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { if w = waits(); len(w) == 2 && w[0].WaitsFor == named && w[1].Said != "" { break } } if len(w) != 2 || w[0].Key != m1Key || w[0].WaitsFor != named || !slices.Contains(w[0].Holds, "a") || !slices.Contains(w[0].Holds, ix) || - w[1].Key != groupKey || w[1].What != delivery.TakeInGroup || len(w[1].Merges) != 1 || !strings.HasSuffix(w[1].Said, "pending."+groupKey) { + w[1].Key != groupKey || w[1].What != takein.TakeInGroup || len(w[1].Merges) != 1 || !strings.HasSuffix(w[1].Said, "pending."+groupKey) { t.Fatalf("indexer_pending = %+v, want the revision %s waiting for %s holding a and the indexer, then the group gathered", w, m1Key, named) } // The named indexer starts (the node-engine now runs that build): it takes the revision in, and leaves the group. running.Store(&named) restart() - if taken, ok := said(delivery.TakeInTaken, m1Key); !ok || len(taken.Merges) != 1 || taken.Merges[0].Merge != m1.Merge { + if taken, ok := said(takein.TakeInTaken, m1Key); !ok || len(taken.Merges) != 1 || taken.Merges[0].Merge != m1.Merge { t.Fatalf("take-in.taken.%s = %+v (said %v) within 20 s of the start of %s", m1Key, taken, ok, named) } for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { @@ -286,15 +288,15 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 // I8b: the group's second member and its end: one take-in of both, said under the group's key. merged(g2) - end, err := delivery.EncodeForgeGroupEnd(delivery.ForgeGroupEnd{Group: "feat/g", Members: []delivery.GroupMerge{ + end, err := forgemsg.EncodeForgeGroupEnd(forgemsg.ForgeGroupEnd{Group: "feat/g", Members: []forgemsg.GroupMerge{ {Repository: g1.Repository, Number: g1.Number, Merge: g1.Merge}, {Repository: g2.Repository, Number: g2.Number, Merge: g2.Merge}}}) if err != nil { t.Fatal(err) } - if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event."+delivery.ForgeGroupEndEvent, end); err != nil { + if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event."+forgemsg.ForgeGroupEndEvent, end); err != nil { t.Fatal(err) } - if taken, ok := said(delivery.TakeInTaken, groupKey); !ok || taken.What != delivery.TakeInGroup || len(taken.Merges) != 2 { + if taken, ok := said(takein.TakeInTaken, groupKey); !ok || taken.What != takein.TakeInGroup || len(taken.Merges) != 2 { t.Fatalf("take-in.taken.%s = %+v (said %v) within 20 s of the group's end, want both members", groupKey, taken, ok) } var hasC, hasK bool @@ -307,7 +309,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 t.Errorf("after the group's end: modules.c %v, modules.k %v, indexer_pending %+v: want both taken in and nothing waiting", hasC, hasK, w) } // One take-in: neither member was said as a revision of its own. - single, _ := delivery.TakeInEvent(delivery.TakeInPending, g1Key) + single, _ := takein.TakeInEvent(takein.TakeInPending, g1Key) if _, err := takeIns.GetLastMsgForSubject(ctx, process.ModuleStreamSubject(ix, single)); err == nil { t.Errorf("the group's first member %s was said pending as a revision of its own", g1Key) } @@ -443,7 +445,7 @@ func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9; I1, I3: the m builds.end(t, "novox/mesh-controller", "build-agent", "modules/build-agent") }) const merge = "89abcdef0123456789abcdef0123456789abcdef" - data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, + data, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: "0123456789abcdef0123456789abcdef01234567", Merge: merge, Root: "r0", Trees: map[string]string{"modules/a": "newtree"}, Manifests: map[string]string{"modules/a": `{"module":"a","version":"1"}`}}) if err != nil { @@ -497,7 +499,7 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4, 15 t.Fatal(err) } }) - forge := delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301, + forge := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301, Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0", Trees: map[string]string{"modules/avahi": "newtree"}, Manifests: map[string]string{"modules/avahi": string(manifest)}, @@ -505,7 +507,7 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4, 15 for name, raw := range judged { forge.Manifests["modules/"+name], forge.Trees["modules/"+name] = raw, "tree-"+name } - data, err := delivery.EncodeForge(forge) + data, err := forgemsg.EncodeForge(forge) if err != nil { t.Fatal(err) } @@ -599,7 +601,7 @@ func TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain(t *tes js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds buildLog) { builds.end(t, "novox/mesh-catalog", "avahi", "modules/avahi") }) - data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301, + data, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301, Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0", Trees: map[string]string{"modules/avahi": "newtree"}, Manifests: map[string]string{"modules/avahi": string(manifest)}, @@ -686,7 +688,7 @@ func TestIndexerGivesACandidateItsFingerprintWhenTheCheckerCreatesThePreviewsBuc t.Fatal(err) } }) - pull, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301, Head: head, Action: delivery.PullUpdated, + pull, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301, Head: head, Action: forgemsg.PullUpdated, Dirs: []string{"modules/avahi"}, DirsKnown: true, Manifests: map[string]string{"modules/avahi": string(manifest)}}) if err != nil { t.Fatal(err) @@ -856,7 +858,7 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) { place(ctx, t, "n1", "mesh-build-go") place(ctx, t, "n2") }) - data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 302, + data, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 302, Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0", Trees: map[string]string{"modules/avahi": "newtree"}, Manifests: map[string]string{"modules/avahi": string(manifest)}, diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index ccdbe7ac..ce9d794b 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -25,6 +25,7 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -157,14 +158,14 @@ type delivered interface { // handleGroupEnd takes a group's end in (index.EndGroup): a body that does not read is terminated and said; an end // whose members are not all held yet, or while the build this program runs is not known, is left for redelivery. func handleGroupEnd(ctx context.Context, m delivered, ix *index.Indexer, h *index.Heard, logger *log.Logger) { - end, err := delivery.DecodeForgeGroupEnd(m.Data()) + end, err := forgemsg.DecodeForgeGroupEnd(m.Data()) if err != nil { logger.Printf("%s: not a group's end, dropped: %v", m.Subject(), err) h.Outcome, h.Words = index.HeardDropped, err.Error() _ = m.Term() return } - h.Kind, h.Branch = delivery.ForgeGroupEndEvent, end.Group + h.Kind, h.Branch = forgemsg.ForgeGroupEndEvent, end.Group applied, words, err := ix.EndGroup(ctx, end) h.Written, h.Late = applied.Written, applied.Late if err != nil { @@ -197,11 +198,11 @@ func handle(ctx context.Context, m delivered, ix *index.Indexer, heard index.Sto } }() // A delivery group's end (novox/hq ADR 0633 §4) has a body of its own: told apart by its name, before the decode. - if strings.HasSuffix(m.Subject(), "."+delivery.ForgeGroupEndEvent) { + if strings.HasSuffix(m.Subject(), "."+forgemsg.ForgeGroupEndEvent) { handleGroupEnd(ctx, m, ix, &h, logger) return } - f, err := delivery.DecodeForge(m.Data()) + f, err := forgemsg.DecodeForge(m.Data()) if err != nil { logger.Printf("%s: not a forge message, dropped: %v", m.Subject(), err) h.Outcome, h.Words = index.HeardDropped, err.Error() @@ -213,7 +214,7 @@ func handle(ctx context.Context, m delivered, ix *index.Indexer, heard index.Sto h.Commit = f.Merge } var applied index.Applied - if f.Kind == delivery.ForgeMerged && h.Sequence > 0 { + if f.Kind == forgemsg.ForgeMerged && h.Sequence > 0 { // A merge is taken in only by the indexer its revision names (ADR 0633 §1): kept in pending until then. applied, h.Words, err = ix.Arrive(ctx, h.Sequence, f) } else { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go index f42d7972..c4c3672c 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go @@ -11,6 +11,7 @@ import ( "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/delivery" "mesh-delivery-indexer/internal/index" @@ -73,8 +74,8 @@ const head = "0123456789abcdef0123456789abcdef01234567" func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part) st := &memStore{vals: map[string][]byte{}} ix := &index.Indexer{Store: st, Builds: index.NewBuilds()} - data, _ := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, - Action: delivery.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true}) + data, _ := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, + Action: forgemsg.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true}) m := &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull", data: data} heard := &memStore{vals: map[string][]byte{}} handle(context.Background(), m, ix, heard, log.New(&bytes.Buffer{}, "", 0)) @@ -84,7 +85,7 @@ func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part) } // Issue 1126 note 76: what the indexer heard of FORGE is readable (indexer_heard): subject, sequence, deliveries, time, // repository, commit and what it did. - if h := heardOf(t, heard); h.Subject != m.subject || h.Sequence != 41 || h.Delivered != 2 || h.Published.Hour() != 13 || h.Kind != delivery.ForgePull || + if h := heardOf(t, heard); h.Subject != m.subject || h.Sequence != 41 || h.Delivered != 2 || h.Published.Hour() != 13 || h.Kind != forgemsg.ForgePull || h.Repository != "novox/mesh-delivery" || h.Number != 12 || h.Commit != head || h.Outcome != index.HeardApplied || h.Handled.IsZero() { t.Errorf("heard = %+v", h) } @@ -111,12 +112,12 @@ func TestAMessageThatIsNoForgeMessageIsTerminatedAndLogged(t *testing.T) { // 12 }{{`{"v":1,"group":"feat/x","members":[{"repository":"novox/mesh-sdk","number":3,"merge":"` + head + `"}]}`, 0, ""}, {`{"v":1,"group":"feat/x"}`, 1, "not a group's end"}} { logs.Reset() - m := &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event." + delivery.ForgeGroupEndEvent, data: []byte(c.data)} + m := &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event." + forgemsg.ForgeGroupEndEvent, data: []byte(c.data)} handle(context.Background(), m, &index.Indexer{Store: st, Builds: index.NewBuilds()}, heard, log.New(&logs, "", 0)) if m.termd != c.termd || m.acked+m.termd != 1 || !strings.Contains(logs.String(), c.logged) { t.Errorf("group end %s: terminated %d acked %d log %q", c.data, m.termd, m.acked, logs.String()) } - if h := heardOf(t, heard); h.Kind != delivery.ForgeGroupEndEvent && c.termd == 0 { + if h := heardOf(t, heard); h.Kind != forgemsg.ForgeGroupEndEvent && c.termd == 0 { t.Errorf("heard of a group's end = %+v", h) } } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go index f10cac87..8d60e4b9 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go @@ -9,6 +9,7 @@ import ( "path" "strings" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/delivery" "mesh-delivery-indexer/internal/fingerprint" @@ -18,7 +19,7 @@ import ( // commitTreesTool is the forge's tool for a commit's trees (mesh-catalog modules/gitea): with `paths`, by module directory // (the root spelled ""), the paths relative to it, it answers {root, trees, paths} with paths by directory then path, each // path's git object id; a path not at the commit fails the call naming it (the interface agreed in R4e; mesh-sdk -// delivery.Forge.Paths; novox/hq ADR 0499 clause 3). +// forgemsg.Forge.Paths; novox/hq ADR 0499 clause 3). const commitTreesTool = "gitea.gitea_commit_trees" // toolCaller calls a module's tool, `.`, as the indexer's own account (mesh-sdk process.Conn.Call). @@ -258,8 +259,8 @@ const ( ) // openVia asks the forge which pull requests into the trunk are open now (gitea_open_pulls), every repository's. -func openVia(call toolCaller) func(ctx context.Context) ([]delivery.OpenPull, error) { - return func(context.Context) ([]delivery.OpenPull, error) { +func openVia(call toolCaller) func(ctx context.Context) ([]forgemsg.OpenPull, error) { + return func(context.Context) ([]forgemsg.OpenPull, error) { raw, err := call(openPullsTool, map[string]any{}) if err != nil { return nil, err @@ -275,10 +276,10 @@ func openVia(call toolCaller) func(ctx context.Context) ([]delivery.OpenPull, er if err := json.Unmarshal(raw, &o); err != nil || o.Pulls == nil { return nil, fmt.Errorf("the answer of %s is not {pulls}: %v", openPullsTool, err) } - out := []delivery.OpenPull{} + out := []forgemsg.OpenPull{} for _, p := range *o.Pulls { if p.Base == trunkBranch && p.Number > 0 { - out = append(out, delivery.OpenPull{Repository: p.Owner + "/" + p.Repo, Number: p.Number}) + out = append(out, forgemsg.OpenPull{Repository: p.Owner + "/" + p.Repo, Number: p.Number}) } } return out, nil diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go index 069be8be..45012efa 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go @@ -13,6 +13,7 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -71,10 +72,10 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T) } serveForge(t, js.Conn(), listing, goSumContent) }) - merged := func(repo string, number int, merge string, f delivery.Forge) { + merged := func(repo string, number int, merge string, f forgemsg.Forge) { t.Helper() - f.Kind, f.Repository, f.Branch, f.Number, f.Head, f.Merge = delivery.ForgeMerged, repo, "feat/x", number, "0123456789abcdef0123456789abcdef01234567", merge - data, err := delivery.EncodeForge(f) + f.Kind, f.Repository, f.Branch, f.Number, f.Head, f.Merge = forgemsg.ForgeMerged, repo, "feat/x", number, "0123456789abcdef0123456789abcdef01234567", merge + data, err := forgemsg.EncodeForge(f) if err != nil { t.Fatal(err) } @@ -101,10 +102,10 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T) "tsconfig.json": "5555555555555555555555555555555555555555"}} } // mesh-sdk at m1, then the catalogue's two modules: their go.sum names mesh-sdk, read by the indexer itself. - merged("novox/mesh-sdk", 70, m1, delivery.Forge{Root: sdkRoot1, Trees: map[string]string{"": sdkRoot1}, + merged("novox/mesh-sdk", 70, m1, forgemsg.Forge{Root: sdkRoot1, Trees: map[string]string{"": sdkRoot1}, Manifests: map[string]string{"": string(sdkManifest)}, Paths: sdkObjects("6666666666666666666666666666666666666666")}) module("mesh-sdk", func(m delivery.Module) bool { return m.Commit == m1 }) - merged("novox/mesh-catalog", 301, catalogMerge, delivery.Forge{Root: "r0", + merged("novox/mesh-catalog", 301, catalogMerge, forgemsg.Forge{Root: "r0", Trees: map[string]string{"modules/importer": "t1", "modules/plain": "t2"}, Manifests: map[string]string{"modules/importer": named("importer"), "modules/plain": named("plain")}, Locks: map[string]map[string]string{"modules/importer": {"go.sum": goSumHash}, "modules/plain": {"go.sum": goSumHash}}}) @@ -140,11 +141,11 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T) // over its named tree, is not written (issue 1758, ADR 0499 clause 6; the operator, 2026-10-25: a module's value moves // only with the module). const goOnly = "4444444444444444444444444444444444444444" - merged("novox/mesh-sdk", 72, goOnly, delivery.Forge{Root: "cccccccccccccccccccccccccccccccccccccccc", + merged("novox/mesh-sdk", 72, goOnly, forgemsg.Forge{Root: "cccccccccccccccccccccccccccccccccccccccc", Trees: map[string]string{"": "cccccccccccccccccccccccccccccccccccccccc"}, Manifests: map[string]string{"": string(sdkManifest)}, Paths: sdkObjects("6666666666666666666666666666666666666666")}) // mesh-sdk merges a change of buildstep alone (its version moves: src changed too). - merged("novox/mesh-sdk", 71, m2, delivery.Forge{Root: sdk2, Trees: map[string]string{"": sdk2}, + merged("novox/mesh-sdk", 71, m2, forgemsg.Forge{Root: sdk2, Trees: map[string]string{"": sdk2}, Manifests: map[string]string{"": string(sdkManifest)}, Paths: sdkObjects("7777777777777777777777777777777777777777")}) sdkAt2 := module("mesh-sdk", func(m delivery.Module) bool { return m.Commit == m2 }) if sdkAt2.Inputs2 == nil || !strings.HasPrefix(sdkAt2.Inputs2.Tree, delivery.NamedTreePrefix) { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/verdicts.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/verdicts.go index 495cb684..3526751a 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/verdicts.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/verdicts.go @@ -6,6 +6,7 @@ import ( "fmt" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" "github.com/nats-io/nats.go" @@ -65,13 +66,13 @@ func sayAccepted(js jetstream.JetStream, cred process.Account, logf func(string, // sayTakeIn publishes a take-in event on the indexer's own stream take-in (novox/hq ADR 0633 §5): // `mesh.stream..event.take-in..`. A publish that fails is the caller's to say. -func sayTakeIn(js jetstream.JetStream, cred process.Account) func(context.Context, delivery.TakeIn) error { - return func(ctx context.Context, t delivery.TakeIn) error { - event, err := delivery.TakeInEvent(t.Kind, t.Key) +func sayTakeIn(js jetstream.JetStream, cred process.Account) func(context.Context, takein.TakeIn) error { + return func(ctx context.Context, t takein.TakeIn) error { + event, err := takein.TakeInEvent(t.Kind, t.Key) if err != nil { return err } - data, err := delivery.EncodeTakeIn(t) + data, err := takein.EncodeTakeIn(t) if err != nil { return err } diff --git a/modules/mesh-delivery-indexer/go.mod b/modules/mesh-delivery-indexer/go.mod index 4c66d058..0f4eee43 100644 --- a/modules/mesh-delivery-indexer/go.mod +++ b/modules/mesh-delivery-indexer/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-indexer go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) @@ -24,6 +24,14 @@ require ( replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun => ../mesh-delivery-builder/buildrun -require git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000 +require ( + git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000 +) replace git.novox.be/novox/mesh-delivery/sdk => ../../sdk + +replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein => ./takein + +replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg => ../mesh-delivery-forge-bridge/forgemsg diff --git a/modules/mesh-delivery-indexer/go.sum b/modules/mesh-delivery-indexer/go.sum index 7679a5dc..1c3af991 100644 --- a/modules/mesh-delivery-indexer/go.sum +++ b/modules/mesh-delivery-indexer/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659 h1:tK7MzxVm/ECNg1Yblhh1ynVEthbZCg2mSPW0wQZC0b0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go b/modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go index a43f4e99..0a25604d 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go +++ b/modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go @@ -7,12 +7,13 @@ import ( "strings" "testing" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/delivery" ) -func manifestPull(t *testing.T, number int, head, manifest string) delivery.Forge { - f := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: catalogueRepo, Branch: "feat/x", Number: number, Head: head, - Action: delivery.PullUpdated, Dirs: []string{"modules/avahi"}, DirsKnown: true} +func manifestPull(t *testing.T, number int, head, manifest string) forgemsg.Forge { + f := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: catalogueRepo, Branch: "feat/x", Number: number, Head: head, + Action: forgemsg.PullUpdated, Dirs: []string{"modules/avahi"}, DirsKnown: true} if manifest != "" { f.Manifests = map[string]string{"modules/avahi": manifest} } diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 75b6ea90..31d23828 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -11,6 +11,8 @@ import ( "sync/atomic" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -98,7 +100,7 @@ type Indexer struct { // ReadOpen asks the forge which pull requests into the trunk are open now (gitea_open_pulls). A catalogue pull message // that finds the candidates of another pull request asks it and deletes those of a pull request no longer open, so a // closed message the indexer missed heals at the next one (issue 1126); nil asks none. - ReadOpen func(ctx context.Context) ([]delivery.OpenPull, error) + ReadOpen func(ctx context.Context) ([]forgemsg.OpenPull, error) mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach source sourceNow @@ -106,7 +108,7 @@ type Indexer struct { toolchains map[string]map[string]delivery.Toolchain // language, then node (toolchains.go) tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go) unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread - moving map[string]delivery.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go) + moving map[string]forgemsg.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go) retrying atomic.Bool // a RetryUnread runs in the background (kickRetry) retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on @@ -125,7 +127,7 @@ type Indexer struct { // takes every merge in at once, as before the take-in step. Pending Store // SayTakeIn publishes a take-in event on the indexer's own stream take-in (ADR 0633 §5); nil says none. - SayTakeIn func(ctx context.Context, t delivery.TakeIn) error + SayTakeIn func(ctx context.Context, t takein.TakeIn) error pendingMu sync.Mutex // one take-in step at a time: a merge, a group's end, the start and a hand act (pending.go) } @@ -139,23 +141,23 @@ func (ix *Indexer) logf(format string, args ...any) { // pull request, a closed one or a merged message takes it out; a group left with none is deleted. A change is written // by compare-and-set on the revision read, and read again on a conflict. A lone pull request is a group of one. A // merged message also writes the fingerprint of each module of its repository (modules.go). -func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error { +func (ix *Indexer) Apply(ctx context.Context, f forgemsg.Forge) error { _, err := ix.ApplyMessage(ctx, f) return err } // ApplyMessage is Apply, and answers which modules a merged message wrote and kept (for the indexer's heard state). Each // applied merged message is said in one line: its repository, commit and pull request, and the modules written. -func (ix *Indexer) ApplyMessage(ctx context.Context, f delivery.Forge) (Applied, error) { +func (ix *Indexer) ApplyMessage(ctx context.Context, f forgemsg.Forge) (Applied, error) { ix.kickJudge(ctx) var applied Applied - if f.Kind == delivery.ForgeOpen { + if f.Kind == forgemsg.ForgeOpen { return applied, ix.applyOpen(ctx, f) } - if f.Kind != delivery.ForgePull && f.Kind != delivery.ForgeMerged { + if f.Kind != forgemsg.ForgePull && f.Kind != forgemsg.ForgeMerged { return applied, nil } - if f.Kind == delivery.ForgeMerged && ix.Modules != nil { + if f.Kind == forgemsg.ForgeMerged && ix.Modules != nil { w, err := ix.applyMerged(ctx, f) applied = Applied{Written: w.written, Late: w.late} if err != nil { @@ -184,7 +186,7 @@ func (ix *Indexer) ApplyMessage(ctx context.Context, f delivery.Forge) (Applied, } var digests map[string]string var held map[string]delivery.Module - if f.Kind == delivery.ForgePull { + if f.Kind == forgemsg.ForgePull { var err error if digests, err = ix.keepManifests(ctx, f); err != nil { return applied, err @@ -251,8 +253,8 @@ func (ix *Indexer) ApplyMessage(ctx context.Context, f delivery.Forge) (Applied, // applyOpen takes an open message into the candidates (novox/hq ADR 0443 §5): every pull request it does not name is // taken out of its group, and a group left with none is deleted. What the message names is left as it is. A pull request // opened after the bridge read the forge is heard after this message and added again by its own. -func (ix *Indexer) applyOpen(ctx context.Context, f delivery.Forge) error { - named := make(map[delivery.OpenPull]bool, len(f.Open)) +func (ix *Indexer) applyOpen(ctx context.Context, f forgemsg.Forge) error { + named := make(map[forgemsg.OpenPull]bool, len(f.Open)) for _, p := range f.Open { named[p] = true } @@ -270,7 +272,7 @@ func (ix *Indexer) applyOpen(ctx context.Context, f delivery.Forge) error { return nil, false, fmt.Errorf("candidates.%s holds what is not a candidate: %w", key, err) } kept := slices.DeleteFunc(slices.Clone(c.Pulls), func(p delivery.CandidatePull) bool { - return !named[delivery.OpenPull{Repository: p.Repository, Number: p.Number}] + return !named[forgemsg.OpenPull{Repository: p.Repository, Number: p.Number}] }) switch { case len(kept) == len(c.Pulls): @@ -337,14 +339,14 @@ func write(ctx context.Context, st Store, key string, compute func(value []byte, // its manifest and why it has no fingerprint of the new form (NoneNoPreview), and the candidate its FPSet (issue 1451); a // pull request whose head is held again without its directories keeps the Fps it had. digests are the manifests' digests by // directory; held are the modules the modules bucket holds, which name the module of a changed directory (moduleOfDir). -func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge, digests map[string]string, held map[string]delivery.Module) *delivery.Candidate { +func (ix *Indexer) next(cur *delivery.Candidate, f forgemsg.Forge, digests map[string]string, held map[string]delivery.Module) *delivery.Candidate { var pulls []delivery.CandidatePull var base uint64 if cur != nil { pulls, base = slices.Clone(cur.Pulls), cur.Base } at := slices.IndexFunc(pulls, func(p delivery.CandidatePull) bool { return p.Repository == f.Repository && p.Number == f.Number }) - if f.Kind == delivery.ForgeMerged || f.Action == delivery.PullClosed { + if f.Kind == forgemsg.ForgeMerged || f.Action == forgemsg.PullClosed { if at < 0 { return cur } diff --git a/modules/mesh-delivery-indexer/internal/index/control_records.go b/modules/mesh-delivery-indexer/internal/index/control_records.go index 0feb4ec3..19941487 100644 --- a/modules/mesh-delivery-indexer/internal/index/control_records.go +++ b/modules/mesh-delivery-indexer/internal/index/control_records.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" ) @@ -27,7 +28,7 @@ var ( // writePredefined keeps the predefined list a merged message carries, unless the key names a later merge: a merge older // than the one it names never replaces it (ADR 0348 §7). The same list at the same merge writes nothing (ADR 0289 §1). -func (ix *Indexer) writePredefined(ctx context.Context, f delivery.Forge) error { +func (ix *Indexer) writePredefined(ctx context.Context, f forgemsg.Forge) error { if ix.Predefined == nil || len(f.Predefined) == 0 { return nil } @@ -203,8 +204,8 @@ func (ix *Indexer) dropConfigCandidate(ctx context.Context, repository string, n // configCandidateOfPull is a pull message's part in config-candidates: a head whose manifests the message carries is written, // a closed one deleted. A head held again without its directories leaves the key as it is. -func (ix *Indexer) configCandidateOfPull(ctx context.Context, f delivery.Forge, held map[string]delivery.Module) error { - if f.Action == delivery.PullClosed { +func (ix *Indexer) configCandidateOfPull(ctx context.Context, f forgemsg.Forge, held map[string]delivery.Module) error { + if f.Action == forgemsg.PullClosed { return ix.dropConfigCandidate(ctx, f.Repository, f.Number) } if !f.DirsKnown { @@ -220,7 +221,7 @@ func (ix *Indexer) configCandidateOfPull(ctx context.Context, f delivery.Forge, // configCandidatesAfterMerge writes again every open pull request's key that names a module the merge changed, since what // it adds is judged against the trunk (ADR 0565 §2), from the head's manifests its candidate keeps; and deletes the merged // pull request's. -func (ix *Indexer) configCandidatesAfterMerge(ctx context.Context, f delivery.Forge, changed []string) error { +func (ix *Indexer) configCandidatesAfterMerge(ctx context.Context, f forgemsg.Forge, changed []string) error { if ix.ConfigCandidates == nil { return nil } diff --git a/modules/mesh-delivery-indexer/internal/index/control_records_test.go b/modules/mesh-delivery-indexer/internal/index/control_records_test.go index 119a7fe8..358d802b 100644 --- a/modules/mesh-delivery-indexer/internal/index/control_records_test.go +++ b/modules/mesh-delivery-indexer/internal/index/control_records_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" ) @@ -77,7 +78,7 @@ func TestTheControlRecords(t *testing.T) { t.Errorf("a head declaring nothing new = %+v, want an empty set", c) } closed := pull(catalogueRepo, 7, "feat/y", strings.Repeat("d", 40)) - closed.Action = delivery.PullClosed + closed.Action = forgemsg.PullClosed if err := ix.Apply(ctx, closed); err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-indexer/internal/index/index_test.go b/modules/mesh-delivery-indexer/internal/index/index_test.go index b584aa67..a29adba7 100644 --- a/modules/mesh-delivery-indexer/internal/index/index_test.go +++ b/modules/mesh-delivery-indexer/internal/index/index_test.go @@ -5,6 +5,7 @@ import ( "strings" "testing" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -91,9 +92,9 @@ func buildRecord(t *testing.T, repo, module, path, at string) []byte { return logged(t, "build-"+module, module, repo, path, "", "", at) } -func pull(repo string, number int, branch, head string, dirs ...string) delivery.Forge { - return delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: repo, Branch: branch, Number: number, Head: head, - Action: delivery.PullUpdated, Dirs: dirs, DirsKnown: dirs != nil} +func pull(repo string, number int, branch, head string, dirs ...string) forgemsg.Forge { + return forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: repo, Branch: branch, Number: number, Head: head, + Action: forgemsg.PullUpdated, Dirs: dirs, DirsKnown: dirs != nil} } func newIndexer(t *testing.T) (*Indexer, *memStore, *strings.Builder) { @@ -212,13 +213,13 @@ func TestClosedAndMergedTakeAPullOutAndTheLastDeletesTheKey(t *testing.T) { // 1 ctx := context.Background() _ = ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a")) _ = ix.Apply(ctx, pull("novox/mesh-catalog", 300, "feat/x", h2, "modules/c")) - gone := func(repo string, n int, head string) delivery.Forge { + gone := func(repo string, n int, head string) forgemsg.Forge { if how == "closed" { f := pull(repo, n, "feat/x", head) - f.Action = delivery.PullClosed + f.Action = forgemsg.PullClosed return f } - return delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: repo, Branch: "feat/x", Number: n, Head: head, Merge: h3, Root: "r", Trees: map[string]string{"a": "t"}} + return forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: repo, Branch: "feat/x", Number: n, Head: head, Merge: h3, Root: "r", Trees: map[string]string{"a": "t"}} } if err := ix.Apply(ctx, gone("novox/mesh-delivery", 12, h1)); err != nil { t.Fatal(err) @@ -330,8 +331,8 @@ func TestAnOddBranchNameKeysEscaped(t *testing.T) { } } -func open(pulls ...delivery.OpenPull) delivery.Forge { - return delivery.Forge{V: delivery.Version, Kind: delivery.ForgeOpen, Open: pulls} +func open(pulls ...forgemsg.OpenPull) forgemsg.Forge { + return forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeOpen, Open: pulls} } // 1279/AC7 (ADR 0443 §5): on an open message, candidates lose every pull request it does not name, a group left empty is @@ -339,7 +340,7 @@ func open(pulls ...delivery.OpenPull) delivery.Forge { func TestAnOpenMessageDropsEveryCandidateItDoesNotName(t *testing.T) { // 1279/AC7 ix, st, _ := newIndexer(t) ctx := context.Background() - for _, f := range []delivery.Forge{ + for _, f := range []forgemsg.Forge{ pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"), pull("novox/mesh-catalog", 300, "feat/x", h2, "modules/c"), pull("novox/mesh-delivery", 13, "feat/y", h2, "modules/a"), pull("novox/mesh-catalog", 280, "feat/old", h3, "modules/c"), @@ -351,7 +352,7 @@ func TestAnOpenMessageDropsEveryCandidateItDoesNotName(t *testing.T) { // 1279/A before, _ := read(t, st, "feat/x") // Open: mesh-delivery#12 only (and a pull request no group holds, which changes nothing). - if err := ix.Apply(ctx, open(delivery.OpenPull{Repository: "novox/mesh-delivery", Number: 12}, delivery.OpenPull{Repository: "novox/mesh-catalog", Number: 999})); err != nil { + if err := ix.Apply(ctx, open(forgemsg.OpenPull{Repository: "novox/mesh-delivery", Number: 12}, forgemsg.OpenPull{Repository: "novox/mesh-catalog", Number: 999})); err != nil { t.Fatal(err) } c, ok := read(t, st, "feat/x") @@ -374,7 +375,7 @@ func TestAnOpenMessageDropsEveryCandidateItDoesNotName(t *testing.T) { // 1279/A } // The same open message again changes nothing (no write), and an empty one drops every group. writes := st.writes - if err := ix.Apply(ctx, open(delivery.OpenPull{Repository: "novox/mesh-delivery", Number: 12}, delivery.OpenPull{Repository: "novox/mesh-catalog", Number: 300})); err != nil { + if err := ix.Apply(ctx, open(forgemsg.OpenPull{Repository: "novox/mesh-delivery", Number: 12}, forgemsg.OpenPull{Repository: "novox/mesh-catalog", Number: 300})); err != nil { t.Fatal(err) } if st.writes != writes { diff --git a/modules/mesh-delivery-indexer/internal/index/manifests.go b/modules/mesh-delivery-indexer/internal/index/manifests.go index 740e9ce7..78340eb7 100644 --- a/modules/mesh-delivery-indexer/internal/index/manifests.go +++ b/modules/mesh-delivery-indexer/internal/index/manifests.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -48,7 +49,7 @@ func (ix *Indexer) keepManifest(ctx context.Context, module, raw string) (string // keepManifests writes the manifest of each directory a message carries, and answers the digest of each by directory. // A manifest that cannot be kept is left out and said in the log, so the module reads as one with no manifest. -func (ix *Indexer) keepManifests(ctx context.Context, f delivery.Forge) (map[string]string, error) { +func (ix *Indexer) keepManifests(ctx context.Context, f forgemsg.Forge) (map[string]string, error) { digests := make(map[string]string, len(f.Manifests)) for dir, raw := range f.Manifests { module := manifestModule(raw) // the name the manifest gives itself: one digest always has one module diff --git a/modules/mesh-delivery-indexer/internal/index/manifests_test.go b/modules/mesh-delivery-indexer/internal/index/manifests_test.go index b9817ec9..132aeed4 100644 --- a/modules/mesh-delivery-indexer/internal/index/manifests_test.go +++ b/modules/mesh-delivery-indexer/internal/index/manifests_test.go @@ -5,6 +5,7 @@ import ( "os" "testing" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -37,7 +38,7 @@ func manifestsIndexer(t *testing.T, recs ...[2]any) (*Indexer, *memStore, *memSt return ix, mods, manifests } -func withManifests(f delivery.Forge, manifests map[string]string, locks map[string]map[string]string) delivery.Forge { +func withManifests(f forgemsg.Forge, manifests map[string]string, locks map[string]map[string]string) forgemsg.Forge { f.Manifests, f.Locks = manifests, locks return f } @@ -81,8 +82,8 @@ func sameManifestAppliedAgainIsWrittenOnce(t *testing.T) { // 1449/AC2 if err := ix.Apply(context.Background(), f); err != nil { t.Errorf("a second merge: %v", err) } - pull := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: catalogueRepo, Branch: "feat/x", Number: 301, Head: h1, - Action: delivery.PullUpdated, Dirs: []string{"modules/avahi"}, DirsKnown: true, Manifests: map[string]string{"modules/avahi": avahi}} + pull := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: catalogueRepo, Branch: "feat/x", Number: 301, Head: h1, + Action: forgemsg.PullUpdated, Dirs: []string{"modules/avahi"}, DirsKnown: true, Manifests: map[string]string{"modules/avahi": avahi}} if err := ix.Apply(context.Background(), pull); err != nil { t.Errorf("a pull message: %v", err) } diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index 241bd88e..87c621f0 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -7,6 +7,7 @@ import ( "slices" "strings" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -31,20 +32,20 @@ import ( // // A merge never moves a module backwards (order.go): a merged message older than the repository's taken merge writes and // deletes nothing, and is said; it is asked once, before anything is written. -func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) (*mergeWrite, error) { +func (ix *Indexer) applyMerged(ctx context.Context, f forgemsg.Forge) (*mergeWrite, error) { return ix.applyManifests(ctx, f, true) } // applyManifests is applyMerged; drop says whether the modules of the repository the message holds no manifest for are // deleted. A message that names only some of a repository's modules (a registered module read on its own, registered.go) // drops none. -func (ix *Indexer) applyManifests(ctx context.Context, f delivery.Forge, drop bool) (*mergeWrite, error) { +func (ix *Indexer) applyManifests(ctx context.Context, f forgemsg.Forge, drop bool) (*mergeWrite, error) { return ix.applyMoving(ctx, f, drop, nil) } // applyMoving is applyManifests; moves names the modules keys the message may take from the repository that holds them // (a move handed over, moves.go). Every other module held of another repository is left, said and kept waiting. -func (ix *Indexer) applyMoving(ctx context.Context, f delivery.Forge, drop bool, moves map[string]bool) (*mergeWrite, error) { +func (ix *Indexer) applyMoving(ctx context.Context, f forgemsg.Forge, drop bool, moves map[string]bool) (*mergeWrite, error) { w := newMergeWrite(f) if drop { if err := ix.lateMerge(ctx, w); err != nil || w.late != "" { @@ -180,7 +181,7 @@ func withV(m delivery.Module) delivery.Module { m.V = delivery.Version; return m // namedObjects are, by module directory, the git objects of the paths each manifest of the merge names for every artifact // (catalogue Manifest.NamedPaths): those the message carries, else asked of the forge at the merge commit in one call // (Named). A forge that cannot say gives, for each directory asked, its words, so the module's reason names them. -func (ix *Indexer) namedObjects(ctx context.Context, f delivery.Forge) (map[string]map[string]string, map[string]string) { +func (ix *Indexer) namedObjects(ctx context.Context, f forgemsg.Forge) (map[string]map[string]string, map[string]string) { objects := map[string]map[string]string{} need := map[string][]string{} for dir, raw := range f.Manifests { @@ -220,7 +221,7 @@ func (ix *Indexer) namedObjects(ctx context.Context, f delivery.Forge) (map[stri // held(key, module) says whether the trunk holds it. A module of another repository, or of none known, is never touched. func (ix *Indexer) dropRemoved(ctx context.Context, repository string, held map[string]delivery.Module, trunkHolds func(key string, m delivery.Module) bool) error { var handed []string - elsewhere := map[string]delivery.Forge{} + elsewhere := map[string]forgemsg.Forge{} for _, key := range slices.Sorted(maps.Keys(held)) { cur := held[key] if cur.Repository == "" || !delivery.SameRepository(cur.Repository, repository) || trunkHolds(key, cur) { @@ -354,7 +355,7 @@ func unreadInputs(tree string, locks, paths map[string]string, packages []string // goSumPackages are the version line's packages of the merge's directory dir (ADR 0573 §2, ADR 0593 §2): the mesh Go // modules of other repositories its go.sum names, and those of its own repository its go.mod replaces by a directory // (siblingPackages), which no go.sum line names. A directory with no go.sum is no Go module and has none. -func (ix *Indexer) goSumPackages(ctx context.Context, f delivery.Forge, dir string, cur delivery.Module, has bool) []string { +func (ix *Indexer) goSumPackages(ctx context.Context, f forgemsg.Forge, dir string, cur delivery.Module, has bool) []string { others := ix.otherPackages(ctx, f, dir, cur, has) if f.Locks[dir]["go.sum"] == "" && len(f.Packages[dir]) == 0 { return others @@ -368,7 +369,7 @@ func (ix *Indexer) goSumPackages(ctx context.Context, f delivery.Forge, dir stri // siblingPackages are the mesh Go modules of the merge's own repository the go.mod of dir replaces by a directory, read // from the forge (GoMod); where it cannot be read, the held module's own-repository packages stand, said. -func (ix *Indexer) siblingPackages(ctx context.Context, f delivery.Forge, dir string, cur delivery.Module, has bool) []string { +func (ix *Indexer) siblingPackages(ctx context.Context, f forgemsg.Forge, dir string, cur delivery.Module, has bool) []string { held := func() []string { if !has || cur.Inputs2 == nil { return nil @@ -395,7 +396,7 @@ func (ix *Indexer) siblingPackages(ctx context.Context, f delivery.Forge, dir st // otherPackages are the mesh Go modules of other repositories the go.sum of the merge's directory dir names: those the // message carries (an older bridge), else the held module's while its go.sum's hash is unchanged, else the go.sum read // from the forge (GoSum). -func (ix *Indexer) otherPackages(ctx context.Context, f delivery.Forge, dir string, cur delivery.Module, has bool) []string { +func (ix *Indexer) otherPackages(ctx context.Context, f forgemsg.Forge, dir string, cur delivery.Module, has bool) []string { if carried := f.Packages[dir]; len(carried) > 0 { return fingerprint.OtherPackages(carried, f.Repository) // its own repository's come with its tree } diff --git a/modules/mesh-delivery-indexer/internal/index/modules_test.go b/modules/mesh-delivery-indexer/internal/index/modules_test.go index 294e2ff5..370b0a88 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules_test.go +++ b/modules/mesh-delivery-indexer/internal/index/modules_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" @@ -71,12 +72,12 @@ func rec(t *testing.T, id, module, repo, path, ref, state, at, inputs string) [2 // mergedMsg is a merged message of the repository with these module directories' trees, each with a manifest naming its // module, the directory's last element ("r" for the root), as the forge bridge sends it (one manifest per directory that // holds a module.json). -func mergedMsg(repo string, trees map[string]string) delivery.Forge { +func mergedMsg(repo string, trees map[string]string) forgemsg.Forge { manifests := map[string]string{} for dir := range trees { manifests[dir] = `{"module":"` + nameOfDir(dir) + `","version":"1"}` } - return delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: repo, Branch: "feat/x", Number: 12, Head: h1, Merge: merge, Root: "r0", Trees: trees, Manifests: manifests} + return forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: repo, Branch: "feat/x", Number: 12, Head: h1, Merge: merge, Root: "r0", Trees: trees, Manifests: manifests} } func nameOfDir(dir string) string { @@ -459,7 +460,7 @@ func TestAModuleMovesBetweenRepositoriesOnlyWhenItsTrunkDropsIt(t *testing.T) { gone.Trees, gone.Manifests = nil, nil for _, tc := range []struct { name string - merges []delivery.Forge + merges []forgemsg.Forge repository string // modules.a's repository at the end, "" for none deletedKeyA bool // Issue 1367: a fresh indexer, no wait in memory, the source naming heads; trunk is the module directories the forge @@ -472,20 +473,20 @@ func TestAModuleMovesBetweenRepositoriesOnlyWhenItsTrunkDropsIt(t *testing.T) { stored bool bGoneAtEnd bool }{ - {name: "named while its trunk holds it: left", merges: []delivery.Forge{held, named}, repository: from}, - {name: "named, then dropped by its trunk: handed over in place", merges: []delivery.Forge{held, named, dropped}, repository: to}, - {name: "dropped by its trunk, then named: deleted, then written new", merges: []delivery.Forge{held, dropped, named}, repository: to, deletedKeyA: true}, - {name: "dropped by its trunk with no merge naming it: deleted", merges: []delivery.Forge{held, dropped}, deletedKeyA: true}, + {name: "named while its trunk holds it: left", merges: []forgemsg.Forge{held, named}, repository: from}, + {name: "named, then dropped by its trunk: handed over in place", merges: []forgemsg.Forge{held, named, dropped}, repository: to}, + {name: "dropped by its trunk, then named: deleted, then written new", merges: []forgemsg.Forge{held, dropped, named}, repository: to, deletedKeyA: true}, + {name: "dropped by its trunk with no merge naming it: deleted", merges: []forgemsg.Forge{held, dropped}, deletedKeyA: true}, {name: "no module.json left, another trunk holds it, nothing waits in memory: written from that trunk, never deleted", - merges: []delivery.Forge{held, gone}, heads: true, trunk: []string{"modules/a", "modules/other"}, repository: to, bGoneAtEnd: true}, + merges: []forgemsg.Forge{held, gone}, heads: true, trunk: []string{"modules/a", "modules/other"}, repository: to, bGoneAtEnd: true}, {name: "no module.json left, no other trunk holds it: deleted", - merges: []delivery.Forge{held, gone}, heads: true, deletedKeyA: true, bGoneAtEnd: true}, + merges: []forgemsg.Forge{held, gone}, heads: true, deletedKeyA: true, bGoneAtEnd: true}, {name: "taken in by repository, a head with no module.json: handed over from the other trunk", - merges: []delivery.Forge{held}, heads: true, trunk: []string{"modules/a"}, takeIn: from, answer: "handed over", stored: true, repository: to, bGoneAtEnd: true}, + merges: []forgemsg.Forge{held}, heads: true, trunk: []string{"modules/a"}, takeIn: from, answer: "handed over", stored: true, repository: to, bGoneAtEnd: true}, {name: "taken in by repository, a head with modules: answered by name", - merges: []delivery.Forge{held}, heads: true, trunk: []string{"modules/a"}, takeIn: to, answer: "by name", repository: from}, + merges: []forgemsg.Forge{held}, heads: true, trunk: []string{"modules/a"}, takeIn: to, answer: "by name", repository: from}, {name: "taken in by repository, no head named: answered so", - merges: []delivery.Forge{held}, heads: true, takeIn: "novox/nobody", answer: "names no head", repository: from}, + merges: []forgemsg.Forge{held}, heads: true, takeIn: "novox/nobody", answer: "names no head", repository: from}, } { t.Run(tc.name, func(t *testing.T) { mods := &deletesSeen{memStore: newMem()} diff --git a/modules/mesh-delivery-indexer/internal/index/moves.go b/modules/mesh-delivery-indexer/internal/index/moves.go index e192731a..04553296 100644 --- a/modules/mesh-delivery-indexer/internal/index/moves.go +++ b/modules/mesh-delivery-indexer/internal/index/moves.go @@ -7,6 +7,7 @@ import ( "slices" "strings" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -14,9 +15,9 @@ import ( // holds it at the head the latest source names: the module directories there are asked of the forge (Trunk) and the manifest of // each directory named like the module is read (Read). It answers the merge to hand the module over with, or none: a forge // that cannot say leaves the module to be dropped as it was, said (issue 1367). -func (ix *Indexer) heldByAnotherTrunk(ctx context.Context, module, repository string) (delivery.Forge, bool) { +func (ix *Indexer) heldByAnotherTrunk(ctx context.Context, module, repository string) (forgemsg.Forge, bool) { if ix.Trunk == nil || ix.Read == nil { - return delivery.Forge{}, false + return forgemsg.Forge{}, false } ix.mu.Lock() heads := maps.Clone(ix.source.heads) @@ -45,7 +46,7 @@ func (ix *Indexer) heldByAnotherTrunk(ctx context.Context, module, repository st if read.Trees[dir] == "" || manifestModule(raw) != module { continue } - f := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: other, Merge: head, Root: read.Root, + f := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: other, Merge: head, Root: read.Root, Trees: map[string]string{dir: read.Trees[dir]}, Manifests: map[string]string{dir: raw}} if l := read.Locks[dir]; len(l) > 0 { f.Locks = map[string]map[string]string{dir: l} @@ -57,7 +58,7 @@ func (ix *Indexer) heldByAnotherTrunk(ctx context.Context, module, repository st return f, true } } - return delivery.Forge{}, false + return forgemsg.Forge{}, false } // nameOfDirectory is the last element of a module directory. @@ -76,8 +77,8 @@ func nameOfDirectory(dir string) string { return dir[strings.LastIndex(dir, "/") // waitToMove keeps, by modules key, the part of a merged message naming one module directory the indexer holds of another // repository. A later merge of the same repository replaces it. -func (ix *Indexer) waitToMove(key string, f delivery.Forge, dir string) { - one := delivery.Forge{V: f.V, Kind: f.Kind, Repository: f.Repository, Branch: f.Branch, Number: f.Number, Merge: f.Merge, Root: f.Root, +func (ix *Indexer) waitToMove(key string, f forgemsg.Forge, dir string) { + one := forgemsg.Forge{V: f.V, Kind: f.Kind, Repository: f.Repository, Branch: f.Branch, Number: f.Number, Merge: f.Merge, Root: f.Root, Trees: map[string]string{dir: f.Trees[dir]}, Manifests: map[string]string{dir: f.Manifests[dir]}} if l := f.Locks[dir]; len(l) > 0 { one.Locks = map[string]map[string]string{dir: l} @@ -91,7 +92,7 @@ func (ix *Indexer) waitToMove(key string, f delivery.Forge, dir string) { ix.mu.Lock() defer ix.mu.Unlock() if ix.moving == nil { - ix.moving = map[string]delivery.Forge{} + ix.moving = map[string]forgemsg.Forge{} } ix.moving[key] = one } @@ -117,7 +118,7 @@ func (ix *Indexer) handOver(ctx context.Context, key string) error { } // handOverFrom writes the module at key from the merge f, which names it, in place. -func (ix *Indexer) handOverFrom(ctx context.Context, key string, f delivery.Forge) error { +func (ix *Indexer) handOverFrom(ctx context.Context, key string, f forgemsg.Forge) error { w, err := ix.applyMoving(ctx, f, false, map[string]bool{key: true}) if err != nil { return fmt.Errorf("modules.%s, moving to %s: %w", key, f.Repository, err) diff --git a/modules/mesh-delivery-indexer/internal/index/order.go b/modules/mesh-delivery-indexer/internal/index/order.go index 5fac94da..221125de 100644 --- a/modules/mesh-delivery-indexer/internal/index/order.go +++ b/modules/mesh-delivery-indexer/internal/index/order.go @@ -7,6 +7,7 @@ import ( "slices" "strings" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" ) @@ -46,7 +47,7 @@ func (w *mergeWrite) taken(digest string) { w.fresh[digest] = true } -func newMergeWrite(f delivery.Forge) *mergeWrite { +func newMergeWrite(f forgemsg.Forge) *mergeWrite { return &mergeWrite{repository: f.Repository, merge: f.Merge} } diff --git a/modules/mesh-delivery-indexer/internal/index/own.go b/modules/mesh-delivery-indexer/internal/index/own.go index bcb0f81f..626b9cf4 100644 --- a/modules/mesh-delivery-indexer/internal/index/own.go +++ b/modules/mesh-delivery-indexer/internal/index/own.go @@ -7,6 +7,7 @@ import ( "path" "strings" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -70,7 +71,7 @@ func moduleAt(held map[string]delivery.Module, repository, dir string) (string, // moduleOfDir is the module of a changed directory of a pull request: the module its manifest in the message names, else // the one the modules bucket holds at that directory of the repository, else the directory's last element (the // catalogue's rule for nearly every manifest). -func moduleOfDir(held map[string]delivery.Module, f delivery.Forge, dir string) string { +func moduleOfDir(held map[string]delivery.Module, f forgemsg.Forge, dir string) string { if name := manifestModule(f.Manifests[dir]); name != "" { return name } diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index 6e660acc..19e91f1b 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -10,6 +10,8 @@ import ( "strings" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" @@ -21,7 +23,7 @@ import ( // build of the indexer writes only what building and moving that indexer needs, and the rest stays pending for it. const ( // PendingState is the indexer's state of held merges: the bucket mesh-delivery-indexer_pending, keyed - // delivery.PendingKey, so the key order is the merge order. An old `state` entry (the indexer is foundation, ADR + // takein.PendingKey, so the key order is the merge order. An old `state` entry (the indexer is foundation, ADR // 0595 §11), history 1, reached by the indexer alone. PendingState = "pending" PendingBucket = delivery.IndexerModule + "_" + PendingState @@ -66,23 +68,23 @@ type PendingEntry struct { Key string Revision uint64 Value Pending - Forge delivery.Forge + Forge forgemsg.Forge } // DecodePending reads one pending value and its merged message. -func DecodePending(data []byte) (Pending, delivery.Forge, error) { +func DecodePending(data []byte) (Pending, forgemsg.Forge, error) { var p Pending if err := json.Unmarshal(data, &p); err != nil { - return p, delivery.Forge{}, fmt.Errorf("not a pending value: %w", err) + return p, forgemsg.Forge{}, fmt.Errorf("not a pending value: %w", err) } if p.V != 1 { - return p, delivery.Forge{}, fmt.Errorf("a pending value of version %d, not 1", p.V) + return p, forgemsg.Forge{}, fmt.Errorf("a pending value of version %d, not 1", p.V) } - f, err := delivery.DecodeForge(p.Forge) + f, err := forgemsg.DecodeForge(p.Forge) if err != nil { return p, f, fmt.Errorf("its merged message: %w", err) } - if f.Kind != delivery.ForgeMerged { + if f.Kind != forgemsg.ForgeMerged { return p, f, fmt.Errorf("a pending value holds a %s message, not a merged one", f.Kind) } return p, f, nil @@ -143,7 +145,7 @@ func WaitsOf(entries []PendingEntry) Waits { w.Groups[e.Value.Group] = append(w.Groups[e.Value.Group], e) for _, m := range e.Forge.Members { if _, ok := blocked[strings.ToLower(m.Repository)]; !ok { - blocked[strings.ToLower(m.Repository)] = delivery.GroupTakeInKey(e.Value.Group) + blocked[strings.ToLower(m.Repository)] = takein.GroupTakeInKey(e.Value.Group) } } continue @@ -158,10 +160,10 @@ func WaitsOf(entries []PendingEntry) Waits { } // mergesOf are the merges of entries, oldest first. -func mergesOf(entries []PendingEntry) []delivery.GroupMerge { - out := make([]delivery.GroupMerge, len(entries)) +func mergesOf(entries []PendingEntry) []takein.Merge { + out := make([]takein.Merge, len(entries)) for i, e := range entries { - out[i] = delivery.GroupMerge{Repository: e.Forge.Repository, Number: e.Forge.Number, Merge: e.Forge.Merge} + out[i] = takein.Merge{Repository: e.Forge.Repository, Number: e.Forge.Number, Merge: e.Forge.Merge} } return out } @@ -177,8 +179,8 @@ func sinceOf(entries []PendingEntry) time.Time { } // groupWait is the take-in body of a group being gathered. -func groupWait(kind, group string, held []PendingEntry) delivery.TakeIn { - t := delivery.TakeIn{Kind: kind, Key: delivery.GroupTakeInKey(group), What: delivery.TakeInGroup, Group: group, +func groupWait(kind, group string, held []PendingEntry) takein.TakeIn { + t := takein.TakeIn{Kind: kind, Key: takein.GroupTakeInKey(group), What: takein.TakeInGroup, Group: group, Merges: mergesOf(held), Since: sinceOf(held)} if len(held) > 0 { t.Members = held[len(held)-1].Forge.Members @@ -187,8 +189,8 @@ func groupWait(kind, group string, held []PendingEntry) delivery.TakeIn { } // revisionWait is the take-in body of a revision: its key is its oldest merge's. -func revisionWait(kind string, ready []PendingEntry, waitsFor string) delivery.TakeIn { - return delivery.TakeIn{Kind: kind, Key: ready[0].Key, What: delivery.TakeInRevision, Merges: mergesOf(ready), +func revisionWait(kind string, ready []PendingEntry, waitsFor string) takein.TakeIn { + return takein.TakeIn{Kind: kind, Key: ready[0].Key, What: takein.TakeInRevision, Merges: mergesOf(ready), WaitsFor: waitsFor, Since: sinceOf(ready)} } @@ -196,7 +198,7 @@ func revisionWait(kind string, ready []PendingEntry, waitsFor string) delivery.T // (issue 1790), and until then every merge is taken in at once, so the build this program runs is not needed. A read // that fails otherwise counts as on the bus: the step that follows answers the error. func (ix *Indexer) pendingOnBus(ctx context.Context) bool { - _, _, _, err := ix.Pending.Get(ctx, delivery.PendingKey(0)) + _, _, _, err := ix.Pending.Get(ctx, takein.PendingKey(0)) return !errors.Is(err, process.ErrNotYet) } @@ -216,7 +218,7 @@ func (ix *Indexer) ownBuildNow(ctx context.Context) string { // sayTakeIn publishes a take-in event and keeps what it said on the entries, so indexer_pending reads the last event of // each wait from the state. A publish that fails is said in the log: the next evaluation says it again. -func (ix *Indexer) sayTakeIn(ctx context.Context, t delivery.TakeIn) bool { +func (ix *Indexer) sayTakeIn(ctx context.Context, t takein.TakeIn) bool { if ix.SayTakeIn == nil { return true } @@ -229,7 +231,7 @@ func (ix *Indexer) sayTakeIn(ctx context.Context, t delivery.TakeIn) bool { // markSaid writes on each entry the wait it was said under, by compare-and-set; a lost one is left (its next // evaluation says it again). -func (ix *Indexer) markSaid(ctx context.Context, entries []PendingEntry, t *delivery.TakeIn, change func(*Pending)) { +func (ix *Indexer) markSaid(ctx context.Context, entries []PendingEntry, t *takein.TakeIn, change func(*Pending)) { for _, e := range entries { p := e.Value if t != nil { @@ -253,7 +255,7 @@ func (ix *Indexer) markSaid(ctx context.Context, entries []PendingEntry, t *deli // holds is evaluated (Evaluate): taken in whole when its revision names the build this program runs, or held for the // indexer it names. It answers what was taken in, and a line saying what waits. A message is never written while the // build this program runs is not known (ErrOwnBuildUnknown). -func (ix *Indexer) Arrive(ctx context.Context, seq uint64, f delivery.Forge) (Applied, string, error) { +func (ix *Indexer) Arrive(ctx context.Context, seq uint64, f forgemsg.Forge) (Applied, string, error) { if ix.Pending == nil || ix.Modules == nil || !ix.pendingOnBus(ctx) { // The state this delivery brings is not on the bus yet (issue 1790): the merge is taken in at once, as before, // whether or not the build this program runs is known. @@ -266,11 +268,11 @@ func (ix *Indexer) Arrive(ctx context.Context, seq uint64, f delivery.Forge) (Ap } ix.pendingMu.Lock() defer ix.pendingMu.Unlock() - raw, err := delivery.EncodeForge(f) + raw, err := forgemsg.EncodeForge(f) if err != nil { return Applied{}, "", fmt.Errorf("%w: %v", ErrUnreadable, err) } - key := delivery.PendingKey(seq) + key := takein.PendingKey(seq) data, err := encodePending(Pending{Forge: raw, Group: f.Group, Since: ix.now(), LastWriter: own}) if err != nil { return Applied{}, "", err @@ -290,7 +292,7 @@ func (ix *Indexer) Arrive(ctx context.Context, seq uint64, f delivery.Forge) (Ap return Applied{}, "", err } if held := WaitsOf(entries).Groups[f.Group]; len(held) > 0 { - t := groupWait(delivery.TakeInPending, f.Group, held) + t := groupWait(takein.TakeInPending, f.Group, held) if ix.sayTakeIn(ctx, t) { ix.markSaid(ctx, held, &t, nil) } @@ -304,7 +306,7 @@ func (ix *Indexer) Arrive(ctx context.Context, seq uint64, f delivery.Forge) (Ap // one revision. A member named and neither held nor taken in yet answers an error, so the end is delivered again once // the member arrived. An end of a group taken in already (every member's merge at or behind its repository's taken // merge) changes nothing and is said: it is never delivered again for ever. -func (ix *Indexer) EndGroup(ctx context.Context, end delivery.ForgeGroupEnd) (Applied, string, error) { +func (ix *Indexer) EndGroup(ctx context.Context, end forgemsg.ForgeGroupEnd) (Applied, string, error) { if ix.Pending == nil || !ix.pendingOnBus(ctx) { return Applied{}, "no pending state: the group's merges were taken in one by one", nil } @@ -327,7 +329,7 @@ func (ix *Indexer) EndGroup(ctx context.Context, end delivery.ForgeGroupEnd) (Ap held = append(held, e) } } - isHeld := func(m delivery.GroupMerge) bool { + isHeld := func(m forgemsg.GroupMerge) bool { return slices.ContainsFunc(held, func(e PendingEntry) bool { return delivery.SameRepository(e.Forge.Repository, m.Repository) && e.Forge.Number == m.Number }) @@ -357,7 +359,7 @@ func (ix *Indexer) EndGroup(ctx context.Context, end delivery.ForgeGroupEnd) (Ap } // The group's wait ends first (take-in.taken under its group key); a revision it then is, pending for another // indexer, is said pending under its oldest merge's key by the evaluation that follows. - t := groupWait(delivery.TakeInTaken, end.Group, held) + t := groupWait(takein.TakeInTaken, end.Group, held) said := &t if !ix.sayTakeIn(ctx, t) { said = nil // said again by the take-in or the hold that follows (saidWaits) @@ -368,7 +370,7 @@ func (ix *Indexer) EndGroup(ctx context.Context, end delivery.ForgeGroupEnd) (Ap } // takenAlready says whether a group member's merge is at or behind its repository's taken merge. -func (ix *Indexer) takenAlready(ctx context.Context, m delivery.GroupMerge) (bool, error) { +func (ix *Indexer) takenAlready(ctx context.Context, m forgemsg.GroupMerge) (bool, error) { if ix.Taken == nil { return false, nil } @@ -622,21 +624,21 @@ func byWords(by string) string { } // saidWaits are the taken events of every wait the entries were said under that is not ended yet. -func saidWaits(entries []PendingEntry) []delivery.TakeIn { +func saidWaits(entries []PendingEntry) []takein.TakeIn { by := map[string][]PendingEntry{} for _, e := range entries { - if e.Value.Said != "" && e.Value.SaidKind == delivery.TakeInPending { + if e.Value.Said != "" && e.Value.SaidKind == takein.TakeInPending { by[e.Value.Said] = append(by[e.Value.Said], e) } } - var out []delivery.TakeIn + var out []takein.TakeIn for _, key := range slices.Sorted(maps.Keys(by)) { held := by[key] if strings.HasPrefix(key, "g.") { - out = append(out, groupWait(delivery.TakeInTaken, held[0].Value.Group, held)) + out = append(out, groupWait(takein.TakeInTaken, held[0].Value.Group, held)) continue } - t := revisionWait(delivery.TakeInTaken, held, held[0].Value.WaitsFor) + t := revisionWait(takein.TakeInTaken, held, held[0].Value.WaitsFor) t.Key = key out = append(out, t) } @@ -666,18 +668,18 @@ func (ix *Indexer) holdFor(ctx context.Context, ready []PendingEntry, names, own } } } - t := revisionWait(delivery.TakeInPending, ready, names) + t := revisionWait(takein.TakeInPending, ready, names) var ended, unmarked []PendingEntry said, changed := false, false for _, e := range ready { switch { - case e.Value.Said == t.Key && e.Value.SaidKind == delivery.TakeInPending: + case e.Value.Said == t.Key && e.Value.SaidKind == takein.TakeInPending: said = true changed = changed || e.Value.WaitsFor != names if e.Value.WaitsFor != names { unmarked = append(unmarked, e) } - case e.Value.Said != "" && e.Value.SaidKind == delivery.TakeInPending: + case e.Value.Said != "" && e.Value.SaidKind == takein.TakeInPending: ended = append(ended, e) unmarked = append(unmarked, e) default: @@ -722,28 +724,28 @@ func copyKey(ctx context.Context, from *overlayStore, to Store, key string) erro // PendingWaits is what indexer_pending answers of the pending state's entries (ADR 0633 §5): the ready merges as one // revision, each group being gathered, and the merges behind a group, each with the modules it holds back and the // last take-in event said of it. -func PendingWaits(entries []PendingEntry) []delivery.PendingWait { +func PendingWaits(entries []PendingEntry) []takein.PendingWait { w := WaitsOf(entries) - var out []delivery.PendingWait + var out []takein.PendingWait if len(w.Ready) > 0 { - out = append(out, waitOf(w.Ready, delivery.TakeInRevision, "")) + out = append(out, waitOf(w.Ready, takein.TakeInRevision, "")) } for _, g := range slices.Sorted(maps.Keys(w.Groups)) { - out = append(out, waitOf(w.Groups[g], delivery.TakeInGroup, "")) + out = append(out, waitOf(w.Groups[g], takein.TakeInGroup, "")) } for _, g := range slices.Sorted(maps.Keys(w.Behind)) { - out = append(out, waitOf(w.Behind[g], delivery.TakeInRevision, g)) + out = append(out, waitOf(w.Behind[g], takein.TakeInRevision, g)) } - slices.SortStableFunc(out, func(a, b delivery.PendingWait) int { return a.Since.Compare(b.Since) }) + slices.SortStableFunc(out, func(a, b takein.PendingWait) int { return a.Since.Compare(b.Since) }) return out } -func waitOf(entries []PendingEntry, what, behind string) delivery.PendingWait { - pw := delivery.PendingWait{Key: entries[0].Key, What: what, Merges: mergesOf(entries), Behind: behind, Since: sinceOf(entries), +func waitOf(entries []PendingEntry, what, behind string) takein.PendingWait { + pw := takein.PendingWait{Key: entries[0].Key, What: what, Merges: mergesOf(entries), Behind: behind, Since: sinceOf(entries), WaitsFor: entries[len(entries)-1].Value.WaitsFor, Holds: []string{}} - if what == delivery.TakeInGroup { + if what == takein.TakeInGroup { pw.Group = entries[0].Value.Group - pw.Key = delivery.GroupTakeInKey(pw.Group) + pw.Key = takein.GroupTakeInKey(pw.Group) pw.Members = entries[len(entries)-1].Forge.Members pw.WaitsFor = "" } @@ -755,7 +757,7 @@ func waitOf(entries []PendingEntry, what, behind string) delivery.PendingWait { } } if e.Value.Said != "" && e.Value.SaidAt.After(pw.SaidAt) { - pw.Said, pw.SaidAt = delivery.TakeInStream+"."+e.Value.SaidKind+"."+e.Value.Said, e.Value.SaidAt + pw.Said, pw.SaidAt = takein.TakeInStream+"."+e.Value.SaidKind+"."+e.Value.Said, e.Value.SaidAt } } slices.Sort(pw.Holds) @@ -785,7 +787,7 @@ func (ix *Indexer) TakeInPendingByHand(ctx context.Context, key, by, why string) switch { case strings.HasPrefix(key, "g."): for g, held := range w.Groups { - if delivery.GroupTakeInKey(g) == key { + if takein.GroupTakeInKey(g) == key { take = held } } @@ -838,7 +840,7 @@ func (ix *Indexer) DropPending(ctx context.Context, key, by, why string) TakenIn return r } ix.logf("take-in: %s (%s#%d at %s) dropped by hand, asked by %s: %s; nothing of it is taken in", key, e.Forge.Repository, e.Forge.Number, e.Forge.Merge, by, why) - if e.Value.Said != "" && e.Value.SaidKind == delivery.TakeInPending { + if e.Value.Said != "" && e.Value.SaidKind == takein.TakeInPending { var same []PendingEntry for _, o := range entries { if o.Key != key && o.Value.Said == e.Value.Said { diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index cca2d25e..51d7157f 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -9,6 +9,8 @@ import ( "strings" "testing" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" ) @@ -19,8 +21,8 @@ type takeInBed struct { ix *Indexer mods *memStore pending *memStore - said []delivery.TakeIn - base delivery.Forge + said []takein.TakeIn + base forgemsg.Forge logs *strings.Builder } @@ -36,7 +38,7 @@ func newTakeInBed(t *testing.T) *takeInBed { b := &takeInBed{ix: ix, mods: mods, pending: newMem(), base: f, logs: &strings.Builder{}} ix.Verdicts = newMem() ix.Pending = b.pending - ix.SayTakeIn = func(_ context.Context, t delivery.TakeIn) error { b.said = append(b.said, t); return nil } + ix.SayTakeIn = func(_ context.Context, t takein.TakeIn) error { b.said = append(b.said, t); return nil } logf := ix.Logf ix.Logf = func(format string, args ...any) { b.logs.WriteString(format + "\n") @@ -65,7 +67,7 @@ func (b *takeInBed) commit(t *testing.T, module string) string { } // merge is a catalogue merge at commit c whose trees of avahi and the indexer are given ("" keeps the base's). -func (b *takeInBed) merge(number int, c, avahi, indexer string) delivery.Forge { +func (b *takeInBed) merge(number int, c, avahi, indexer string) forgemsg.Forge { f := b.base f.Number, f.Merge, f.Head = number, c, c f.Trees = map[string]string{} @@ -81,7 +83,7 @@ func (b *takeInBed) merge(number int, c, avahi, indexer string) delivery.Forge { return f } -func (b *takeInBed) arrive(t *testing.T, seq uint64, f delivery.Forge) { +func (b *takeInBed) arrive(t *testing.T, seq uint64, f forgemsg.Forge) { t.Helper() if _, _, err := b.ix.Arrive(context.Background(), seq, f); err != nil { t.Fatalf("arrive %d: %v", seq, err) @@ -144,7 +146,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) if n := len(b.ix.Verdicts.(*memStore).vals); n > verdicts+1 { t.Errorf("verdicts %d, want at most %d: only the new indexer manifest's", n, verdicts+1) } - if want := []string{delivery.PendingKey(2)}; !reflect.DeepEqual(b.keys(), want) || !slices.Equal(b.events(), []string{"pending " + delivery.PendingKey(2)}) || b.said[0].WaitsFor != named { + if want := []string{takein.PendingKey(2)}; !reflect.DeepEqual(b.keys(), want) || !slices.Equal(b.events(), []string{"pending " + takein.PendingKey(2)}) || b.said[0].WaitsFor != named { t.Fatalf("pending %v, said %+v: want m.2 pending for %s", b.keys(), b.said, named) } // T5: the merge delivered again writes no second key and says nothing again. @@ -162,7 +164,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) if len(b.keys()) != 0 || b.fp(t, delivery.IndexerModule) != own || b.commit(t, "avahi") != sha('e') { t.Fatalf("revert: pending %v, indexer %s, avahi at %s: want all taken in", b.keys(), b.fp(t, delivery.IndexerModule), b.commit(t, "avahi")) } - if want := []string{"pending " + delivery.PendingKey(2), "taken " + delivery.PendingKey(2)}; !slices.Equal(b.events(), want) { + if want := []string{"pending " + takein.PendingKey(2), "taken " + takein.PendingKey(2)}; !slices.Equal(b.events(), want) { t.Errorf("said %v, want %v", b.events(), want) } }) @@ -192,7 +194,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) if _, _, err := next.TakeInAtStart(ctx, func(context.Context) bool { return true }); err != nil { t.Fatal(err) } - if asked < 2 || !reflect.DeepEqual(b.keys(), []string{delivery.PendingKey(5)}) || b.commit(t, "avahi") != sha('c') { + if asked < 2 || !reflect.DeepEqual(b.keys(), []string{takein.PendingKey(5)}) || b.commit(t, "avahi") != sha('c') { t.Errorf("start: asked %d, pending %v, avahi at %s: want the revision taken in, the group left", asked, b.keys(), b.commit(t, "avahi")) } }) @@ -204,7 +206,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) b.ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { return strings.IndexByte(order, commit[0]) <= strings.IndexByte(order, in[0]), nil } - member := func(repo string, number int, c string) delivery.Forge { + member := func(repo string, number int, c string) forgemsg.Forge { f := b.merge(number, c, "", "") f.Group = "feat/take-in" f.Members = []delivery.Head{{Repository: catalogueRepo, Number: 40, Head: sha('1')}, {Repository: "novox/mesh-sdk", Number: 41, Head: sha('2')}} @@ -223,10 +225,10 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) alone.Repository, alone.Trees, alone.Manifests, alone.Locks = "novox/hq", map[string]string{}, nil, nil b.arrive(t, 2, alone) b.arrive(t, 3, b.merge(51, sha('6'), "tree-avahi-9", "")) - if !reflect.DeepEqual(b.keys(), []string{delivery.PendingKey(1), delivery.PendingKey(3)}) { + if !reflect.DeepEqual(b.keys(), []string{takein.PendingKey(1), takein.PendingKey(3)}) { t.Fatalf("pending %v: want the member and the merge behind it", b.keys()) } - end := delivery.ForgeGroupEnd{Group: "feat/take-in", Members: []delivery.GroupMerge{{Repository: catalogueRepo, Number: 40, Merge: sha('7')}, {Repository: "novox/mesh-sdk", Number: 41, Merge: sha('b')}}} + end := forgemsg.ForgeGroupEnd{Group: "feat/take-in", Members: []forgemsg.GroupMerge{{Repository: catalogueRepo, Number: 40, Merge: sha('7')}, {Repository: "novox/mesh-sdk", Number: 41, Merge: sha('b')}}} if _, _, err := b.ix.EndGroup(ctx, end); err == nil { t.Fatal("an end before its last member is taken in") } @@ -250,10 +252,10 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) g.Group, g.Trees = "feat/two", b.merge(0, "", "", "tree-indexer-7").Trees g.Members = g.Members[:1] b.arrive(t, 9, g) - if _, _, err := b.ix.EndGroup(ctx, delivery.ForgeGroupEnd{Group: "feat/two", Members: []delivery.GroupMerge{{Repository: catalogueRepo, Number: 40, Merge: sha('c')}}}); err != nil { + if _, _, err := b.ix.EndGroup(ctx, forgemsg.ForgeGroupEnd{Group: "feat/two", Members: []forgemsg.GroupMerge{{Repository: catalogueRepo, Number: 40, Merge: sha('c')}}}); err != nil { t.Fatal(err) } - if want := []string{"pending g.feat/two", "taken g.feat/two", "pending " + delivery.PendingKey(9)}; !slices.Equal(b.events(), want) { + if want := []string{"pending g.feat/two", "taken g.feat/two", "pending " + takein.PendingKey(9)}; !slices.Equal(b.events(), want) { t.Errorf("a group naming another indexer: said %v, want %v", b.events(), want) } }) @@ -266,12 +268,12 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) b.ix.ownMu.Unlock() // The other indexer claims the merge between this one's read and its claim. b.pending.beforeWrite = func() { - v := b.pending.vals[delivery.PendingKey(2)] + v := b.pending.vals[takein.PendingKey(2)] var p Pending _ = json.Unmarshal(v, &p) p.Taking = "the other indexer" data, _ := json.Marshal(p) - b.pending.put(delivery.PendingKey(2), data) + b.pending.put(takein.PendingKey(2), data) } if _, _, err := b.ix.TakeInAtStart(ctx, func(context.Context) bool { return false }); err != nil { t.Fatal(err) @@ -284,32 +286,32 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) b := newTakeInBed(t) b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")) - var answer delivery.PendingAnswer + var answer takein.PendingAnswer for _, tool := range PendingTools(b.pending, nil, nil) { if tool.Name == PendingTool { out, err := tool.Run(map[string]any{}) if err != nil { t.Fatal(err) } - answer = out.(delivery.PendingAnswer) + answer = out.(takein.PendingAnswer) } } - if len(answer.Waits) != 1 || answer.Waits[0].Key != delivery.PendingKey(2) || !slices.Equal(answer.Waits[0].Holds, []string{"avahi", delivery.IndexerModule}) || - answer.Waits[0].Said != "take-in.pending."+delivery.PendingKey(2) || len(answer.Waits[0].Keys) != 2 { + if len(answer.Waits) != 1 || answer.Waits[0].Key != takein.PendingKey(2) || !slices.Equal(answer.Waits[0].Holds, []string{"avahi", delivery.IndexerModule}) || + answer.Waits[0].Said != "take-in.pending."+takein.PendingKey(2) || len(answer.Waits[0].Keys) != 2 { t.Fatalf("indexer_pending = %+v", answer) } b.said = nil - if r := b.ix.DropPending(ctx, delivery.PendingKey(2), "lead@shanks", "replaced by m.3"); !r.Stored { + if r := b.ix.DropPending(ctx, takein.PendingKey(2), "lead@shanks", "replaced by m.3"); !r.Stored { t.Fatalf("drop: %+v", r) } - if len(b.said) < 2 || !b.said[0].Dropped || b.said[0].Key != delivery.PendingKey(2) || b.said[0].By != "lead@shanks" || b.said[1].Key != delivery.PendingKey(3) { + if len(b.said) < 2 || !b.said[0].Dropped || b.said[0].Key != takein.PendingKey(2) || b.said[0].By != "lead@shanks" || b.said[1].Key != takein.PendingKey(3) { t.Fatalf("drop said %+v: want m.2 taken (dropped, by) and m.3 pending", b.said) } b.said = nil - if r := b.ix.TakeInPendingByHand(ctx, delivery.PendingKey(3), "lead@shanks", "the named indexer can never run"); !r.Stored { + if r := b.ix.TakeInPendingByHand(ctx, takein.PendingKey(3), "lead@shanks", "the named indexer can never run"); !r.Stored { t.Fatalf("hand take-in: %+v", r) } - if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('d') || len(b.said) != 1 || b.said[0].Kind != delivery.TakeInTaken || b.said[0].By != "lead@shanks" { + if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('d') || len(b.said) != 1 || b.said[0].Kind != takein.TakeInTaken || b.said[0].By != "lead@shanks" { t.Errorf("hand take-in: pending %v, avahi at %s, said %+v", b.keys(), b.commit(t, "avahi"), b.said) } }) diff --git a/modules/mesh-delivery-indexer/internal/index/pending_tools.go b/modules/mesh-delivery-indexer/internal/index/pending_tools.go index 33c08f4c..bf3ab9b7 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_tools.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_tools.go @@ -5,8 +5,8 @@ import ( "fmt" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" stdio "git.novox.be/novox/mesh-sdk/go" - "git.novox.be/novox/mesh-sdk/go/delivery" ) // PendingTools are the indexer's tools of what waits for a take-in (novox/hq ADR 0633 §5): indexer_pending reads the @@ -46,7 +46,7 @@ func PendingTools(pending Store, ask func(key string, value []byte) (TakeInAsk, if err != nil { return nil, err } - return delivery.PendingAnswer{Waits: PendingWaits(entries), Unread: unread}, nil + return takein.PendingAnswer{Waits: PendingWaits(entries), Unread: unread}, nil }}, {Name: TakeInPendingTool, Description: "Take a wait in by hand with the running indexer and its judge (ADR 0633 §5): the revision under its m. key, " + diff --git a/modules/mesh-delivery-indexer/internal/index/previews_test.go b/modules/mesh-delivery-indexer/internal/index/previews_test.go index 8d25c915..282ad860 100644 --- a/modules/mesh-delivery-indexer/internal/index/previews_test.go +++ b/modules/mesh-delivery-indexer/internal/index/previews_test.go @@ -7,6 +7,7 @@ import ( "strings" "testing" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/delivery" "mesh-delivery-indexer/internal/fingerprint" @@ -38,7 +39,7 @@ func avahiOnTools(t *testing.T) string { } // avahiPull is the pull message of the catalogue's pull request 301 touching avahi, with its manifest. -func avahiPull(t *testing.T, head string) delivery.Forge { +func avahiPull(t *testing.T, head string) forgemsg.Forge { t.Helper() f := manifestPull(t, 301, head, avahiOnTools(t)) f.Manifests = map[string]string{"modules/avahi": avahiOnTools(t)} diff --git a/modules/mesh-delivery-indexer/internal/index/registered.go b/modules/mesh-delivery-indexer/internal/index/registered.go index cf0940bb..09bf3d94 100644 --- a/modules/mesh-delivery-indexer/internal/index/registered.go +++ b/modules/mesh-delivery-indexer/internal/index/registered.go @@ -7,6 +7,7 @@ import ( "slices" "strings" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -125,7 +126,7 @@ func (ix *Indexer) indexAt(ctx context.Context, r Registration, force, reread bo t.why = fmt.Sprintf("the manifest at %q of %s at %s names the module %q", dir, repository, commit, name) return t, nil } - f := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: repository, Merge: commit, Root: read.Root, + f := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: repository, Merge: commit, Root: read.Root, Trees: map[string]string{dir: read.Trees[dir]}, Manifests: map[string]string{dir: raw}} if l := read.Locks[dir]; len(l) > 0 { @@ -396,7 +397,7 @@ func (ix *Indexer) TakeInRepository(ctx context.Context, repository string) Take out.Answer = "the modules state cannot be read: " + err.Error() return out } - f := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: out.Repository, Merge: head} + f := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: out.Repository, Merge: head} if _, err := ix.ApplyMessage(ctx, f); err != nil { out.Answer = fmt.Sprintf("the merge of %s at %.12s could not be taken in: %v", repository, head, err) return out diff --git a/modules/mesh-delivery-indexer/internal/index/taken.go b/modules/mesh-delivery-indexer/internal/index/taken.go index 89c02931..a66d7e08 100644 --- a/modules/mesh-delivery-indexer/internal/index/taken.go +++ b/modules/mesh-delivery-indexer/internal/index/taken.go @@ -8,6 +8,7 @@ import ( "slices" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" ) @@ -19,13 +20,13 @@ import ( // say keeps the key as it is and says so. // writeTaken writes the repository's taken key for the merge f carries, unless it names that merge or a later one. -func (ix *Indexer) writeTaken(ctx context.Context, f delivery.Forge) error { +func (ix *Indexer) writeTaken(ctx context.Context, f forgemsg.Forge) error { return ix.writeTakenKnown(ctx, f, "") } // writeTakenKnown is writeTaken where the merge is known not to be older than notOlder (lateMerge asked it, issue 1126): a // key naming that merge is written over without asking the forge again. -func (ix *Indexer) writeTakenKnown(ctx context.Context, f delivery.Forge, notOlder string) error { +func (ix *Indexer) writeTakenKnown(ctx context.Context, f forgemsg.Forge, notOlder string) error { if ix.Taken == nil || f.Merge == "" { return nil } @@ -101,7 +102,7 @@ func (ix *Indexer) FillTaken(ctx context.Context) error { } } for _, repository := range slices.Sorted(maps.Keys(pending)) { - err := ix.writeTaken(ctx, delivery.Forge{Repository: repository, Merge: pending[repository]}) + err := ix.writeTaken(ctx, forgemsg.Forge{Repository: repository, Merge: pending[repository]}) if err == nil { ix.mu.Lock() if ix.untaken[repository] == pending[repository] { diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index 435c00c6..4e62b312 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" @@ -91,7 +92,7 @@ func newFormIndexer(t *testing.T, mods ...mod) (*Indexer, *memStore) { } // newFormIndexerUnmerged is the indexer and the catalogue merge of these modules, not yet applied. -func newFormIndexerUnmerged(t *testing.T, mods ...mod) (*Indexer, *memStore, delivery.Forge) { +func newFormIndexerUnmerged(t *testing.T, mods ...mod) (*Indexer, *memStore, forgemsg.Forge) { t.Helper() var recs [][2]any trees, manifests := map[string]string{}, map[string]string{} @@ -468,7 +469,7 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) { tools, _ := moduleOf(t, mods, "mesh-tools") sdkMerge := func(commit, tree string) delivery.Module { t.Helper() - f := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: "novox/mesh-sdk", Branch: "feat/x", Number: 3, Head: h1, + f := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-sdk", Branch: "feat/x", Number: 3, Head: h1, Merge: commit, Root: tree, Trees: map[string]string{"": tree}, Manifests: map[string]string{"": `{"module":"mesh-sdk","version":"1"}`}} if err := ix.Apply(context.Background(), f); err != nil { t.Fatal(err) diff --git a/modules/mesh-delivery-indexer/internal/index/verdicts_test.go b/modules/mesh-delivery-indexer/internal/index/verdicts_test.go index b8ce4a49..6dd718a7 100644 --- a/modules/mesh-delivery-indexer/internal/index/verdicts_test.go +++ b/modules/mesh-delivery-indexer/internal/index/verdicts_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -319,7 +320,7 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) { t.Fatalf("the start pass: %+v, this judge's own refusal %+v (%v)", verdict(atStart), verdict(ownRefusal), err) } byPull := stale("tools", `{"module":"tools","version":"8"}`) - pull := delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-tools", Number: 3, Branch: "main", + pull := forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: "novox/mesh-tools", Number: 3, Branch: "main", Manifests: map[string]string{"modules/tools": `{"module":"tools","version":"8"}`}} if _, err := ix.ApplyMessage(ctx, pull); err != nil || verdict(byPull).Refused() || verdict(byPull).Judge != ix.judgeID() { t.Fatalf("a pull carrying a digest an older judge refused: %+v %v", verdict(byPull), err) diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index d8f68d3b..ace5d0ed 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -12,6 +12,7 @@ import ( "strings" "testing" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -147,7 +148,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build } // 1585/T11 (ADR 0633 §1, §5): its one stream, take-in, an old streams entry (the indexer is foundation, ADR 0595 §11), // carries the take-in events. - if want := []streamDecl{{Name: delivery.TakeInStream, Events: []string{"take-in.**"}, MaxAgeDays: 7}}; !reflect.DeepEqual(m.Streams, want) { + if want := []streamDecl{{Name: takein.TakeInStream, Events: []string{"take-in.**"}, MaxAgeDays: 7}}; !reflect.DeepEqual(m.Streams, want) { t.Errorf("streams = %+v, want %+v", m.Streams, want) } // No other bus section, no other seat (ADR 0437 §4). diff --git a/modules/mesh-delivery-indexer/takein/go.mod b/modules/mesh-delivery-indexer/takein/go.mod new file mode 100644 index 00000000..48ab0844 --- /dev/null +++ b/modules/mesh-delivery-indexer/takein/go.mod @@ -0,0 +1,5 @@ +module git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein + +go 1.26.0 + +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659 diff --git a/modules/mesh-delivery-indexer/takein/go.sum b/modules/mesh-delivery-indexer/takein/go.sum new file mode 100644 index 00000000..5e18d943 --- /dev/null +++ b/modules/mesh-delivery-indexer/takein/go.sum @@ -0,0 +1,14 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659 h1:tK7MzxVm/ECNg1Yblhh1ynVEthbZCg2mSPW0wQZC0b0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= diff --git a/modules/mesh-delivery-indexer/takein/takein.go b/modules/mesh-delivery-indexer/takein/takein.go new file mode 100644 index 00000000..dc53626c --- /dev/null +++ b/modules/mesh-delivery-indexer/takein/takein.go @@ -0,0 +1,251 @@ +// Package takein is the indexer's take-in contract (novox/hq ADR 0633 §1, §5): the events it publishes on its own stream +// `take-in` while a source revision waits for the indexer it names or a delivery group is gathered, their keys, and the +// answer of its tool indexer_pending. The indexer writes them; the watchman and the view read them. It is the indexer's +// own package: one module owns these types, so they live with it and not in mesh-sdk (the operator, 2026-11-04). +package takein + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Merge is one merge a take-in event or a wait holds: the repository as `/`, its pull request and the +// merge commit the forge made of it. +type Merge struct { + Repository string `json:"repository"` + Number int `json:"number"` + Merge string `json:"merge"` +} + +// checkRepository refuses a repository that is not `/`: one slash, neither part empty. +func checkRepository(repository string) error { + owner, name, ok := strings.Cut(repository, "/") + if !ok || owner == "" || name == "" || strings.Contains(name, "/") { + return fmt.Errorf("delivery: the repository %q is not /, with exactly one slash", repository) + } + return nil +} + +// decode reads a value of delivery.Version, refusing another or none with delivery.ErrUnknownVersion, as mesh-sdk +// delivery's values are read. +func decode(data []byte, into any) error { + var probe struct { + V *int `json:"v"` + } + if err := json.Unmarshal(data, &probe); err != nil { + return fmt.Errorf("delivery: value is not JSON: %w", err) + } + if probe.V == nil || *probe.V != delivery.Version { + return fmt.Errorf("%w: this reader reads %d", delivery.ErrUnknownVersion, delivery.Version) + } + if err := json.Unmarshal(data, into); err != nil { + return fmt.Errorf("delivery: value does not fit its shape: %w", err) + } + return nil +} + +// The take-in events (novox/hq ADR 0633 §1, §5): the indexer publishes them on its own stream, which keeps the last +// message per subject, `take-in..`. A wait starts with `take-in.pending` and ends with `take-in.taken`, +// whoever ended it (the named indexer, a revert, a hand take-in or a drop). The stream's name is the indexer's manifest's; +// these names are its events, relative to it. +const ( + TakeInStream = "take-in" + TakeInPending = "pending" + TakeInTaken = "taken" +) + +// What waits in a take-in event: a source revision for the indexer build it names, or a delivery group for its end. +const ( + TakeInRevision = "revision" + TakeInGroup = "group" +) + +// PendingKey is the key of one held merge in the indexer's pending state, and the key a revision's take-in events +// carry (its oldest merge's): `m.`, so the key order is the merge order. +func PendingKey(seq uint64) string { return fmt.Sprintf("m.%020d", seq) } + +// GroupTakeInKey is the key a group's take-in events carry while it is gathered: `g.`, with every character a +// bus subject cannot carry (a space, `*`, `>`) written `_`. The event's body names the group exactly. +func GroupTakeInKey(group string) string { + return "g." + strings.Map(func(r rune) rune { + if r == '*' || r == '>' || r <= ' ' || r == 0x7f { + return '_' + } + return r + }, group) +} + +// TakeInEvent is the name of a take-in event of the given kind and key: `take-in..`. +func TakeInEvent(kind, key string) (string, error) { + if kind != TakeInPending && kind != TakeInTaken { + return "", fmt.Errorf("delivery: the take-in kind %q is not pending or taken", kind) + } + if err := checkTakeInKey(key); err != nil { + return "", err + } + return TakeInStream + "." + kind + "." + key, nil +} + +// ParseTakeInEvent is the inverse of TakeInEvent. +func ParseTakeInEvent(event string) (kind, key string, err error) { + parts := strings.SplitN(event, ".", 3) + if len(parts) != 3 || parts[0] != TakeInStream { + return "", "", fmt.Errorf("delivery: %q is not a take-in event, take-in..", event) + } + if _, err := TakeInEvent(parts[1], parts[2]); err != nil { + return "", "", fmt.Errorf("delivery: %q is not a take-in event: %w", event, err) + } + return parts[1], parts[2], nil +} + +func checkTakeInKey(key string) error { + if !strings.HasPrefix(key, "m.") && !strings.HasPrefix(key, "g.") { + return fmt.Errorf("delivery: the take-in key %q is neither m. nor g.", key) + } + for _, tok := range strings.Split(key, ".") { + if tok == "" || strings.ContainsAny(tok, "*> \t\r\n") { + return fmt.Errorf("delivery: the take-in key %q is not a bus subject's tokens", key) + } + } + return nil +} + +// TakeIn is the body of a take-in event. Kind is the event's kind and Key its key. What says what waits: a revision +// (Key its oldest merge's PendingKey, WaitsFor the indexer build it names) or a group being gathered (Key its +// GroupTakeInKey, Members every approved member with its head, Merges the members held so far). Merges are the merges +// it holds, oldest first; Since is when the wait began. A `taken` event names the same once the wait ended; By and Why +// name a hand act (a hand take-in, or a drop, Dropped), and are carried by nothing else. +type TakeIn struct { + V int `json:"v"` + Kind string `json:"kind"` + Key string `json:"key"` + What string `json:"what"` + Merges []Merge `json:"merges"` + Group string `json:"group,omitempty"` + Members []delivery.Head `json:"members,omitempty"` + WaitsFor string `json:"waits-for,omitempty"` + Since time.Time `json:"since"` + By string `json:"by,omitempty"` + Why string `json:"why,omitempty"` + Dropped bool `json:"dropped,omitempty"` +} + +func (t TakeIn) check() error { + if _, err := TakeInEvent(t.Kind, t.Key); err != nil { + return err + } + if len(t.Merges) == 0 { + return fmt.Errorf("delivery: the take-in event %s names no merge", t.Key) + } + for _, m := range t.Merges { + if err := checkRepository(m.Repository); err != nil { + return err + } + if m.Merge == "" { + return fmt.Errorf("delivery: the take-in event %s names a merge of %s without its commit", t.Key, m.Repository) + } + } + if t.Since.IsZero() { + return fmt.Errorf("delivery: the take-in event %s does not say since when it waits", t.Key) + } + switch t.What { + case TakeInRevision: + if !strings.HasPrefix(t.Key, "m.") { + return fmt.Errorf("delivery: a revision's take-in key is its oldest merge's m., not %q", t.Key) + } + if t.Kind == TakeInPending && t.WaitsFor == "" { + return fmt.Errorf("delivery: a pending revision %s names the indexer build it waits for", t.Key) + } + case TakeInGroup: + if t.Group == "" { + return fmt.Errorf("delivery: a group's take-in event %s names its group", t.Key) + } + if t.Key != GroupTakeInKey(t.Group) { + return fmt.Errorf("delivery: the take-in key of the group %q is %s, not %s", t.Group, GroupTakeInKey(t.Group), t.Key) + } + if t.WaitsFor != "" { + return fmt.Errorf("delivery: a group being gathered waits for its end, not for an indexer build") + } + default: + return fmt.Errorf("delivery: a take-in event's what %q is not revision or group", t.What) + } + if t.Kind != TakeInTaken && (t.By != "" || t.Why != "" || t.Dropped) { + return fmt.Errorf("delivery: only a taken event names the hand act that ended a wait") + } + if (t.Why != "" || t.Dropped) && t.By == "" { + return fmt.Errorf("delivery: a hand act's take-in event names who acted (by)") + } + return nil +} + +// EncodeTakeIn is t as JSON at Version; it refuses what its kind and what do not carry. +func EncodeTakeIn(t TakeIn) ([]byte, error) { + if err := t.check(); err != nil { + return nil, err + } + t.V = delivery.Version + return json.Marshal(t) +} + +// DecodeTakeIn reads a take-in event's body; it refuses an unknown version (ErrUnknownVersion) and what EncodeTakeIn +// refuses. +func DecodeTakeIn(data []byte) (TakeIn, error) { + var t TakeIn + if err := decode(data, &t); err != nil { + return TakeIn{}, err + } + if err := t.check(); err != nil { + return TakeIn{}, err + } + return t, nil +} + +// SameTakeIn says whether two take-in bodies are the same value. +func SameTakeIn(a, b TakeIn) bool { return reflect.DeepEqual(a, b) } + +// PendingWait is one wait the indexer's tool indexer_pending answers (novox/hq ADR 0633 §5), as the view reads it to +// name a module waiting for its revision's indexer. Key and What are a take-in event's: a revision under its oldest +// merge's key, waiting for the indexer build WaitsFor (empty while it is worked out), or a group being gathered under +// its group key, with Members every approved member and Merges those held so far. A revision that waits behind a +// group being gathered names that group's key in Behind. Keys are the pending keys of its merges, in Merges' order (indexer_drop_pending drops one). Holds are the modules it holds back: every module of its +// merges' trees. Said is the last take-in event said of it (`take-in..`) and SaidAt when. +type PendingWait struct { + Key string `json:"key"` + What string `json:"what"` + Merges []Merge `json:"merges"` + Keys []string `json:"keys"` + Group string `json:"group,omitempty"` + Members []delivery.Head `json:"members,omitempty"` + Behind string `json:"behind,omitempty"` + WaitsFor string `json:"waits-for,omitempty"` + Since time.Time `json:"since"` + Holds []string `json:"holds"` + Said string `json:"said,omitempty"` + SaidAt time.Time `json:"said-at,omitzero"` +} + +// PendingAnswer is indexer_pending's answer: every wait, oldest first, and the pending keys that do not read. +type PendingAnswer struct { + Waits []PendingWait `json:"waits"` + Unread []string `json:"unread,omitempty"` +} + +// DecodePendingAnswer reads indexer_pending's answer, refusing a wait that is not a revision or a group, or names no +// key. +func DecodePendingAnswer(data []byte) (PendingAnswer, error) { + var a PendingAnswer + if err := json.Unmarshal(data, &a); err != nil { + return a, fmt.Errorf("delivery: not an answer of indexer_pending: %w", err) + } + for _, w := range a.Waits { + if w.Key == "" || (w.What != TakeInRevision && w.What != TakeInGroup) { + return a, fmt.Errorf("delivery: indexer_pending answers a wait %q of %q, not a revision or a group", w.Key, w.What) + } + } + return a, nil +} diff --git a/modules/mesh-delivery-indexer/takein/takein_test.go b/modules/mesh-delivery-indexer/takein/takein_test.go new file mode 100644 index 00000000..a37bc4c6 --- /dev/null +++ b/modules/mesh-delivery-indexer/takein/takein_test.go @@ -0,0 +1,95 @@ +package takein + +import ( + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// sampleTakeIns are a pending revision, a group being gathered and a hand take-in's taken (1585/T2). +func sampleTakeIns(since time.Time) []TakeIn { + m := []Merge{{Repository: "novox/mesh-delivery", Number: 9, Merge: "eeee"}} + return []TakeIn{ + {Kind: TakeInPending, Key: PendingKey(12), What: TakeInRevision, Merges: m, WaitsFor: "src2:abc", Since: since}, + {Kind: TakeInPending, Key: GroupTakeInKey("feat/take-in"), What: TakeInGroup, Merges: m, Group: "feat/take-in", + Members: []delivery.Head{{Repository: "novox/mesh-delivery", Number: 9, Head: "ffff"}, {Repository: "novox/mesh-sdk", Number: 37, Head: "dddd"}}, Since: since}, + {Kind: TakeInTaken, Key: PendingKey(12), What: TakeInRevision, Merges: m, WaitsFor: "src2:abc", Since: since, + By: "lead-47@shanks", Why: "the named indexer can never run"}, + } +} + +// TestTakeInEvents holds the indexer's take-in contract, moved from mesh-sdk go/delivery with its tests (1585/T2, T12). +func TestTakeInEvents(t *testing.T) { + // The take-in events, `take-in..`, a group's key with its dots and no subject wildcard. + if PendingKey(42) != "m.00000000000000000042" || GroupTakeInKey("feat/a.b *>") != "g.feat/a.b___" { + t.Errorf("take-in keys %q %q", PendingKey(42), GroupTakeInKey("feat/a.b *>")) + } + for _, c := range []struct{ kind, key string }{{TakeInPending, PendingKey(7)}, {TakeInTaken, GroupTakeInKey("feat/a.b")}} { + ev, err := TakeInEvent(c.kind, c.key) + if err != nil || ev != "take-in."+c.kind+"."+c.key { + t.Fatalf("take-in event = %q, %v", ev, err) + } + if k, key, err := ParseTakeInEvent(ev); err != nil || k != c.kind || key != c.key { + t.Errorf("parse take-in event %q = %q %q %v", ev, k, key, err) + } + } + for _, bad := range []string{"", "take-in.pending", "take-in.held.m.1", "take-in.taken.x.1", "take-in.taken.m..1", "forge.taken.m.1"} { + if _, _, err := ParseTakeInEvent(bad); err == nil { + t.Errorf("parse take-in event %q accepted", bad) + } + } + // The bodies round-trip: a pending revision, a group being gathered, and a hand take-in's taken with its by. + since := time.Date(2026, 11, 4, 23, 0, 0, 0, time.UTC) + for _, ti := range sampleTakeIns(since) { + data, err := EncodeTakeIn(ti) + if err != nil { + t.Fatalf("take-in %s %s: %v", ti.Kind, ti.What, err) + } + ti.V = delivery.Version + if got, err := DecodeTakeIn(data); err != nil || !SameTakeIn(got, ti) { + t.Errorf("take-in %s round trip: %+v %v", ti.Key, got, err) + } + } + for _, in := range []string{`{"key":"m.1"}`, `{"v":2,"key":"m.1"}`} { + if _, err := DecodeTakeIn([]byte(in)); !errors.Is(err, delivery.ErrUnknownVersion) { + t.Errorf("decode of %s = %v, want delivery.ErrUnknownVersion", in, err) + } + } + // A body that does not hold is refused, naming why. + for _, c := range []struct { + name, want string + i int + mut func(*TakeIn) + }{ + {"group take-in without its group", "names its group", 1, func(t *TakeIn) { t.Group = "" }}, + {"group take-in under another group's key", "g.other", 1, func(t *TakeIn) { t.Key = GroupTakeInKey("other") }}, + {"pending revision waiting for no build", "waits for", 0, func(t *TakeIn) { t.WaitsFor = "" }}, + {"pending with a hand act", "only a taken event", 0, func(t *TakeIn) { t.By = "someone@node" }}, + {"hand act without its by", "by", 2, func(t *TakeIn) { t.By = "" }}, + {"take-in without merges", "no merge", 0, func(t *TakeIn) { t.Merges = nil }}, + {"take-in without since", "since", 0, func(t *TakeIn) { t.Since = time.Time{} }}, + {"a merge of no repository", "owner", 0, func(t *TakeIn) { t.Merges = []Merge{{Repository: "nope", Merge: "m"}} }}, + } { + ti := sampleTakeIns(since)[c.i] + c.mut(&ti) + if _, err := EncodeTakeIn(ti); err == nil || !strings.Contains(err.Error(), c.want) { + t.Errorf("%s: %v, want a refusal naming %q", c.name, err, c.want) + } + } + // indexer_pending's answer reads back; a wait of neither kind is refused. + pa := PendingAnswer{Waits: []PendingWait{{Key: PendingKey(12), What: TakeInRevision, Merges: sampleTakeIns(since)[0].Merges, + Keys: []string{PendingKey(12)}, WaitsFor: "src2:abc", Since: since, Holds: []string{"mesh-delivery-indexer"}, Said: "take-in.pending." + PendingKey(12), SaidAt: since}}} + if data, err := json.Marshal(pa); err != nil { + t.Fatal(err) + } else if got, err := DecodePendingAnswer(data); err != nil || !reflect.DeepEqual(got, pa) { + t.Errorf("pending answer round trip: %+v %v", got, err) + } + if _, err := DecodePendingAnswer([]byte(`{"waits":[{"key":"m.1","what":"held"}]}`)); err == nil { + t.Error("a pending answer with a wait of neither kind accepted") + } +} diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go index 834e8c39..bbe5f2bd 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go @@ -12,6 +12,7 @@ import ( "mesh-delivery-view/internal/view" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-delivery/sdk/graph" @@ -340,10 +341,10 @@ func main() { } // pendingOf asks the indexer's tool indexer_pending (declared in invokes): what it holds back and why (ADR 0633 §5). -func pendingOf() (delivery.PendingAnswer, error) { +func pendingOf() (takein.PendingAnswer, error) { raw, err := stdio.Ask(delivery.IndexerModule+".indexer_pending", map[string]any{}) if err != nil { - return delivery.PendingAnswer{}, err + return takein.PendingAnswer{}, err } - return delivery.DecodePendingAnswer(raw) + return takein.DecodePendingAnswer(raw) } diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/tiers_rows_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/tiers_rows_test.go index c9f8bba5..baa4406a 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/tiers_rows_test.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/tiers_rows_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-delivery/sdk/graph" @@ -106,10 +107,10 @@ func tiersToolSaysEachModulesWait(t *testing.T) { // wait's since; a held module the revision does not move is named in the first tier with that wait alone, a group's wait // says its group, and a tool that cannot be asked is said and names no module. heldSince := time.Date(2026, 11, 4, 22, 0, 0, 0, time.UTC) - r.Pending = func() (delivery.PendingAnswer, error) { - return delivery.PendingAnswer{Waits: []delivery.PendingWait{ - {Key: "m.00000000000000000042", What: delivery.TakeInRevision, WaitsFor: "src2:abc", Since: heldSince, Holds: []string{"web", "newmod"}}, - {Key: "g.feat", What: delivery.TakeInGroup, Group: "feat", Since: heldSince.Add(time.Hour), Holds: []string{"lib"}}}}, nil + r.Pending = func() (takein.PendingAnswer, error) { + return takein.PendingAnswer{Waits: []takein.PendingWait{ + {Key: "m.00000000000000000042", What: takein.TakeInRevision, WaitsFor: "src2:abc", Since: heldSince, Holds: []string{"web", "newmod"}}, + {Key: "g.feat", What: takein.TakeInGroup, Group: "feat", Since: heldSince.Add(time.Hour), Holds: []string{"lib"}}}}, nil } out, err = run(t, r, view.TiersTool, map[string]any{"revision": float64(9)}) held, _ := out.(view.Tiers) @@ -126,7 +127,7 @@ func tiersToolSaysEachModulesWait(t *testing.T) { if nm := held.Tiers[0][len(held.Tiers[0])-1]; nm.Module != "newmod" || len(nm.Waits) != 1 || nm.Waits[0].Edge != view.TakeInEdge { t.Errorf("the first tier = %+v, want newmod, which the revision does not move, named with its take-in wait alone", held.Tiers[0]) } - r.Pending = func() (delivery.PendingAnswer, error) { return delivery.PendingAnswer{}, errors.New("no answer") } + r.Pending = func() (takein.PendingAnswer, error) { return takein.PendingAnswer{}, errors.New("no answer") } if out, err = run(t, r, view.TiersTool, map[string]any{"revision": float64(9)}); err != nil || !strings.Contains(out.(view.Tiers).Pending, "no answer") || strings.Contains(fmt.Sprint(out.(view.Tiers).Tiers), view.TakeInEdge) { t.Errorf("view_tiers with no answer of indexer_pending = %+v, %v", out, err) diff --git a/modules/mesh-delivery-view/go.mod b/modules/mesh-delivery-view/go.mod index 035f39f7..45992491 100644 --- a/modules/mesh-delivery-view/go.mod +++ b/modules/mesh-delivery-view/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-view go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) @@ -21,6 +21,11 @@ require ( golang.org/x/time v0.15.0 // indirect ) -require git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000 +require ( + git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000 +) replace git.novox.be/novox/mesh-delivery/sdk => ../../sdk + +replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein => ../mesh-delivery-indexer/takein diff --git a/modules/mesh-delivery-view/go.sum b/modules/mesh-delivery-view/go.sum index 7679a5dc..1c3af991 100644 --- a/modules/mesh-delivery-view/go.sum +++ b/modules/mesh-delivery-view/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659 h1:tK7MzxVm/ECNg1Yblhh1ynVEthbZCg2mSPW0wQZC0b0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-view/internal/view/read.go b/modules/mesh-delivery-view/internal/view/read.go index 7e7afa6a..3cf91505 100644 --- a/modules/mesh-delivery-view/internal/view/read.go +++ b/modules/mesh-delivery-view/internal/view/read.go @@ -14,6 +14,7 @@ import ( "sync/atomic" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -59,7 +60,7 @@ type Reader struct { // ProvisionsState the provisioner's keys: what view_tiers reads (tiers.go). Unset reads none, said in the answer. GraphState, SourceState, ProvisionsState StateRead // Pending reads the indexer's waits through its tool indexer_pending (ADR 0633 §5); unset names no module held back. - Pending func() (delivery.PendingAnswer, error) + Pending func() (takein.PendingAnswer, error) // BuildLog reads the builder's log from a time (stdio.Log(delivery.BuildLogRead).Since); unset reads none. BuildLog LogSince } diff --git a/modules/mesh-delivery-view/internal/view/tiers.go b/modules/mesh-delivery-view/internal/view/tiers.go index 59d05bf1..0d09e4f1 100644 --- a/modules/mesh-delivery-view/internal/view/tiers.go +++ b/modules/mesh-delivery-view/internal/view/tiers.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-delivery/sdk/graph" @@ -227,7 +228,7 @@ func (r Reader) heldBack(out *Tiers) (map[string]TierWait, []string) { } for _, w := range a.Waits { words := "the indexer holds back " + w.Key + ", which waits for the indexer build " + w.WaitsFor - if w.What == delivery.TakeInGroup { + if w.What == takein.TakeInGroup { words = "the indexer gathers the group " + w.Group + " and its end has not come" } else if w.Behind != "" { words = "the indexer holds back " + w.Key + " behind " + w.Behind diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go index 239ad6e4..9e280190 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go @@ -22,6 +22,7 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/provisions" @@ -379,10 +380,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The indexer's stream take-in (ADR 0633 §5): a revision or a group it holds back is a wait, ended by take-in.taken. The // durable consumer _ is made by the controller from this module's consumes; the whole stream is heard at // the start, so a wait that began while the watchman was away is read with its own since. - switch tc, err := cred.EventConsumer(ctx, js, catalogue.StreamBusName(delivery.TakeInStream)); { + switch tc, err := cred.EventConsumer(ctx, js, catalogue.StreamBusName(takein.TakeInStream)); { case err != nil: logger.Printf("the consumer %s_%s on %s does not exist yet, so no take-in wait is said until the watchman starts again: %v", - cred.Node, cred.Module, catalogue.StreamBusName(delivery.TakeInStream), err) + cred.Node, cred.Module, catalogue.StreamBusName(takein.TakeInStream), err) default: // The replay has caught up when a message says none is pending after it, or when the consumer had none pending at its // bind (the stream holds nothing it has not acknowledged): the waits past their bound are said then, not before. diff --git a/modules/mesh-delivery-watchman/go.mod b/modules/mesh-delivery-watchman/go.mod index 33a7e8dc..2ce3eef0 100644 --- a/modules/mesh-delivery-watchman/go.mod +++ b/modules/mesh-delivery-watchman/go.mod @@ -3,12 +3,13 @@ module mesh-delivery-watchman go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104221203-10c91fba4a28 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) require ( + git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein v0.0.0-00010101000000-000000000000 github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect github.com/google/go-tpm v0.9.8 // indirect github.com/klauspost/compress v1.20.1 // indirect @@ -22,3 +23,5 @@ require ( ) replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun => ../mesh-delivery-builder/buildrun + +replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein => ../mesh-delivery-indexer/takein diff --git a/modules/mesh-delivery-watchman/go.sum b/modules/mesh-delivery-watchman/go.sum index 5928f081..711a3687 100644 --- a/modules/mesh-delivery-watchman/go.sum +++ b/modules/mesh-delivery-watchman/go.sum @@ -1,7 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104221203-10c91fba4a28 h1:Zoi+myMvKaRYdM8w/96W6GnoAiTvgwvEF1QPA6HelaU= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104221203-10c91fba4a28/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659 h1:tK7MzxVm/ECNg1Yblhh1ynVEthbZCg2mSPW0wQZC0b0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-watchman/internal/watch/findings_test.go b/modules/mesh-delivery-watchman/internal/watch/findings_test.go index 0454d76f..8f81f1a1 100644 --- a/modules/mesh-delivery-watchman/internal/watch/findings_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/findings_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/provisions" ) @@ -36,11 +37,11 @@ func TestTheWatchmansFindings(t *testing.T) { }{ {"ADR 0633 §5 (1585/T21): a revision waiting for an indexer build is a finding past 60 minutes from its since, not within; a group past 30", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { w.TakeInCaughtUp() // a live message, heard after the replay - w.TakeIn(takeInEvent(t, delivery.TakeInPending, revKey), takeInBody(t, delivery.TakeInPending, revKey, t0)) + w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), takeInBody(t, takein.TakeInPending, revKey, t0)) if o.ttls[signalKey("take-in", revKey)] != TakeInRevisionBound || len(o.findings) != 0 { t.Fatalf("a revision's wait: ttls %v, findings %v", o.ttls, o.findings) } - w.TakeIn(takeInEvent(t, delivery.TakeInPending, groupKey), takeInBody(t, delivery.TakeInPending, groupKey, t0)) + w.TakeIn(takeInEvent(t, takein.TakeInPending, groupKey), takeInBody(t, takein.TakeInPending, groupKey, t0)) if o.ttls[signalKey("take-in", groupKey)] != TakeInGroupBound { t.Fatalf("a group's wait: ttls %v", o.ttls) } @@ -53,41 +54,41 @@ func TestTheWatchmansFindings(t *testing.T) { {"ADR 0633 §5 (1585/T21): the take-in.taken of a named indexer, a hand take-in or a drop clears the finding", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { w.TakeInCaughtUp() // a live message, heard after the replay for _, by := range []string{"", "operator@shanks"} { - w.TakeIn(takeInEvent(t, delivery.TakeInPending, revKey), takeInBody(t, delivery.TakeInPending, revKey, t0)) + w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), takeInBody(t, takein.TakeInPending, revKey, t0)) expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound) if len(o.findings) != 1 { t.Fatalf("findings past the bound: %v", o.findings) } - taken, err := delivery.EncodeTakeIn(delivery.TakeIn{Kind: delivery.TakeInTaken, Key: revKey, What: delivery.TakeInRevision, Since: t0, - Merges: []delivery.GroupMerge{{Repository: "novox/a", Merge: "c0ffee"}}, By: by, Why: map[bool]string{true: "dropped", false: ""}[by != ""], Dropped: by != ""}) + taken, err := takein.EncodeTakeIn(takein.TakeIn{Kind: takein.TakeInTaken, Key: revKey, What: takein.TakeInRevision, Since: t0, + Merges: []takein.Merge{{Repository: "novox/a", Merge: "c0ffee"}}, By: by, Why: map[bool]string{true: "dropped", false: ""}[by != ""], Dropped: by != ""}) if err != nil { t.Fatal(err) } - w.TakeIn(takeInEvent(t, delivery.TakeInTaken, revKey), taken) + w.TakeIn(takeInEvent(t, takein.TakeInTaken, revKey), taken) c.t = t0 } }, map[string]string{}}, {"ADR 0633 §5 (1585/T21): a group's take-in.taken ends its 30 minutes, and the revision then pending under m. starts its own 60", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { w.TakeInCaughtUp() // a live message, heard after the replay - w.TakeIn(takeInEvent(t, delivery.TakeInPending, groupKey), takeInBody(t, delivery.TakeInPending, groupKey, t0)) + w.TakeIn(takeInEvent(t, takein.TakeInPending, groupKey), takeInBody(t, takein.TakeInPending, groupKey, t0)) expire(w, c, signalKey("take-in", groupKey), 25*time.Minute) - w.TakeIn(takeInEvent(t, delivery.TakeInTaken, groupKey), takeInBody(t, delivery.TakeInTaken, groupKey, t0)) - w.TakeIn(takeInEvent(t, delivery.TakeInPending, revKey), takeInBody(t, delivery.TakeInPending, revKey, c.t)) + w.TakeIn(takeInEvent(t, takein.TakeInTaken, groupKey), takeInBody(t, takein.TakeInTaken, groupKey, t0)) + w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), takeInBody(t, takein.TakeInPending, revKey, c.t)) if len(o.findings) != 0 || o.ttls[signalKey("take-in", revKey)] != TakeInRevisionBound { t.Fatalf("after the hand-over: findings %v, ttls %v", o.findings, o.ttls) } expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound) }, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}}, {"ADR 0633 §5 (1585/T21): a pending event heard again at the start, since past its bound, is said once the replay has caught up, not before", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { - w.TakeIn(takeInEvent(t, delivery.TakeInPending, revKey), takeInBody(t, delivery.TakeInPending, revKey, t0.Add(-2*time.Hour))) + w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), takeInBody(t, takein.TakeInPending, revKey, t0.Add(-2*time.Hour))) if len(o.findings) != 0 || len(o.pulses) != 0 { t.Fatalf("a replayed old wait before the replay caught up: findings %v, pulses %v", o.findings, o.pulses) } w.TakeInCaughtUp() }, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}}, {"ADR 0633 §5 (1585/T21): a replayed pending followed by its taken never raises", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { - w.TakeIn(takeInEvent(t, delivery.TakeInPending, revKey), takeInBody(t, delivery.TakeInPending, revKey, t0.Add(-2*time.Hour))) - w.TakeIn(takeInEvent(t, delivery.TakeInTaken, revKey), takeInBody(t, delivery.TakeInTaken, revKey, t0.Add(-2*time.Hour))) + w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), takeInBody(t, takein.TakeInPending, revKey, t0.Add(-2*time.Hour))) + w.TakeIn(takeInEvent(t, takein.TakeInTaken, revKey), takeInBody(t, takein.TakeInTaken, revKey, t0.Add(-2*time.Hour))) w.TakeInCaughtUp() if o.findingWrites != 0 || len(o.pulses) != 0 { t.Fatalf("a wait that ended in the replay wrote %d findings and left pulses %v", o.findingWrites, o.pulses) @@ -95,17 +96,17 @@ func TestTheWatchmansFindings(t *testing.T) { }, map[string]string{}}, {"ADR 0633 §5 (1585/T21): a drop clears the wait, and the merges left under it said pending again with the same since are raised from it", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { w.TakeInCaughtUp() // a live message, heard after the replay - w.TakeIn(takeInEvent(t, delivery.TakeInPending, revKey), takeInBody(t, delivery.TakeInPending, revKey, t0)) + w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), takeInBody(t, takein.TakeInPending, revKey, t0)) expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound) if len(o.findings) != 1 { t.Fatalf("findings past the bound: %v", o.findings) } - w.TakeIn(takeInEvent(t, delivery.TakeInTaken, revKey), takeInBody(t, delivery.TakeInTaken, revKey, t0)) + w.TakeIn(takeInEvent(t, takein.TakeInTaken, revKey), takeInBody(t, takein.TakeInTaken, revKey, t0)) if len(o.findings) != 0 { t.Fatalf("the drop left %v", o.findings) } next := "m.00000000000000000043" - w.TakeIn(takeInEvent(t, delivery.TakeInPending, next), takeInBody(t, delivery.TakeInPending, next, t0)) + w.TakeIn(takeInEvent(t, takein.TakeInPending, next), takeInBody(t, takein.TakeInPending, next, t0)) }, map[string]string{FindingKey(ScopeDelivery, "m.00000000000000000043", KindTakeInWaits): Warning}}, {"ADR 0578 §1: a module whose value carries a refusal of its manifest is a finding until it is accepted", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { value := func(verdict string, problems ...string) []byte { @@ -609,7 +610,7 @@ const ( func takeInEvent(t *testing.T, kind, key string) string { t.Helper() - e, err := delivery.TakeInEvent(kind, key) + e, err := takein.TakeInEvent(kind, key) if err != nil { t.Fatal(err) } @@ -619,14 +620,14 @@ func takeInEvent(t *testing.T, kind, key string) string { // takeInBody is a take-in event's body for a key: a group's when it starts with g. func takeInBody(t *testing.T, kind, key string, since time.Time) []byte { t.Helper() - body := delivery.TakeIn{Kind: kind, Key: key, What: delivery.TakeInRevision, Since: since, - Merges: []delivery.GroupMerge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}} + body := takein.TakeIn{Kind: kind, Key: key, What: takein.TakeInRevision, Since: since, + Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}} if key[0] == 'g' { - body.What, body.Group = delivery.TakeInGroup, "feat/take-in" - } else if kind == delivery.TakeInPending { + body.What, body.Group = takein.TakeInGroup, "feat/take-in" + } else if kind == takein.TakeInPending { body.WaitsFor = "src2:abc" } - b, err := delivery.EncodeTakeIn(body) + b, err := takein.EncodeTakeIn(body) if err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-watchman/internal/watch/takein.go b/modules/mesh-delivery-watchman/internal/watch/takein.go index 9cff07b2..4d3e4276 100644 --- a/modules/mesh-delivery-watchman/internal/watch/takein.go +++ b/modules/mesh-delivery-watchman/internal/watch/takein.go @@ -7,7 +7,7 @@ import ( "strings" "time" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" ) // KindTakeInWaits is a source revision or a delivery group the indexer holds back and has not taken in within its bound @@ -28,12 +28,12 @@ const takeInRow = "take-in" // a pending one starts the wait for its key, whose bound runs from the event's since, and a taken one ends it. A group's // taken event ends the group's wait only: the revision it becomes is its own key and starts its own, longer wait. func (w *Watchman) TakeIn(event string, body []byte) { - kind, key, err := delivery.ParseTakeInEvent(event) + kind, key, err := takein.ParseTakeInEvent(event) if err != nil { w.say("a take-in event is left: %v", err) return } - t, err := delivery.DecodeTakeIn(body) + t, err := takein.DecodeTakeIn(body) if err != nil { w.say("the body of %s cannot be read, left: %v", event, err) return @@ -41,14 +41,14 @@ func (w *Watchman) TakeIn(event string, body []byte) { w.mu.Lock() defer w.mu.Unlock() finding := FindingKey(ScopeDelivery, key, KindTakeInWaits) - if kind == delivery.TakeInTaken { + if kind == takein.TakeInTaken { delete(w.takeInHeld, key) w.unsignal(takeInRow, key) w.unfind(finding) return } bound, what := TakeInRevisionBound, "a source revision waits for the indexer build "+t.WaitsFor - if t.What == delivery.TakeInGroup { + if t.What == takein.TakeInGroup { bound, what = TakeInGroupBound, "a delivery group is gathered and its end has not come" } var merges []string From 7b7096f37ea76e19b5b6d98e43d5a637a6d4d006 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 5 Nov 2026 00:28:56 +0100 Subject: [PATCH 17/18] Take-in, review round 1: no claim left behind, no sleep at the start, waits seeded at a restart, ends asked readable (issue 1585, ADR 0633) - A take-in releases its claims (compare-and-set, only while the claim is still its own) on a lost claim, on a claim it cannot read back, and on a merge it fails to take in: a claim left behind refused every later take-in until a restart. T5 row with a store that fails once, both ways. - The start step waits for nothing: with the build it runs unknown it says so and the forge consumer starts; a merged message left unacknowledged asks the build again (clause 1). T10 row: the start answers at once and takes nothing in. - A toolchain report evaluates pending again (ToolchainReported), since the build a revision names moves with the toolchain. T7 row: the revision is taken in once the report makes it name the running build. - The group's end is told by its whole subject (process.ModuleStreamSubject), so a repository named forge/group is never taken for one; row in the indexer's handle test. - The watchman asks the indexer's waits through indexer_pending at its start (invokes), before its durable consumer, which does not hear again what it acknowledged before a restart; the comment that said the whole stream is heard is corrected. W2 row: a wait seeded within its bound is raised past it. - bridge_groups names each group's last end ask, its who, why and when, and the bridge's answer (state end-asks). - The check's words for the indexer's rules: the trunk's manifest validator. - The bridge's end drops the merged-state test its merge commit already implies. - Test budget: groupask_test folded into groups_test, and the K1 mesh/check test into the owed status one: 804 unit tests (was 806). Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn --- .../internal/check/examiner_test.go | 16 +-- .../internal/check/manifests_judge.go | 25 ++--- .../mesh-delivery-forge-bridge-tools/main.go | 26 +++-- .../main_test.go | 13 ++- .../groupask_test.go | 64 ------------ .../cmd/mesh-delivery-forge-bridge/groups.go | 4 +- .../mesh-delivery-forge-bridge/groups_test.go | 46 ++++++++- .../internal/bridge/acts_test.go | 32 +++--- .../internal/groups/groups.go | 19 +++- .../cmd/mesh-delivery-indexer/main.go | 26 ++--- .../cmd/mesh-delivery-indexer/main_test.go | 7 ++ .../internal/index/index_test.go | 11 +++ .../internal/index/pending.go | 84 +++++++++++++--- .../internal/index/pending_test.go | 98 ++++++++++++++++++- .../internal/index/toolchains.go | 14 ++- .../cmd/mesh-delivery-watchman/main.go | 30 +++++- .../internal/watch/findings_test.go | 12 +++ .../internal/watch/takein.go | 19 ++++ .../mesh-delivery-watchman/manifest_test.go | 5 + modules/mesh-delivery-watchman/module.json | 3 + 20 files changed, 402 insertions(+), 152 deletions(-) delete mode 100644 modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groupask_test.go diff --git a/modules/mesh-delivery-checker/internal/check/examiner_test.go b/modules/mesh-delivery-checker/internal/check/examiner_test.go index fca34153..5cf259ff 100644 --- a/modules/mesh-delivery-checker/internal/check/examiner_test.go +++ b/modules/mesh-delivery-checker/internal/check/examiner_test.go @@ -1035,18 +1035,18 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) { // mail's tool while mail awaits web's database: both left out, so neither is refused (ADR 0614 §3, §6). cycle string trunkCycle bool - // indexer makes web the holder of the seat mesh-catalog, and the trunk's judge answers for it: "held" with no verdict, + // indexer makes web the holder of the seat mesh-catalog, and the trunk's manifest validator answers for it: "held" with no verdict, // "aside" with a field it does not know; ownJudge has the group change the judge, so the manifest is also judged by the // trunk's, in a second ask (ADR 0633 clause 6, 1585/T22, T23) indexer string ownJudge bool }{ {name: "accepted"}, - {name: "the indexer accepted by the trunk's judge", indexer: "ok"}, - {name: "the indexer held by the trunk's judge", indexer: "held", fails: "the indexer web, the holder of the seat mesh-catalog, would never run: the trunk's judge at mesh-delivery's trunk " + strings.Repeat("d", 40) + " does not accept its manifest: the judge holds it, with no verdict yet"}, - {name: "the indexer with a field the trunk's judge does not know", indexer: "aside", fails: "the judge does not know its fields new_field, so it waits"}, - {name: "the indexer refused by the trunk's judge", indexer: "refused", refused: "web has no id", fails: "does not accept its manifest: web has no id"}, - {name: "the indexer under a changed judge is judged by the trunk's as well", indexer: "aside", ownJudge: true, fails: "the judge does not know its fields new_field, so it waits"}, + {name: "the indexer accepted by the trunk's manifest validator", indexer: "ok"}, + {name: "the indexer held by the trunk's manifest validator", indexer: "held", fails: "the indexer web, the holder of the seat mesh-catalog, would never run: the trunk's manifest validator at mesh-delivery's trunk " + strings.Repeat("d", 40) + " does not accept its manifest: the manifest validator holds it, with no verdict yet"}, + {name: "the indexer with a field the trunk's manifest validator does not know", indexer: "aside", fails: "the manifest validator does not know its fields new_field, so it waits"}, + {name: "the indexer refused by the trunk's manifest validator", indexer: "refused", refused: "web has no id", fails: "does not accept its manifest: web has no id"}, + {name: "the indexer under a changed judge is judged by the trunk's as well", indexer: "aside", ownJudge: true, fails: "the manifest validator does not know its fields new_field, so it waits"}, {name: "a touched manifest the judge holds", indexer: "heldplain", fails: "holds the manifest of web: it gave no verdict"}, {name: "an untouched manifest's problem the trunk had", untouched: "mail says no credential form", trunkMail: "mail says no credential form"}, {name: "a problem the change causes in an untouched manifest", untouched: "mail reads a state web no longer keeps", @@ -1122,9 +1122,9 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) { cs := caseOf(t, p) v := b.judge(t, cs).Verdict if row.indexer == "ok" && !slices.ContainsFunc(cs.notes, func(n string) bool { - return strings.Contains(n, "the new indexer's manifest of web is accepted by the trunk's judge") + return strings.Contains(n, "the new indexer's manifest of web is accepted by the trunk's manifest validator") }) { - t.Errorf("%s: the case says %v, want the new indexer's manifest said accepted by the trunk's judge", row.name, cs.notes) + t.Errorf("%s: the case says %v, want the new indexer's manifest said accepted by the trunk's manifest validator", row.name, cs.notes) } if row.ownJudge && (len(judged) != 2 || judged[0].Fetch["commit"] != head1 || judged[1].Fetch["commit"] != trunk) { t.Fatalf("%s: judged %+v, want the head's judge and then the trunk's", row.name, judged) diff --git a/modules/mesh-delivery-checker/internal/check/manifests_judge.go b/modules/mesh-delivery-checker/internal/check/manifests_judge.go index c8320df3..c34e8308 100644 --- a/modules/mesh-delivery-checker/internal/check/manifests_judge.go +++ b/modules/mesh-delivery-checker/internal/check/manifests_judge.go @@ -206,12 +206,13 @@ func (st *state) runJudge(ctx context.Context, req composer.Request, touched []s // never by its name (novox/hq ADR 0633 clause 6, ADR 0601 §4). const CatalogSeat = "mesh-catalog" -// refuseUnmovableIndexer fails the check when the trunk's judge, the one the running parts hold, would refuse or hold the -// change's new indexer manifest: such an indexer would never run, and every revision behind it would wait for ever -// (ADR 0633 clause 6). The indexer is the touched module whose manifest claims the seat mesh-catalog. When the change -// judges with its own head's judge (it changes the judge), the manifest is judged by the trunk's as well. The verdicts -// read are the trunk's judge's: its change case, where refused, held (not accepted with no reason) or with a field set -// aside (a field the running judge does not know waits, ADR 0601 §3) each fail it, naming the module and the problems. +// refuseUnmovableIndexer fails the check when the trunk's manifest validator, the one the running parts hold, would +// refuse or hold the change's new indexer manifest: such an indexer would never run, and every revision behind it would +// wait for ever (ADR 0633 clause 6). The indexer is the touched module whose manifest claims the seat mesh-catalog. +// When the change judges with its own head's manifest validator (it changes it), the manifest is judged by the trunk's +// as well. The verdicts read are the trunk's manifest validator's: its change case, where refused, held (not accepted +// with no reason) or with a field set aside (a field the running manifest validator does not know waits, ADR 0601 §3) +// each fail it, naming the module and the problems. func (st *state) refuseUnmovableIndexer(ctx context.Context, req composer.Request, touched []string, cv composer.CaseVerdict, judgedAt string) *delivery.Verdict { if st.j.Read == nil { return nil @@ -237,7 +238,7 @@ func (st *state) refuseUnmovableIndexer(ctx context.Context, req composer.Reques trunk = st.j.Trunks.Heads()[DeliveryRepository] } if trunk == "" { - return fail(goModule, "the new indexer cannot be judged by the trunk's judge: the trunk head of "+DeliveryRepository+" is not known") + return fail(goModule, "the new indexer cannot be judged by the trunk's manifest validator: the trunk head of "+DeliveryRepository+" is not known") } var v *delivery.Verdict judgedAt = "mesh-delivery's trunk " + trunk @@ -249,19 +250,19 @@ func (st *state) refuseUnmovableIndexer(ctx context.Context, req composer.Reques var problems []string switch { case !ok: - problems = append(problems, "the judge gave no verdict") + problems = append(problems, "the manifest validator gave no verdict") case r.Refused != "": problems = append(problems, strings.Split(r.Refused, "\n")...) case !r.Accepted: - problems = append(problems, "the judge holds it, with no verdict yet") + problems = append(problems, "the manifest validator holds it, with no verdict yet") case r.SetAside != "": - problems = append(problems, "the judge does not know its fields "+r.SetAside+", so it waits") + problems = append(problems, "the manifest validator does not know its fields "+r.SetAside+", so it waits") } if len(problems) > 0 { - return fail(goModule, fmt.Sprintf("the indexer %s, the holder of the seat %s, would never run: the trunk's judge at %s does not accept its manifest: %s", + return fail(goModule, fmt.Sprintf("the indexer %s, the holder of the seat %s, would never run: the trunk's manifest validator at %s does not accept its manifest: %s", indexer, CatalogSeat, judgedAt, strings.Join(problems, "; "))) } - st.c.note(fmt.Sprintf("the new indexer's manifest of %s is accepted by the trunk's judge at %s", indexer, judgedAt)) + st.c.note(fmt.Sprintf("the new indexer's manifest of %s is accepted by the trunk's manifest validator at %s", indexer, judgedAt)) return nil } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go index f3638d1e..86faa05a 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go @@ -37,7 +37,8 @@ func Tools(ask asker, now func() time.Time, groupsRead func() ([]groups.Row, []s Description: "The delivery groups the gate holds (more than one member), each with its members' merge state and whether each " + "merged message is written on the forge stream, and whether the group's end (forge.group.merged) is written: ended, " + "or open with why (gathered, or merged and its end not written yet). An open group with every member merged is " + - "what bridge_group_end_again says the end of. A group's name narrows it to that group; open lists only open ones.", + "what bridge_group_end_again says the end of. Each group names its last end ask (who, why, when) and the bridge's " + + "answer to it, from the state end-asks. A group's name narrows it to that group; open lists only open ones.", Input: map[string]any{"type": "object", "properties": map[string]any{ "group": map[string]any{"type": "string", "description": "only this group, by the name the gate keys it"}, "open": map[string]any{"type": "boolean", "description": "only the groups whose end is not written"}}}, @@ -80,8 +81,9 @@ func Tools(ask asker, now func() time.Time, groupsRead func() ([]groups.Row, []s } } -// read is the groups the gate holds with what the bridge kept of them. -func read(merges, performed stdio.KeptState) ([]groups.Row, []string, error) { +// read is the groups the gate holds with what the bridge kept of them: their ends in performed, and the last ask of +// each in end-asks with the bridge's answer. +func read(merges, performed, asks stdio.KeptState) ([]groups.Row, []string, error) { keys, err := merges.Keys() if err != nil { return nil, nil, fmt.Errorf("the gate's merges cannot be listed: %w", err) @@ -110,7 +112,17 @@ func read(merges, performed stdio.KeptState) ([]groups.Row, []string, error) { } } } - rows, unread := groups.Rows(raws, published, ended) + akeys, err := asks.Keys() + if err != nil { + return nil, nil, fmt.Errorf("the bridge's state %s cannot be listed: %w", groups.EndAsksState, err) + } + askRaws := map[string][]byte{} + for _, k := range akeys { + if e, err := asks.Get(k); err == nil && e != nil { + askRaws[k] = e.Value + } + } + rows, unread := groups.Rows(raws, published, ended, askRaws) return rows, unread, nil } @@ -164,9 +176,9 @@ func askEnd(s stdio.KeptState, bound time.Duration) asker { } func main() { - merges, performed := stdio.State(delivery.GateModule+".merges"), stdio.State(groups.PerformedState) - tools := Tools(askEnd(stdio.State(groups.EndAsksState), 25*time.Second), time.Now, - func() ([]groups.Row, []string, error) { return read(merges, performed) }) + merges, performed, asks := stdio.State(delivery.GateModule+".merges"), stdio.State(groups.PerformedState), stdio.State(groups.EndAsksState) + tools := Tools(askEnd(asks, 25*time.Second), time.Now, + func() ([]groups.Row, []string, error) { return read(merges, performed, asks) }) if err := stdio.Serve("", tools); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-forge-bridge-tools] %v\n", err) os.Exit(1) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go index cdfe4250..2d1b9e6f 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go @@ -34,6 +34,10 @@ func TestTheBridgeToolsReadTheGroupsAndAskTheEndAgain(t *testing.T) { for _, c := range []string{c1, c2} { performed[groups.PublishedKey("novox/a", 1, c)], performed[groups.PublishedKey("novox/b", 2, c)] = true, true } + // The state end-asks keeps the last ask of the group ready, answered, and one that does not read. + answeredAsk, _ := json.Marshal(groups.EndAsk{V: 1, Group: "ready", Why: "lost", By: "lead@shanks", At: time.Date(2026, 11, 4, 22, 0, 0, 0, time.UTC), + Written: true, Answer: "the end of the group is written again", Answered: time.Date(2026, 11, 4, 22, 0, 1, 0, time.UTC)}) + asks := map[string][]byte{"ready": answeredAsk, "gathered": []byte("{")} var asked groups.EndAsk var askedKey string ask := func(key string, value []byte) (groups.EndAsk, error) { @@ -45,7 +49,7 @@ func TestTheBridgeToolsReadTheGroupsAndAskTheEndAgain(t *testing.T) { } tools := Tools(ask, func() time.Time { return time.Date(2026, 11, 4, 23, 0, 0, 0, time.UTC) }, func() ([]groups.Row, []string, error) { - rows, unread := groups.Rows(merges, performed, map[string]uint64{"ended": 7}) + rows, unread := groups.Rows(merges, performed, map[string]uint64{"ended": 7}, asks) return rows, unread, nil }) call := func(name string, args map[string]any) any { @@ -74,8 +78,8 @@ func TestTheBridgeToolsReadTheGroupsAndAskTheEndAgain(t *testing.T) { for _, r := range out.Groups { m[r.Group] = r } - if len(args) == 0 && (len(out.Unread) != 1 || out.Unread[0] != "bad") { - t.Errorf("unread = %v, want bad", out.Unread) + if len(args) == 0 && strings.Join(out.Unread, " ") != "bad end-asks.gathered" { + t.Errorf("unread = %v, want bad and end-asks.gathered", out.Unread) } return m } @@ -86,6 +90,9 @@ func TestTheBridgeToolsReadTheGroupsAndAskTheEndAgain(t *testing.T) { if r := all["ready"]; !r.Open || !strings.Contains(r.Words, "bridge_group_end_again") || !r.Members[1].Published { t.Errorf("ready = %+v, want open, merged and published, its end the thing to say", r) } + if a := all["ready"].LastEndAsk; a == nil || a.By != "lead@shanks" || a.Why != "lost" || !a.Written || a.Answer == "" || all["ended"].LastEndAsk != nil { + t.Errorf("ready's last end ask = %+v, ended's %+v: want ready's ask and its answer, none for ended", a, all["ended"].LastEndAsk) + } if r := all["gathered"]; !r.Open || !strings.Contains(r.Words, "gathered") || r.Members[1].State != string(delivery.MergeMerging) { t.Errorf("gathered = %+v", r) } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groupask_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groupask_test.go deleted file mode 100644 index de6ded88..00000000 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groupask_test.go +++ /dev/null @@ -1,64 +0,0 @@ -package main - -import ( - "bytes" - "context" - "log" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/delivery" - - "mesh-delivery-forge-bridge/internal/groups" -) - -// 1585/T19 (ADR 0633 §4): an ask to say a group's end again is answered by the bridge from the gate's merges, with the ask's -// time on the message id; not written while a member is not merged, with the reason; not for a group the gate does not hold; -// and the keys of performed the tools read are the ones the process writes. -func TestAnEndAskSaysTheGroupsEndAgainOrWhyNot(t *testing.T) { - ctx := context.Background() - h1, h2 := strings.Repeat("1", 40), strings.Repeat("2", 40) - c1, c2 := strings.Repeat("c", 40), strings.Repeat("d", 40) - pull := func(repo string, n int, head string, state delivery.MergeState, commit string) delivery.MergePull { - return delivery.MergePull{Head: delivery.Head{Repository: repo, Number: n, Head: head}, State: state, Commit: commit} - } - merge := func(second delivery.MergeState, commit2 string) []byte { - v, err := delivery.EncodeMerge(delivery.Merge{Group: "feat/take-in", Ask: "ask", Approved: map[string]string{"novox/mesh-sdk@37": h1, "novox/mesh-delivery@9": h2}, - Pulls: []delivery.MergePull{pull("novox/mesh-sdk", 37, h1, delivery.MergeMerged, c1), pull("novox/mesh-delivery", 9, h2, second, commit2)}}) - if err != nil { - t.Fatal(err) - } - return v - } - ask := groups.EndAsk{V: 1, Group: "feat/take-in", By: "operator@shanks", Why: "the indexer never took it in", At: time.Date(2026, 11, 4, 23, 0, 0, 123, time.UTC)} - for _, tc := range []struct { - name string - gate map[string][]byte - published bool - written bool - answer string - }{ - {"merged and published: written again", map[string][]byte{"feat/take-in": merge(delivery.MergeMerged, c2)}, true, true, "written again"}, - {"a member's message not written", map[string][]byte{"feat/take-in": merge(delivery.MergeMerged, c2)}, false, false, "not written"}, - {"a member not merged", map[string][]byte{"feat/take-in": merge(delivery.MergeMerging, "")}, true, false, "not written"}, - {"no merge at the gate", map[string][]byte{}, true, false, "holds no merge"}, - } { - pub, done := &idPub{}, memRevisions{} - if tc.published { - done[groups.PublishedKey("novox/mesh-sdk", 37, c1)], done[groups.PublishedKey("novox/mesh-delivery", 9, c2)] = 1, 1 - } - g := &grouper{send: pub.send, done: done, logger: log.New(&bytes.Buffer{}, "", 0), - merges: func(_ context.Context, key string) ([]byte, uint64, bool, error) { - v, ok := tc.gate[key] - return v, 4, ok, nil - }} - got := g.answerEnd(ctx, "feat/take-in", ask, ask.At.Add(time.Second)) - if got.Written != tc.written || !strings.Contains(got.Answer, tc.answer) || !got.Done() || len(pub.ids) != map[bool]int{true: 1}[tc.written] { - t.Errorf("%s: %+v, ids %v", tc.name, got, pub.ids) - } - if tc.written && pub.ids[0] != groups.EndID("feat/take-in", c2, groups.AgainStamp(ask.At)) { - t.Errorf("%s: the id %s does not carry the ask's time", tc.name, pub.ids[0]) - } - } -} diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups.go index f00c6688..82b1eb12 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups.go @@ -169,8 +169,8 @@ func (g *grouper) end(ctx context.Context, key string, rev uint64, value []byte, at := slices.IndexFunc(m.Pulls, func(p delivery.MergePull) bool { return delivery.SameRepository(p.Repository, member.Repository) && p.Number == member.Number }) - if at < 0 || m.Pulls[at].State != delivery.MergeMerged || m.Pulls[at].Commit == "" { - return false // not every member is merged: the group stays gathered + if at < 0 || m.Pulls[at].Commit == "" { + return false // not every member is merged (a merge commit is written only for a merged pull): the group stays gathered } if _, ok, err := g.done.Get(ctx, groupkeys.PublishedKey(member.Repository, member.Number, m.Pulls[at].Commit)); err != nil || !ok { return false // its merged message is not written yet: said when it is (published) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups_test.go index 17c57b22..a81c2c41 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups_test.go @@ -6,6 +6,7 @@ import ( "log" "strings" "testing" + "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -24,7 +25,7 @@ func (p *idPub) send(_ context.Context, subject string, data []byte, id string) return nil } -func TestAGroupsMembersNameItAndItsEndIsSaidOnce(t *testing.T) { // 1585/T16, T17 (ADR 0633 §4) +func TestAGroupsMembersNameItAndItsEndIsSaidOnce(t *testing.T) { // 1585/T16, T17, T19 (ADR 0633 §4) ctx := context.Background() h1, h2 := strings.Repeat("1", 40), strings.Repeat("2", 40) c1, c2 := strings.Repeat("c", 40), strings.Repeat("d", 40) @@ -94,4 +95,47 @@ func TestAGroupsMembersNameItAndItsEndIsSaidOnce(t *testing.T) { // 1585/T16, T1 if !g.end(ctx, "feat/take-in", 10, gate["feat/take-in"], "20261104T2300") || pub.ids[1] == pub.ids[0] { t.Errorf("the end again by hand: ids %v", pub.ids) } + // 1585/T19: an ask to say a group's end again is answered by the bridge from the gate's merges, with the ask's time on + // the message id; not written while a member is not merged, with the reason; not for a group the gate does not hold. + pull := func(repo string, n int, head string, state delivery.MergeState, commit string) delivery.MergePull { + return delivery.MergePull{Head: delivery.Head{Repository: repo, Number: n, Head: head}, State: state, Commit: commit} + } + askMerge := func(second delivery.MergeState, commit2 string) []byte { + v, err := delivery.EncodeMerge(delivery.Merge{Group: "feat/take-in", Ask: "ask", Approved: map[string]string{"novox/mesh-sdk@37": h1, "novox/mesh-delivery@9": h2}, + Pulls: []delivery.MergePull{pull("novox/mesh-sdk", 37, h1, delivery.MergeMerged, c1), pull("novox/mesh-delivery", 9, h2, second, commit2)}}) + if err != nil { + t.Fatal(err) + } + return v + } + ask := groupkeys.EndAsk{V: 1, Group: "feat/take-in", By: "operator@shanks", Why: "the indexer never took it in", At: time.Date(2026, 11, 4, 23, 0, 0, 123, time.UTC)} + for _, tc := range []struct { + name string + gate map[string][]byte + published bool + written bool + answer string + }{ + {"merged and published: written again", map[string][]byte{"feat/take-in": askMerge(delivery.MergeMerged, c2)}, true, true, "written again"}, + {"a member's message not written", map[string][]byte{"feat/take-in": askMerge(delivery.MergeMerged, c2)}, false, false, "not written"}, + {"a member not merged", map[string][]byte{"feat/take-in": askMerge(delivery.MergeMerging, "")}, true, false, "not written"}, + {"no merge at the gate", map[string][]byte{}, true, false, "holds no merge"}, + } { + pub, done := &idPub{}, memRevisions{} + if tc.published { + done[groupkeys.PublishedKey("novox/mesh-sdk", 37, c1)], done[groupkeys.PublishedKey("novox/mesh-delivery", 9, c2)] = 1, 1 + } + g := &grouper{send: pub.send, done: done, logger: log.New(&bytes.Buffer{}, "", 0), + merges: func(_ context.Context, key string) ([]byte, uint64, bool, error) { + v, ok := tc.gate[key] + return v, 4, ok, nil + }} + got := g.answerEnd(ctx, "feat/take-in", ask, ask.At.Add(time.Second)) + if got.Written != tc.written || !strings.Contains(got.Answer, tc.answer) || !got.Done() || len(pub.ids) != map[bool]int{true: 1}[tc.written] { + t.Errorf("%s: %+v, ids %v", tc.name, got, pub.ids) + } + if tc.written && pub.ids[0] != groupkeys.EndID("feat/take-in", c2, groupkeys.AgainStamp(ask.At)) { + t.Errorf("%s: the id %s does not carry the ask's time", tc.name, pub.ids[0]) + } + } } diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/acts_test.go b/modules/mesh-delivery-forge-bridge/internal/bridge/acts_test.go index 26fa4f41..c9b9fe2d 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/acts_test.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/acts_test.go @@ -28,7 +28,7 @@ func owedValue(t *testing.T, repository string, number int, act string, o delive return key, b } -func TestOwedStatusIsACommitStatusCall(t *testing.T) { // 1341/AC1 +func TestOwedStatusIsACommitStatusCall(t *testing.T) { // 1341/AC1, K1 key, v := owedValue(t, "novox/mesh-delivery", 12, "status=mesh/merge-gate", delivery.Owed{Act: delivery.OwedStatus, Context: "mesh/merge-gate", State: "pending", Text: "waiting", Target: "https://forge.example/novox/mesh-delivery/pulls/12"}) a, err := OwedAct(key, v) @@ -43,6 +43,19 @@ func TestOwedStatusIsACommitStatusCall(t *testing.T) { // 1341/AC1 if a.Repository != "novox/mesh-delivery" || a.Number != 12 || a.Head != headA || a.Name != "status=mesh/merge-gate" { t.Errorf("act = %+v", a) } + // R4e findings K1 and R2-1 (issue 1301 S4; ADR 0503 §3, ADR 0521): no tool writes mesh/check, since a tool is told no + // caller and node-tools invokes every tool; the forge module writes it from the checker's owed state, and the bridge + // performs no mesh/check act. + ck, cv := checkerKV(t, delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: "success", Text: "pass", Target: "https://view.example/checks/g.1"}) + if a, err := OwedAct(ck, cv); err == nil || !strings.Contains(err.Error(), "mesh-delivery-checker.owed") { + t.Fatalf("OwedAct(mesh/check) = %+v, %v; want refused, naming where the forge module reads it", a, err) + } + // Nor through the gate's own owed bucket: the context is refused whichever key carries it. + gk, gv := owedValue(t, "novox/mesh-delivery", 12, "status=mesh/check", + delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: "success", Text: "pass", Target: "https://x.example"}) + if a, err := OwedAct(gk, gv); err == nil || !strings.Contains(err.Error(), "mesh-delivery-checker.owed") { + t.Fatalf("OwedAct(gate's mesh/check) = %+v, %v; want refused", a, err) + } } // checkerKV is the status the checker owes, as it writes it (mesh-sdk go/delivery CheckerOwedKey, EncodeCheckerOwed). @@ -59,23 +72,6 @@ func checkerKV(t *testing.T, o delivery.Owed) (string, []byte) { return key, v } -// R4e findings K1 and R2-1 (issue 1301 S4; ADR 0503 §3, ADR 0521): no tool writes mesh/check, since a tool is told no -// caller and node-tools invokes every tool; the forge module writes it from the checker's owed state, and the bridge -// performs no mesh/check act. -func TestOwedMeshCheckIsNoToolCall(t *testing.T) { // K1 - key, v := checkerKV(t, delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: "success", Text: "pass", Target: "https://view.example/checks/g.1"}) - a, err := OwedAct(key, v) - if err == nil || !strings.Contains(err.Error(), "mesh-delivery-checker.owed") { - t.Fatalf("OwedAct(mesh/check) = %+v, %v; want refused, naming where the forge module reads it", a, err) - } - // Nor through the gate's own owed bucket: the context is refused whichever key carries it. - gk, gv := owedValue(t, "novox/mesh-delivery", 12, "status=mesh/check", - delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: "success", Text: "pass", Target: "https://x.example"}) - if a, err := OwedAct(gk, gv); err == nil || !strings.Contains(err.Error(), "mesh-delivery-checker.owed") { - t.Fatalf("OwedAct(gate's mesh/check) = %+v, %v; want refused", a, err) - } -} - func TestOwedViewNoteAndCommentCalls(t *testing.T) { // 1341/AC2 cases := []struct { act string diff --git a/modules/mesh-delivery-forge-bridge/internal/groups/groups.go b/modules/mesh-delivery-forge-bridge/internal/groups/groups.go index 006e1703..bd09dbb7 100644 --- a/modules/mesh-delivery-forge-bridge/internal/groups/groups.go +++ b/modules/mesh-delivery-forge-bridge/internal/groups/groups.go @@ -102,6 +102,9 @@ type Row struct { EndedAt uint64 `json:"ended-at-revision,omitempty"` Open bool `json:"open"` Words string `json:"words"` + // LastEndAsk is the last ask to say the group's end again (bridge_group_end_again), its who, why and when, and the + // bridge's answer once it read it; absent when none was asked. The state end-asks keeps the last per group. + LastEndAsk *EndAsk `json:"last-end-ask,omitempty"` } // RowMember is one approved member of a group: its merge's state at the gate and whether its merged message is written. @@ -116,8 +119,18 @@ type RowMember struct { // Rows are the groups the gate's merges hold with more than one member, each with what the bridge kept: ended (its end // written, the revision it was worked out from) or open (some member is not merged or its merged message not written, or its // end not written). merges is the gate's merges by key, performed the bridge's performed keys by key; unread are the gate's -// keys that do not read. -func Rows(merges map[string][]byte, performed map[string]bool, endedAt map[string]uint64) (rows []Row, unread []string) { +// keys that do not read. asks are the end-asks values by the same key; one that does not read is named unread as +// `end-asks.`. +func Rows(merges map[string][]byte, performed map[string]bool, endedAt map[string]uint64, asks map[string][]byte) (rows []Row, unread []string) { + lastAsk := map[string]*EndAsk{} + for key, raw := range asks { + a, err := DecodeEndAsk(raw) + if err != nil { + unread = append(unread, EndAsksState+"."+key) + continue + } + lastAsk[key] = &a + } for key, raw := range merges { m, err := delivery.DecodeMerge(raw) if err != nil { @@ -140,6 +153,7 @@ func Rows(merges map[string][]byte, performed map[string]bool, endedAt map[strin allMerged = false } r.EndedAt, r.Ended = endedAt[key] + r.LastEndAsk = lastAsk[key] r.Open = !r.Ended switch { case r.Ended: @@ -152,6 +166,7 @@ func Rows(merges map[string][]byte, performed map[string]bool, endedAt map[strin rows = append(rows, r) } sortRows(rows) + sort.Strings(unread) return rows, unread } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index ce9d794b..3c24a297 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -16,7 +16,6 @@ import ( "log" "os" "os/signal" - "strings" "sync/atomic" "syscall" "time" @@ -155,6 +154,10 @@ type delivered interface { Metadata() (*jetstream.MsgMetadata, error) } +// groupEndSubject is the whole subject of a delivery group's end on the forge bridge's stream: compared exactly, so a +// repository named forge/group, whose merged message ends the same way, is never taken for one. +var groupEndSubject = process.ModuleStreamSubject(delivery.ForgeBridgeModule, forgemsg.ForgeGroupEndEvent) + // handleGroupEnd takes a group's end in (index.EndGroup): a body that does not read is terminated and said; an end // whose members are not all held yet, or while the build this program runs is not known, is left for redelivery. func handleGroupEnd(ctx context.Context, m delivered, ix *index.Indexer, h *index.Heard, logger *log.Logger) { @@ -198,7 +201,7 @@ func handle(ctx context.Context, m delivered, ix *index.Indexer, heard index.Sto } }() // A delivery group's end (novox/hq ADR 0633 §4) has a body of its own: told apart by its name, before the decode. - if strings.HasSuffix(m.Subject(), "."+forgemsg.ForgeGroupEndEvent) { + if m.Subject() == groupEndSubject { handleGroupEnd(ctx, m, ix, &h, logger) return } @@ -409,7 +412,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if !gate.Toolchain(module, key, value, deleted) { return nil } - return ix.Recompute(ctx) + // The modules written again, and pending evaluated again (ADR 0633 §1): the build a revision names moves with + // the toolchain. + return ix.ToolchainReported(ctx) }, recompute: ix.Recompute}, logger) if err := tools.start(); err != nil { @@ -500,18 +505,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro defer pc.Stop() } // The pending revisions that name this build are taken in now, oldest first, before the consumer on the forge bridge's - // stream starts (ADR 0633 §2); the build this program runs is asked until it is known, said every minute. A hand - // take-in (indexer_take_in_pending) is answered meanwhile: the takeins watch runs already. - if _, words, err := ix.TakeInAtStart(ctx, func(ctx context.Context) bool { - // A context's bound between two asks of the node-engine, no timer (ADR 0437 §8), as askTakeIn bounds its wait. - again, cancel := context.WithTimeout(ctx, time.Minute) - defer cancel() - <-again.Done() - return ctx.Err() == nil - }); err != nil { + // stream starts (ADR 0633 §2). When the build this program runs is not known yet, nothing waits for it: the consumer + // starts, and a merged message left unacknowledged asks the build again at its redelivery (clause 1). A hand take-in + // (indexer_take_in_pending) is answered at any time: the takeins watch runs already. + if _, words, err := ix.TakeInAtStart(ctx); err != nil { return fmt.Errorf("the take-in at start: %w", err) } else if words != "" { - logger.Printf("take-in at start: still waiting: %s", words) + logger.Printf("take-in at start: %s", words) } cons, err := cred.EventConsumer(ctx, js, forgeStream) if err != nil { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go index c4c3672c..b63c77ed 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go @@ -121,6 +121,13 @@ func TestAMessageThatIsNoForgeMessageIsTerminatedAndLogged(t *testing.T) { // 12 t.Errorf("heard of a group's end = %+v", h) } } + // The repository forge/group's merged message ends as a group's end does: it is a forge message all the same. + logs.Reset() + m = &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event.forge.forge.group.merged", data: []byte(`{"v":1,"group":"feat/x","members":[{"repository":"novox/mesh-sdk","number":3,"merge":"` + head + `"}]}`)} + handle(context.Background(), m, &index.Indexer{Store: st, Builds: index.NewBuilds()}, heard, log.New(&logs, "", 0)) + if !strings.Contains(logs.String(), "not a forge message") { + t.Errorf("forge/group's merged message read as a group's end: %q", logs.String()) + } } func TestTheSourceMainKeyIsTakenAtItsRevision(t *testing.T) { // 1344/AC1 (the watch's side: key, revision, value) diff --git a/modules/mesh-delivery-indexer/internal/index/index_test.go b/modules/mesh-delivery-indexer/internal/index/index_test.go index a29adba7..8c0cff64 100644 --- a/modules/mesh-delivery-indexer/internal/index/index_test.go +++ b/modules/mesh-delivery-indexer/internal/index/index_test.go @@ -26,6 +26,7 @@ type memStore struct { next uint64 writes int beforeWrite func() + failWrite func(key string) error // a write's own error, asked before each Create and Update } func newMem() *memStore { return &memStore{vals: map[string][]byte{}, revs: map[string]uint64{}} } @@ -59,6 +60,11 @@ func (m *memStore) put(key string, value []byte) { } func (m *memStore) Create(_ context.Context, key string, value []byte) error { m.hook() + if f := m.failWrite; f != nil { + if err := f(key); err != nil { + return err + } + } if _, ok := m.vals[key]; ok { return ErrConflict } @@ -68,6 +74,11 @@ func (m *memStore) Create(_ context.Context, key string, value []byte) error { } func (m *memStore) Update(_ context.Context, key string, value []byte, rev uint64) error { m.hook() + if f := m.failWrite; f != nil { + if err := f(key); err != nil { + return err + } + } if _, ok := m.vals[key]; !ok || m.revs[key] != rev { return ErrConflict } diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index 19e91f1b..3eca3a36 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -396,21 +396,20 @@ func (ix *Indexer) takenAlready(ctx context.Context, m forgemsg.GroupMerge) (boo } // TakeInAtStart is the take-in at the start (ADR 0633 §2), before the consumer on the forge bridge's stream starts: -// it asks the build this program runs until it is known (wait says when to ask again, false ends it), then evaluates -// pending, taking in the whole revisions that name this build; a group still being gathered stays, and a revision -// naming yet another build is held for that one. A claim left by an indexer that ran before is void here. -func (ix *Indexer) TakeInAtStart(ctx context.Context, wait func(context.Context) bool) (Applied, string, error) { +// it evaluates pending once with the build this program runs, taking in the whole revisions that name this build; a +// group still being gathered stays, and a revision naming yet another build is held for that one. A claim left by an +// indexer that ran before is void here. When the build is not known (the node-engine did not answer), it waits for +// nothing: it says so and the consumer starts all the same; a merged message is then left unacknowledged until the +// build is known (clause 1), and its redelivery, or a toolchain report (ToolchainReported), asks again. No timer. +func (ix *Indexer) TakeInAtStart(ctx context.Context) (Applied, string, error) { if ix.Pending == nil || !ix.pendingOnBus(ctx) { // Nothing can wait in a state not on the bus: the start asks no build and does not wait for one. return Applied{}, "the pending state is not on the bus yet: nothing waits", nil } own := ix.ownBuildNow(ctx) - for own == "" { - ix.logf("take-in: the build this program runs is not known yet, so nothing pending is taken in; asked again") - if !wait(ctx) { - return Applied{}, "", ErrOwnBuildUnknown - } - own = ix.ownBuildNow(ctx) + if own == "" { + ix.logf("take-in: the build this program runs is not known at its start, so nothing pending is taken in now; the forge consumer starts, and a merged message left unacknowledged asks it again") + return Applied{}, "the build this program runs is not known yet: nothing pending is taken in at the start", nil } ix.pendingMu.Lock() defer ix.pendingMu.Unlock() @@ -421,6 +420,29 @@ func (ix *Indexer) TakeInAtStart(ctx context.Context, wait func(context.Context) return a, words, err } +// PendingAgain evaluates pending again after something a fingerprint is computed from changed, a toolchain report +// (ADR 0633 §1, §3): a revision that now names the running build is taken in, and one that names another build is held +// for that one. It does nothing while the pending state is not on the bus or the build this program runs is not known. +func (ix *Indexer) PendingAgain(ctx context.Context) error { + if ix.Pending == nil || !ix.pendingOnBus(ctx) { + return nil + } + own := ix.ownBuildNow(ctx) + if own == "" { + return nil + } + ix.pendingMu.Lock() + defer ix.pendingMu.Unlock() + _, words, err := ix.evaluate(ctx, own, false) + if errors.Is(err, process.ErrNotYet) { + return nil + } + if err == nil && words != "" { + ix.logf("take-in: evaluated again after a toolchain report: %s", words) + } + return err +} + // evaluate works out what pending's ready merges name, together (ADR 0633 §3), and takes them in or holds them. func (ix *Indexer) evaluate(ctx context.Context, own string, start bool) (Applied, string, error) { entries, err := ReadPending(ctx, ix.Pending, ix.logf) @@ -547,8 +569,9 @@ func (ix *Indexer) computeOver(ctx context.Context, ready []PendingEntry) (strin return m.FP2, d, nil } -// claim marks the entries as taken in by this take-in, by compare-and-set on the revisions read; a lost one leaves -// every claim made and answers false, and nothing is taken in. +// claim marks the entries as taken in by this take-in, by compare-and-set on the revisions read. A lost one (another +// take-in wrote the key, or the store failed) releases every claim this take-in made and answers false: nothing is +// taken in, and no claim of this one is left behind. func (ix *Indexer) claim(ctx context.Context, entries []PendingEntry, own, by string) ([]PendingEntry, bool) { id := fmt.Sprintf("%s at %s", own, ix.now().Format(time.RFC3339Nano)) if by != "" { @@ -564,18 +587,52 @@ func (ix *Indexer) claim(ctx context.Context, entries []PendingEntry, own, by st } if err != nil { ix.logf("take-in: the claim of %s.%s is lost (%v): another take-in took it, so this one takes nothing in", PendingState, e.Key, err) + ix.release(ctx, claimed) return nil, false } + e.Value = p _, rev, _, err := ix.Pending.Get(ctx, e.Key) if err != nil { + ix.logf("take-in: the claim of %s.%s cannot be read back (%v), so this take-in takes nothing in", PendingState, e.Key, err) + ix.release(ctx, append(claimed, e)) return nil, false } - e.Revision, e.Value = rev, p + e.Revision = rev claimed = append(claimed, e) } return claimed, true } +// release clears this take-in's claim on each entry, so a later merge, a restart or a hand take-in takes it in. It +// reads the key again and clears it by compare-and-set only while the claim is still this take-in's: a key another +// take-in wrote since, or removed, is left to it. A release that fails is said; the start of the next indexer voids +// the claim (TakeInAtStart). +func (ix *Indexer) release(ctx context.Context, claimed []PendingEntry) { + for _, e := range claimed { + value, rev, found, err := ix.Pending.Get(ctx, e.Key) + if err == nil && !found { + continue + } + var cur Pending + if err == nil { + err = json.Unmarshal(value, &cur) + } + if err == nil && cur.Taking != e.Value.Taking { + continue + } + if err == nil { + cur.Taking = "" + var data []byte + if data, err = encodePending(cur); err == nil { + err = ix.Pending.Update(ctx, e.Key, data, rev) + } + } + if err != nil { + ix.logf("take-in: the claim of %s.%s is not released (%v): the next indexer's start voids it", PendingState, e.Key, err) + } + } +} + // takeInWhole takes the entries in with this indexer, as merges are taken in (ApplyMessage), oldest first, then // removes them from pending and says each wait they were said under taken. by names a hand act ("" for none). func (ix *Indexer) takeInWhole(ctx context.Context, entries []PendingEntry, own, by string) (Applied, error) { @@ -600,6 +657,7 @@ func (ix *Indexer) takeInWholeBy(ctx context.Context, entries []PendingEntry, ow continue } if err != nil { + ix.release(ctx, claimed) return applied, fmt.Errorf("taking %s (%s#%d) in: %w", e.Key, e.Forge.Repository, e.Forge.Number, err) } } diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index 51d7157f..966f599f 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -118,7 +118,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) // Before the pending state is on the bus nothing waits: the start asks no build, and the merge is taken in at once, // as before ADR 0633, though the build is unknown (TestIndexerWhole found the start waiting on it for ever). b.ix.Pending = &memStore{unbound: process.ErrNotYet} - if _, _, err := b.ix.TakeInAtStart(ctx, func(context.Context) bool { t.Fatal("the start waits for the build"); return false }); err != nil { + if _, _, err := b.ix.TakeInAtStart(ctx); err != nil { t.Fatal(err) } if _, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", "")); err != nil || b.commit(t, "avahi") != sha('b') { @@ -183,7 +183,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) next.toolchains = b.ix.toolchains asked := 0 next.OwnBuild = func(context.Context) (string, error) { - if asked++; asked < 2 { + if asked++; asked < 3 { // the hand ask's and the first start's return "", errors.New("not yet") } return named, nil @@ -191,10 +191,15 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) if r := next.TakeInPendingByHand(ctx, "m.nothing", "lead@shanks", "a test"); r.Stored || !strings.Contains(r.Answer, "nothing waits") { t.Errorf("a hand ask while the own build is unknown: %+v", r) } - if _, _, err := next.TakeInAtStart(ctx, func(context.Context) bool { return true }); err != nil { + // The first start, its build not known, waits for nothing (no sleep, no timer): it answers at once, takes nothing + // in, and the forge consumer starts after it. + if _, words, err := next.TakeInAtStart(ctx); err != nil || asked != 2 || len(b.keys()) != 2 || !strings.Contains(words, "not known") { + t.Fatalf("a start with its build unknown: %q %v, asked %d, pending %v: want it to answer at once and take nothing in", words, err, asked, b.keys()) + } + if _, _, err := next.TakeInAtStart(ctx); err != nil { t.Fatal(err) } - if asked < 2 || !reflect.DeepEqual(b.keys(), []string{takein.PendingKey(5)}) || b.commit(t, "avahi") != sha('c') { + if asked < 3 || !reflect.DeepEqual(b.keys(), []string{takein.PendingKey(5)}) || b.commit(t, "avahi") != sha('c') { t.Errorf("start: asked %d, pending %v, avahi at %s: want the revision taken in, the group left", asked, b.keys(), b.commit(t, "avahi")) } }) @@ -275,13 +280,96 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) data, _ := json.Marshal(p) b.pending.put(takein.PendingKey(2), data) } - if _, _, err := b.ix.TakeInAtStart(ctx, func(context.Context) bool { return false }); err != nil { + if _, _, err := b.ix.TakeInAtStart(ctx); err != nil { t.Fatal(err) } if b.commit(t, "avahi") == sha('c') || len(b.keys()) != 1 || !strings.Contains(b.logs.String(), "is lost") { t.Errorf("avahi at %s, pending %v: want nothing taken in by the loser, and it said", b.commit(t, "avahi"), b.keys()) } }) + t.Run("T5 a take-in whose store fails once leaves no claim behind, and is taken in after", func(t *testing.T) { + claims := func(b *takeInBed) []string { + var out []string + for _, k := range b.keys() { + var p Pending + _ = json.Unmarshal(b.pending.vals[k], &p) + if p.Taking != "" { + out = append(out, k) + } + } + return out + } + once := func(match func(key string) bool) func(string) error { + done := false + return func(key string) error { + if !done && match(key) { + done = true + return errors.New("the store failed once") + } + return nil + } + } + for _, c := range []struct { + name string + fail func(b *takeInBed) + }{ + // The second merge's claim fails: the first's, already made, is released. + {"a claim lost", func(b *takeInBed) { + b.pending.failWrite = once(func(k string) bool { return k == takein.PendingKey(3) }) + }}, + // The take-in itself fails on a module's write after both claims: both are released. + {"a take-in that fails", func(b *takeInBed) { b.mods.failWrite = once(func(string) bool { return true }) }}, + } { + b := newTakeInBed(t) + b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) + b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")) + named := b.fp(t, delivery.IndexerModule) + b.ix.ownMu.Lock() + b.ix.ownFP = named + b.ix.ownMu.Unlock() + c.fail(b) + _, _, _ = b.ix.TakeInAtStart(ctx) + if got := claims(b); len(got) != 0 || len(b.keys()) != 2 { + t.Fatalf("%s: pending %v, claimed %v: want both pending and no claim left", c.name, b.keys(), got) + } + // The next take-in, not at a start, is not refused by a claim left behind: it takes both in. + if _, _, err := b.ix.Arrive(ctx, 4, b.merge(23, sha('e'), "tree-avahi-5", "tree-indexer-2")); err != nil { + t.Fatal(err) + } + if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('e') { + t.Errorf("%s: after: pending %v, avahi at %s: want all taken in", c.name, b.keys(), b.commit(t, "avahi")) + } + } + }) + t.Run("T7 a toolchain report evaluates pending again: the revision then names the running build and is taken in", func(t *testing.T) { + b := newTakeInBed(t) + b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) + entries, err := ReadPending(ctx, b.pending, nil) + if err != nil { + t.Fatal(err) + } + // The build the revision names under the next toolchain, worked out on a dry indexer; this one runs it. + x := b.ix.dry().ix + for _, node := range []string{"novox", "ace"} { + key, _ := delivery.ToolchainKey(node, "go") + x.PutToolchain("mesh-build-go", key, toolchainValue(t, "go", "go1.27.2", "1"), false) + } + next, _, err := x.computeOver(ctx, WaitsOf(entries).Ready) + if err != nil || next == "" || next == b.said[0].WaitsFor { + t.Fatalf("the build under the next toolchain: %q %v (the revision names %s)", next, err, b.said[0].WaitsFor) + } + b.ix.ownMu.Lock() + b.ix.ownFP = next + b.ix.ownMu.Unlock() + reportToolchain(t, b.ix, "mesh-build-go", "novox", "go", "go1.27.2") + if len(b.keys()) != 1 { + t.Fatalf("one node's report: pending %v, want the revision still held (the nodes disagree)", b.keys()) + } + reportToolchain(t, b.ix, "mesh-build-go", "ace", "go", "go1.27.2") + if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('c') { + t.Errorf("after the toolchain report: pending %v, avahi at %s: want the revision taken in", b.keys(), b.commit(t, "avahi")) + } + }) t.Run("T12 T13 indexer_pending names the wait; a hand take-in and a drop say what they ended", func(t *testing.T) { b := newTakeInBed(t) b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index 8218c5f7..18452c86 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -78,12 +78,22 @@ func (ix *Indexer) PutToolchain(module, key string, value []byte, deleted bool) } // ApplyToolchain takes one change of a build module's toolchains in (PutToolchain) and, when the language's toolchain -// changed, writes the modules again that it moves (Recompute). The same version and recipe seen again writes nothing. +// changed, writes the modules again that it moves and evaluates pending again (ToolchainReported). The same version and +// recipe seen again writes nothing. func (ix *Indexer) ApplyToolchain(ctx context.Context, module, key string, value []byte, deleted bool) error { if !ix.PutToolchain(module, key, value, deleted) { return nil } - return ix.Recompute(ctx) + return ix.ToolchainReported(ctx) +} + +// ToolchainReported is what a changed toolchain moves: the modules written again (Recompute), then pending evaluated +// again (PendingAgain), since the indexer build a pending revision names is a fingerprint the toolchain is part of. +func (ix *Indexer) ToolchainReported(ctx context.Context) error { + if err := ix.Recompute(ctx); err != nil { + return err + } + return ix.PendingAgain(ctx) } // agreed is the toolchain of each language every reporting node agrees on, or why there is none. diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go index 9e280190..6be59ed0 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go @@ -7,6 +7,7 @@ package main import ( "context" + "encoding/json" "errors" "fmt" "io" @@ -378,8 +379,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro dead.check() provisioned(ctx, js, w, logger.Printf) // The indexer's stream take-in (ADR 0633 §5): a revision or a group it holds back is a wait, ended by take-in.taken. The - // durable consumer _ is made by the controller from this module's consumes; the whole stream is heard at - // the start, so a wait that began while the watchman was away is read with its own since. + // durable consumer _ is made by the controller from this module's consumes, and it is durable: what it + // acknowledged before a restart is not heard again. So the waits the indexer holds now are asked of it first, through + // its tool indexer_pending (this module's invokes), each with its own since; the consumer then hears what follows. + if calls, err := process.Connect(cred, nats.Name(cred.Node+"."+cred.Module+".calls")); err != nil { + logger.Printf("take-in: no connection to ask %s (%v), so a wait the indexer held before this start is said only once a take-in event of it is heard", pendingTool, err) + } else { + seedTakeIn(w, calls.Call, logger.Printf) + calls.Close() + } switch tc, err := cred.EventConsumer(ctx, js, catalogue.StreamBusName(takein.TakeInStream)); { case err != nil: logger.Printf("the consumer %s_%s on %s does not exist yet, so no take-in wait is said until the watchman starts again: %v", @@ -465,6 +473,24 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } } +// pendingTool is the indexer's tool that answers the waits it holds (ADR 0633 §5), as the view asks it. +const pendingTool = delivery.IndexerModule + ".indexer_pending" + +// seedTakeIn asks the indexer the waits it holds now and starts each (Watchman.SeedTakeIn). An indexer that does not +// answer is said: the waits are then known only from the events heard after. +func seedTakeIn(w *watch.Watchman, call func(key string, body any) (json.RawMessage, error), logf func(string, ...any)) { + raw, err := call(pendingTool, map[string]any{}) + if err == nil { + var a takein.PendingAnswer + if a, err = takein.DecodePendingAnswer(raw); err == nil { + w.SeedTakeIn(a) + logf("take-in: %d waits the indexer holds are watched from their own since (%s)", len(a.Waits), pendingTool) + return + } + } + logf("take-in: %s cannot be asked (%v), so a wait the indexer held before this start is said only once a take-in event of it is heard", pendingTool, err) +} + // takeInHeard hands one event of a take-in stream to the watchman, by its name relative to the stream, and acknowledges it; // one that is not a take-in event is said and acknowledged, since hearing it again would not read it. func takeInHeard(w *watch.Watchman, m jetstream.Msg, logf func(string, ...any)) { diff --git a/modules/mesh-delivery-watchman/internal/watch/findings_test.go b/modules/mesh-delivery-watchman/internal/watch/findings_test.go index 8f81f1a1..f6802626 100644 --- a/modules/mesh-delivery-watchman/internal/watch/findings_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/findings_test.go @@ -94,6 +94,18 @@ func TestTheWatchmansFindings(t *testing.T) { t.Fatalf("a wait that ended in the replay wrote %d findings and left pulses %v", o.findingWrites, o.pulses) } }, map[string]string{}}, + {"ADR 0633 §5 (1585/T21): a wait the indexer holds at a restart within its bound is seeded from indexer_pending and raised past it", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + // The restarted watchman's durable consumer hears nothing of the pending event it acknowledged before; the wait + // began ten minutes ago, and the indexer still holds it. + since := c.t.Add(-10 * time.Minute) + w.SeedTakeIn(takein.PendingAnswer{Waits: []takein.PendingWait{{Key: revKey, What: takein.TakeInRevision, Keys: []string{revKey}, + Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}, WaitsFor: "src2:x", Since: since}}}) + w.TakeInCaughtUp() + if len(o.findings) != 0 || o.ttls[signalKey("take-in", revKey)] != TakeInRevisionBound-10*time.Minute { + t.Fatalf("within its bound: findings %v, ttl %v; want none yet and the rest of the bound", o.findings, o.ttls[signalKey("take-in", revKey)]) + } + expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound-10*time.Minute) + }, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}}, {"ADR 0633 §5 (1585/T21): a drop clears the wait, and the merges left under it said pending again with the same since are raised from it", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { w.TakeInCaughtUp() // a live message, heard after the replay w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), takeInBody(t, takein.TakeInPending, revKey, t0)) diff --git a/modules/mesh-delivery-watchman/internal/watch/takein.go b/modules/mesh-delivery-watchman/internal/watch/takein.go index 4d3e4276..9bb48246 100644 --- a/modules/mesh-delivery-watchman/internal/watch/takein.go +++ b/modules/mesh-delivery-watchman/internal/watch/takein.go @@ -40,6 +40,25 @@ func (w *Watchman) TakeIn(event string, body []byte) { } w.mu.Lock() defer w.mu.Unlock() + w.takeIn(kind, key, t) +} + +// SeedTakeIn starts, at the watchman's start and before its consumer on the take-in stream, the wait of every revision +// and group the indexer holds now, as indexer_pending answers them (ADR 0633 §5): the consumer is durable, so the +// pending events it acknowledged before a restart are not heard again, and a wait within its bound would otherwise be +// lost. Each wait's bound runs from its own since, as a pending event's does; the stream's events that follow replace +// or end it. +func (w *Watchman) SeedTakeIn(a takein.PendingAnswer) { + w.mu.Lock() + defer w.mu.Unlock() + for _, wait := range a.Waits { + w.takeIn(takein.TakeInPending, wait.Key, takein.TakeIn{Kind: takein.TakeInPending, Key: wait.Key, What: wait.What, + Merges: wait.Merges, Group: wait.Group, Members: wait.Members, WaitsFor: wait.WaitsFor, Since: wait.Since}) + } +} + +// takeIn is one take-in event of key, w.mu held. +func (w *Watchman) takeIn(kind, key string, t takein.TakeIn) { finding := FindingKey(ScopeDelivery, key, KindTakeInWaits) if kind == takein.TakeInTaken { delete(w.takeInHeld, key) diff --git a/modules/mesh-delivery-watchman/manifest_test.go b/modules/mesh-delivery-watchman/manifest_test.go index 6a54308e..342caead 100644 --- a/modules/mesh-delivery-watchman/manifest_test.go +++ b/modules/mesh-delivery-watchman/manifest_test.go @@ -52,6 +52,7 @@ type manifest struct { Slug string `json:"slug"` Reads []string `json:"reads"` Consumes []string `json:"consumes"` + Invokes []string `json:"invokes"` State []state `json:"state"` OwnSecrets map[string]string `json:"own-secrets"` SecretsOwner string `json:"secrets-owner"` @@ -99,6 +100,10 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6 if !reflect.DeepEqual(m.Consumes, []string{"mesh-delivery-indexer.take-in.**"}) { t.Errorf("consumes = %v", m.Consumes) } + // The waits the indexer holds at the watchman's start are asked of its tool (ADR 0633 §5), a verb edge as the view's. + if !reflect.DeepEqual(m.Invokes, []string{"mesh-delivery-indexer.indexer_pending"}) { + t.Errorf("invokes = %v", m.Invokes) + } // 1952/WM1: the watchman holds the seat mesh-watchdogs at mesh scope, its first seat (novox/hq ADR 0564 §1, §4). if !reflect.DeepEqual(m.Claims, []claim{{Name: "mesh-watchdogs", Scope: "mesh", Serves: []string{"signals", "stuck", "presence", "dead-letters", "resync"}}}) { t.Errorf("claims = %+v", m.Claims) diff --git a/modules/mesh-delivery-watchman/module.json b/modules/mesh-delivery-watchman/module.json index 2de43956..7cdc94d5 100644 --- a/modules/mesh-delivery-watchman/module.json +++ b/modules/mesh-delivery-watchman/module.json @@ -29,6 +29,9 @@ "consumes": [ "mesh-delivery-indexer.take-in.**" ], + "invokes": [ + "mesh-delivery-indexer.indexer_pending" + ], "tools": [ "watchman_findings" ], From c78941e80b59f6a0d727cee43abfad1530df9881 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 5 Nov 2026 00:59:27 +0100 Subject: [PATCH 18/18] =?UTF-8?q?Take-in=20events=20are=20the=20indexer's?= =?UTF-8?q?=20ordinary=20events:=20no=20stream=20of=20its=20own,=20which?= =?UTF-8?q?=20the=20manifest=20validator=20refuses=20(issue=201585,=20ADR?= =?UTF-8?q?=200633=20=C2=A75)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The group's check failed at 7b7096f3: the group's own manifest validator refuses the indexer's stream "take-in", outside the bus's predefined forms. The lead's ruling: the indexer says take-in.pending and take-in.taken as its ordinary events (its emits, mesh.mod..event.take-in.), as manifest-accepted is said, and keeps no stream. The wait's key is in the body only: an emitted event names one event (no wildcard), and a key such as g. is no event name. The watchman consumes the two events by name on the shared events stream; it needs no replay, since it seeds its waits from indexer_pending at its start. indexer_pending says the last event of each wait as take-in.. Run locally, as the check runs it: cmd/mesh-judge over every manifest of mesh-catalog, mesh-delivery, mesh-host, mesh-tools, mesh-networking and mesh-media-catalog, with this branch's and mesh-catalog's feat/take-in manifests in the change case and origin/main's in the trunk case: 171 of 171 accepted in both. The same run with the indexer's manifest at 7b7096f3 refuses it with the check's words. Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn --- .../mesh-delivery-indexer/integration_test.go | 37 ++++++++++----- .../cmd/mesh-delivery-indexer/verdicts.go | 8 ++-- .../internal/index/pending.go | 2 +- .../internal/index/pending_test.go | 2 +- .../mesh-delivery-indexer/manifest_test.go | 12 ++--- modules/mesh-delivery-indexer/module.json | 7 ++- .../mesh-delivery-indexer/takein/takein.go | 45 ++++++++++--------- .../takein/takein_test.go | 17 +++---- .../cmd/mesh-delivery-watchman/main.go | 14 +++--- .../internal/watch/findings_test.go | 2 +- .../internal/watch/takein.go | 11 +++-- .../mesh-delivery-watchman/manifest_test.go | 2 +- modules/mesh-delivery-watchman/module.json | 3 +- 13 files changed, 95 insertions(+), 67 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index db89bb59..345cb0f2 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -74,8 +74,8 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 t.Fatal(err) } } - for name, subject := range map[string]string{"TAKE_IN": "mesh.stream.mesh-delivery-indexer.event.take-in.>", - "INDEXER_EVENTS": "mesh.mod.mesh-delivery-indexer.event.>"} { + // The take-in events are the indexer's ordinary events (ADR 0633 §5), on the path manifest-accepted takes. + for name, subject := range map[string]string{"INDEXER_EVENTS": "mesh.mod.mesh-delivery-indexer.event.>"} { if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: name, Subjects: []string{subject}}); err != nil { t.Fatal(err) } @@ -164,18 +164,34 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 return ack.Sequence } m1Key := takein.PendingKey(merged(m1)) - takeIns, err := js.Stream(ctx, "TAKE_IN") + events, err := js.Stream(ctx, "INDEXER_EVENTS") if err != nil { t.Fatal(err) } + // saidAll is every take-in event of kind said under key so far, oldest first. + saidAll := func(kind, key string) []takein.TakeIn { + event, _ := takein.TakeInEvent(kind) + info, err := events.Info(ctx) + if err != nil { + t.Fatal(err) + } + var out []takein.TakeIn + for seq := info.State.FirstSeq; seq <= info.State.LastSeq && seq > 0; seq++ { + msg, err := events.GetMsg(ctx, seq) + if err != nil || msg.Subject != "mesh.mod."+ix+".event."+event { + continue + } + if ti, err := takein.DecodeTakeIn(msg.Data); err == nil && ti.Key == key { + out = append(out, ti) + } + } + return out + } // said is the last take-in event under kind and key, within 20 s. said := func(kind, key string) (takein.TakeIn, bool) { - event, _ := takein.TakeInEvent(kind, key) for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { - if msg, err := takeIns.GetLastMsgForSubject(ctx, process.ModuleStreamSubject(ix, event)); err == nil { - if ti, err := takein.DecodeTakeIn(msg.Data); err == nil { - return ti, true - } + if all := saidAll(kind, key); len(all) > 0 { + return all[len(all)-1], true } } return takein.TakeIn{}, false @@ -258,7 +274,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 } } if len(w) != 2 || w[0].Key != m1Key || w[0].WaitsFor != named || !slices.Contains(w[0].Holds, "a") || !slices.Contains(w[0].Holds, ix) || - w[1].Key != groupKey || w[1].What != takein.TakeInGroup || len(w[1].Merges) != 1 || !strings.HasSuffix(w[1].Said, "pending."+groupKey) { + w[1].Key != groupKey || w[1].What != takein.TakeInGroup || len(w[1].Merges) != 1 || w[1].Said != takein.TakeInPendingEvent { t.Fatalf("indexer_pending = %+v, want the revision %s waiting for %s holding a and the indexer, then the group gathered", w, m1Key, named) } @@ -309,8 +325,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 t.Errorf("after the group's end: modules.c %v, modules.k %v, indexer_pending %+v: want both taken in and nothing waiting", hasC, hasK, w) } // One take-in: neither member was said as a revision of its own. - single, _ := takein.TakeInEvent(takein.TakeInPending, g1Key) - if _, err := takeIns.GetLastMsgForSubject(ctx, process.ModuleStreamSubject(ix, single)); err == nil { + if len(saidAll(takein.TakeInPending, g1Key)) != 0 { t.Errorf("the group's first member %s was said pending as a revision of its own", g1Key) } } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/verdicts.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/verdicts.go index 3526751a..3d887748 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/verdicts.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/verdicts.go @@ -64,11 +64,11 @@ func sayAccepted(js jetstream.JetStream, cred process.Account, logf func(string, } } -// sayTakeIn publishes a take-in event on the indexer's own stream take-in (novox/hq ADR 0633 §5): -// `mesh.stream..event.take-in..`. A publish that fails is the caller's to say. +// sayTakeIn says a take-in event (novox/hq ADR 0633 §5) as the indexer's ordinary event, on the path its other events +// take: mesh.mod..event.take-in., the wait's key in the body. A publish that fails is the caller's to say. func sayTakeIn(js jetstream.JetStream, cred process.Account) func(context.Context, takein.TakeIn) error { return func(ctx context.Context, t takein.TakeIn) error { - event, err := takein.TakeInEvent(t.Kind, t.Key) + event, err := takein.TakeInEvent(t.Kind) if err != nil { return err } @@ -76,7 +76,7 @@ func sayTakeIn(js jetstream.JetStream, cred process.Account) func(context.Contex if err != nil { return err } - msg := nats.NewMsg(process.ModuleStreamSubject(cred.Module, event)) + msg := nats.NewMsg(cred.EventSubject(event)) msg.Header.Set("x-source", cred.Module) msg.Header.Set("x-node", cred.Node) msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano)) diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index 3eca3a36..b3a15633 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -815,7 +815,7 @@ func waitOf(entries []PendingEntry, what, behind string) takein.PendingWait { } } if e.Value.Said != "" && e.Value.SaidAt.After(pw.SaidAt) { - pw.Said, pw.SaidAt = takein.TakeInStream+"."+e.Value.SaidKind+"."+e.Value.Said, e.Value.SaidAt + pw.Said, pw.SaidAt = "take-in."+e.Value.SaidKind, e.Value.SaidAt } } slices.Sort(pw.Holds) diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index 966f599f..5a5396ce 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -385,7 +385,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) } } if len(answer.Waits) != 1 || answer.Waits[0].Key != takein.PendingKey(2) || !slices.Equal(answer.Waits[0].Holds, []string{"avahi", delivery.IndexerModule}) || - answer.Waits[0].Said != "take-in.pending."+takein.PendingKey(2) || len(answer.Waits[0].Keys) != 2 { + answer.Waits[0].Said != takein.TakeInPendingEvent || len(answer.Waits[0].Keys) != 2 { t.Fatalf("indexer_pending = %+v", answer) } b.said = nil diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index ace5d0ed..5ce577c3 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -104,8 +104,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // 1126: placement is the seat mesh-placement's (ADR 0567): the indexer holds no placement state. ADR 0578 §1: it says // a refused manifest as manifest-refused, and (issue 1966 D3, 1966/AC10) a newly accepted one asking for bus state as // manifest-accepted. - if !reflect.DeepEqual(m.Emits, []string{"manifest-refused", "manifest-accepted"}) { - t.Errorf("emits = %v, want manifest-refused and manifest-accepted", m.Emits) + if !reflect.DeepEqual(m.Emits, []string{"manifest-refused", "manifest-accepted", takein.TakeInPendingEvent, takein.TakeInTakenEvent}) { + t.Errorf("emits = %v, want manifest-refused, manifest-accepted and the two take-in events", m.Emits) } // 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard. if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: delivery.PackagesState, History: delivery.PackagesHistory}, @@ -146,10 +146,10 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build !slices.Equal(m.Claims[0].Serves, []string{"modules", "module", "provides", "dependents"}) { t.Errorf("claims = %+v, want mesh-catalog with its four verbs", m.Claims) } - // 1585/T11 (ADR 0633 §1, §5): its one stream, take-in, an old streams entry (the indexer is foundation, ADR 0595 §11), - // carries the take-in events. - if want := []streamDecl{{Name: takein.TakeInStream, Events: []string{"take-in.**"}, MaxAgeDays: 7}}; !reflect.DeepEqual(m.Streams, want) { - t.Errorf("streams = %+v, want %+v", m.Streams, want) + // 1585/T11 (ADR 0633 §1, §5): the take-in events are its ordinary events, with no stream of its own: a stream is + // outside the bus's predefined forms, and the manifest validator refuses it. + if len(m.Streams) != 0 || !slices.Contains(m.Emits, takein.TakeInPendingEvent) || !slices.Contains(m.Emits, takein.TakeInTakenEvent) { + t.Errorf("streams = %+v, emits = %v: want no stream, and both take-in events emitted", m.Streams, m.Emits) } // No other bus section, no other seat (ADR 0437 §4). for _, k := range []string{"logs", "provides", "requires", "uses", "defines-seats", "seat", "seats", "bus"} { diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index fafd9bcc..10bb8227 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -44,12 +44,11 @@ { "name": "graph", "history": 5 }, { "name": "pending", "history": 1 } ], - "streams": [ - { "name": "take-in", "events": ["take-in.**"], "max-age-days": 7 } - ], "emits": [ "manifest-refused", - "manifest-accepted" + "manifest-accepted", + "take-in.pending", + "take-in.taken" ], "reads": [ "mesh-build-go.toolchains", diff --git a/modules/mesh-delivery-indexer/takein/takein.go b/modules/mesh-delivery-indexer/takein/takein.go index dc53626c..31fbfb8e 100644 --- a/modules/mesh-delivery-indexer/takein/takein.go +++ b/modules/mesh-delivery-indexer/takein/takein.go @@ -49,14 +49,17 @@ func decode(data []byte, into any) error { return nil } -// The take-in events (novox/hq ADR 0633 §1, §5): the indexer publishes them on its own stream, which keeps the last -// message per subject, `take-in..`. A wait starts with `take-in.pending` and ends with `take-in.taken`, -// whoever ended it (the named indexer, a revert, a hand take-in or a drop). The stream's name is the indexer's manifest's; -// these names are its events, relative to it. +// The take-in events (novox/hq ADR 0633 §1, §5): the indexer's ordinary events, `take-in.pending` and `take-in.taken`, on +// the path its other events take (the module's event subject; its manifest's emits), with no stream of its own. A wait +// starts with `take-in.pending` and ends with `take-in.taken`, whoever ended it (the named indexer, a revert, a hand +// take-in or a drop); the wait's key is in the body (TakeIn.Key), since an emitted event names one event and a key is +// no event name. A reader that starts later asks the waits of indexer_pending, not a replay. const ( - TakeInStream = "take-in" TakeInPending = "pending" TakeInTaken = "taken" + // TakeInPendingEvent and TakeInTakenEvent are the two events' names, relative to the indexer. + TakeInPendingEvent = "take-in." + TakeInPending + TakeInTakenEvent = "take-in." + TakeInTaken ) // What waits in a take-in event: a source revision for the indexer build it names, or a delivery group for its end. @@ -80,27 +83,24 @@ func GroupTakeInKey(group string) string { }, group) } -// TakeInEvent is the name of a take-in event of the given kind and key: `take-in..`. -func TakeInEvent(kind, key string) (string, error) { +// TakeInEvent is the name of the take-in event of the given kind: `take-in.`. +func TakeInEvent(kind string) (string, error) { if kind != TakeInPending && kind != TakeInTaken { return "", fmt.Errorf("delivery: the take-in kind %q is not pending or taken", kind) } - if err := checkTakeInKey(key); err != nil { - return "", err - } - return TakeInStream + "." + kind + "." + key, nil + return "take-in." + kind, nil } -// ParseTakeInEvent is the inverse of TakeInEvent. -func ParseTakeInEvent(event string) (kind, key string, err error) { - parts := strings.SplitN(event, ".", 3) - if len(parts) != 3 || parts[0] != TakeInStream { - return "", "", fmt.Errorf("delivery: %q is not a take-in event, take-in..", event) +// ParseTakeInEvent is the inverse of TakeInEvent: the kind of a take-in event's name. +func ParseTakeInEvent(event string) (kind string, err error) { + k, ok := strings.CutPrefix(event, "take-in.") + if !ok { + return "", fmt.Errorf("delivery: %q is not a take-in event, take-in.", event) } - if _, err := TakeInEvent(parts[1], parts[2]); err != nil { - return "", "", fmt.Errorf("delivery: %q is not a take-in event: %w", event, err) + if _, err := TakeInEvent(k); err != nil { + return "", fmt.Errorf("delivery: %q is not a take-in event: %w", event, err) } - return parts[1], parts[2], nil + return k, nil } func checkTakeInKey(key string) error { @@ -136,7 +136,10 @@ type TakeIn struct { } func (t TakeIn) check() error { - if _, err := TakeInEvent(t.Kind, t.Key); err != nil { + if _, err := TakeInEvent(t.Kind); err != nil { + return err + } + if err := checkTakeInKey(t.Key); err != nil { return err } if len(t.Merges) == 0 { @@ -213,7 +216,7 @@ func SameTakeIn(a, b TakeIn) bool { return reflect.DeepEqual(a, b) } // merge's key, waiting for the indexer build WaitsFor (empty while it is worked out), or a group being gathered under // its group key, with Members every approved member and Merges those held so far. A revision that waits behind a // group being gathered names that group's key in Behind. Keys are the pending keys of its merges, in Merges' order (indexer_drop_pending drops one). Holds are the modules it holds back: every module of its -// merges' trees. Said is the last take-in event said of it (`take-in..`) and SaidAt when. +// merges' trees. Said is the last take-in event said of it (`take-in.`, of its Key) and SaidAt when. type PendingWait struct { Key string `json:"key"` What string `json:"what"` diff --git a/modules/mesh-delivery-indexer/takein/takein_test.go b/modules/mesh-delivery-indexer/takein/takein_test.go index a37bc4c6..f9a5d6cd 100644 --- a/modules/mesh-delivery-indexer/takein/takein_test.go +++ b/modules/mesh-delivery-indexer/takein/takein_test.go @@ -25,21 +25,21 @@ func sampleTakeIns(since time.Time) []TakeIn { // TestTakeInEvents holds the indexer's take-in contract, moved from mesh-sdk go/delivery with its tests (1585/T2, T12). func TestTakeInEvents(t *testing.T) { - // The take-in events, `take-in..`, a group's key with its dots and no subject wildcard. + // The take-in events, `take-in.`; a group's key with its dots and no subject wildcard, in the body. if PendingKey(42) != "m.00000000000000000042" || GroupTakeInKey("feat/a.b *>") != "g.feat/a.b___" { t.Errorf("take-in keys %q %q", PendingKey(42), GroupTakeInKey("feat/a.b *>")) } - for _, c := range []struct{ kind, key string }{{TakeInPending, PendingKey(7)}, {TakeInTaken, GroupTakeInKey("feat/a.b")}} { - ev, err := TakeInEvent(c.kind, c.key) - if err != nil || ev != "take-in."+c.kind+"."+c.key { + for kind, want := range map[string]string{TakeInPending: TakeInPendingEvent, TakeInTaken: TakeInTakenEvent} { + ev, err := TakeInEvent(kind) + if err != nil || ev != want || ev != "take-in."+kind { t.Fatalf("take-in event = %q, %v", ev, err) } - if k, key, err := ParseTakeInEvent(ev); err != nil || k != c.kind || key != c.key { - t.Errorf("parse take-in event %q = %q %q %v", ev, k, key, err) + if k, err := ParseTakeInEvent(ev); err != nil || k != kind { + t.Errorf("parse take-in event %q = %q %v", ev, k, err) } } - for _, bad := range []string{"", "take-in.pending", "take-in.held.m.1", "take-in.taken.x.1", "take-in.taken.m..1", "forge.taken.m.1"} { - if _, _, err := ParseTakeInEvent(bad); err == nil { + for _, bad := range []string{"", "take-in", "take-in.held", "take-in.pending.m.1", "forge.taken"} { + if _, err := ParseTakeInEvent(bad); err == nil { t.Errorf("parse take-in event %q accepted", bad) } } @@ -74,6 +74,7 @@ func TestTakeInEvents(t *testing.T) { {"take-in without merges", "no merge", 0, func(t *TakeIn) { t.Merges = nil }}, {"take-in without since", "since", 0, func(t *TakeIn) { t.Since = time.Time{} }}, {"a merge of no repository", "owner", 0, func(t *TakeIn) { t.Merges = []Merge{{Repository: "nope", Merge: "m"}} }}, + {"a key that is neither a merge's nor a group's", "neither", 0, func(t *TakeIn) { t.Key = "x.1" }}, } { ti := sampleTakeIns(since)[c.i] c.mut(&ti) diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go index 6be59ed0..443b6455 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go @@ -24,7 +24,6 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" - "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/provisions" @@ -378,7 +377,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro dead.ages() dead.check() provisioned(ctx, js, w, logger.Printf) - // The indexer's stream take-in (ADR 0633 §5): a revision or a group it holds back is a wait, ended by take-in.taken. The + // The indexer's take-in events (ADR 0633 §5): a revision or a group it holds back is a wait, ended by take-in.taken. The // durable consumer _ is made by the controller from this module's consumes, and it is durable: what it // acknowledged before a restart is not heard again. So the waits the indexer holds now are asked of it first, through // its tool indexer_pending (this module's invokes), each with its own since; the consumer then hears what follows. @@ -388,10 +387,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro seedTakeIn(w, calls.Call, logger.Printf) calls.Close() } - switch tc, err := cred.EventConsumer(ctx, js, catalogue.StreamBusName(takein.TakeInStream)); { + // The take-in events are the indexer's ordinary events (its emits), heard on the shared events stream through this + // module's durable consumer, which its consumes filter to them. + switch tc, err := cred.EventConsumer(ctx, js, eventsStream); { case err != nil: logger.Printf("the consumer %s_%s on %s does not exist yet, so no take-in wait is said until the watchman starts again: %v", - cred.Node, cred.Module, catalogue.StreamBusName(takein.TakeInStream), err) + cred.Node, cred.Module, eventsStream, err) default: // The replay has caught up when a message says none is pending after it, or when the consumer had none pending at its // bind (the stream holds nothing it has not acknowledged): the waits past their bound are said then, not before. @@ -473,6 +474,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } } +// eventsStream is the shared stream of the modules' ordinary events, as the bus names it. +const eventsStream = "EVENTS" + // pendingTool is the indexer's tool that answers the waits it holds (ADR 0633 §5), as the view asks it. const pendingTool = delivery.IndexerModule + ".indexer_pending" @@ -491,7 +495,7 @@ func seedTakeIn(w *watch.Watchman, call func(key string, body any) (json.RawMess logf("take-in: %s cannot be asked (%v), so a wait the indexer held before this start is said only once a take-in event of it is heard", pendingTool, err) } -// takeInHeard hands one event of a take-in stream to the watchman, by its name relative to the stream, and acknowledges it; +// takeInHeard hands one take-in event of the indexer to the watchman, by its name relative to the indexer, and acknowledges it; // one that is not a take-in event is said and acknowledged, since hearing it again would not read it. func takeInHeard(w *watch.Watchman, m jetstream.Msg, logf func(string, ...any)) { _, event, ok := strings.Cut(m.Subject(), ".event.") diff --git a/modules/mesh-delivery-watchman/internal/watch/findings_test.go b/modules/mesh-delivery-watchman/internal/watch/findings_test.go index f6802626..e03c00ec 100644 --- a/modules/mesh-delivery-watchman/internal/watch/findings_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/findings_test.go @@ -622,7 +622,7 @@ const ( func takeInEvent(t *testing.T, kind, key string) string { t.Helper() - e, err := takein.TakeInEvent(kind, key) + e, err := takein.TakeInEvent(kind) if err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-watchman/internal/watch/takein.go b/modules/mesh-delivery-watchman/internal/watch/takein.go index 9bb48246..3c3465ff 100644 --- a/modules/mesh-delivery-watchman/internal/watch/takein.go +++ b/modules/mesh-delivery-watchman/internal/watch/takein.go @@ -24,11 +24,11 @@ const ( // takeInRow is the signals table's row of a take-in wait. const takeInRow = "take-in" -// TakeIn hears one event of the indexer's stream `take-in`, by its name relative to the stream (`take-in..`): -// a pending one starts the wait for its key, whose bound runs from the event's since, and a taken one ends it. A group's +// TakeIn hears one take-in event of the indexer, by its name relative to the indexer (`take-in.`), the wait's key +// in its body: a pending one starts the wait for its key, whose bound runs from the event's since, and a taken one ends it. A group's // taken event ends the group's wait only: the revision it becomes is its own key and starts its own, longer wait. func (w *Watchman) TakeIn(event string, body []byte) { - kind, key, err := takein.ParseTakeInEvent(event) + kind, err := takein.ParseTakeInEvent(event) if err != nil { w.say("a take-in event is left: %v", err) return @@ -38,6 +38,11 @@ func (w *Watchman) TakeIn(event string, body []byte) { w.say("the body of %s cannot be read, left: %v", event, err) return } + if t.Kind != kind { + w.say("the body of %s says the kind %q, left", event, t.Kind) + return + } + key := t.Key w.mu.Lock() defer w.mu.Unlock() w.takeIn(kind, key, t) diff --git a/modules/mesh-delivery-watchman/manifest_test.go b/modules/mesh-delivery-watchman/manifest_test.go index 342caead..b6c85b07 100644 --- a/modules/mesh-delivery-watchman/manifest_test.go +++ b/modules/mesh-delivery-watchman/manifest_test.go @@ -97,7 +97,7 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6 t.Errorf("reads = %v", reads) } // 1585/T20: it hears the indexer's stream take-in, whole, through a durable consumer the controller makes from consumes. - if !reflect.DeepEqual(m.Consumes, []string{"mesh-delivery-indexer.take-in.**"}) { + if !reflect.DeepEqual(m.Consumes, []string{"mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken"}) { t.Errorf("consumes = %v", m.Consumes) } // The waits the indexer holds at the watchman's start are asked of its tool (ADR 0633 §5), a verb edge as the view's. diff --git a/modules/mesh-delivery-watchman/module.json b/modules/mesh-delivery-watchman/module.json index 7cdc94d5..ae654b78 100644 --- a/modules/mesh-delivery-watchman/module.json +++ b/modules/mesh-delivery-watchman/module.json @@ -27,7 +27,8 @@ "node-engine.have" ], "consumes": [ - "mesh-delivery-indexer.take-in.**" + "mesh-delivery-indexer.take-in.pending", + "mesh-delivery-indexer.take-in.taken" ], "invokes": [ "mesh-delivery-indexer.indexer_pending"