From 20a7bd0f3c0e5e34fc91766cec1dc2ed6b84f918 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 16:54:26 +0100 Subject: [PATCH 1/3] mesh-delivery-indexer: the restart tests wait for the take-in's end and the bus's close MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TestIndexerCountsOnlyThePlacedBuildNodesToolchains failed on the build node under load (pipeline main.18) in two ways, both of the test, not of the indexer's count of the placed build nodes' toolchains: - it restarted the indexer as soon as the module was written, before its take-in ended: the stop cancelled the take-in, its source-moved stayed unacknowledged, and the branch's durable (one message at a time) held the next one behind it for its acknowledgement wait, past the test's 20 s bound. Each restart now waits for the source-moved's acknowledgement; TestIndexerWhole's restart the same way. - a source-moved said right after a restart could reach the stopped program's pull, still counted until the bus closed its connection. restart now returns only once the bus has closed every connection of the stopped program, heard on the system account's disconnect events, with no sleep. Under load (8 runs side by side, GOMAXPROCS=1) the test failed 5 of 80 runs before, 2 of 240 with the first change alone, and 0 of 600 with both; TestIndexerWhole 0 of 600. No test added. Builds: design 47 H §743 --- .../mesh-delivery-indexer/integration_test.go | 62 ++++++++++++++++++- .../takein_integration_test.go | 5 ++ 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index 27b82a53..88112068 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -49,10 +49,12 @@ import ( ) // busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the -// controller's user and the indexer's own principal .. +// controller's user and the indexer's own principal .; and the bus's system account, whose disconnect events +// tell a restart when the stopped indexer's connections are gone (restartWaits). const busConf = ` listen: 127.0.0.1:-1 jetstream { store_dir: %q } +system_account: SYS accounts { MESH { jetstream: enabled @@ -61,6 +63,11 @@ accounts { { user: "n1.mesh-delivery-indexer", password: "pw" } ] } + SYS { + users = [ + { user: "sys", password: "pw" } + ] + } } ` @@ -420,12 +427,54 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before . } start() t.Cleanup(func() { stop() }) - // restart stops the program and starts it again on the same bus, as the node-engine does with a new build. - restart := func() { stop(); start() } + // restart stops the program and starts it again on the same bus, as the node-engine does with a new build, once the bus has + // closed every connection of the stopped one (restartWaits). + gone := restartWaits(t, srv) + restart := func() { gone(stop); start() } return js, ctx, candidates, modules, restart } +// restartWaits gives a stop that returns only once the bus has closed every connection the indexer held, heard on the system +// account's disconnect events, never by sleeping. A program's stop closes its connections on its side alone; until the bus +// has closed them its pulls on a durable still count, and a source-moved said in between is delivered to the stopped +// program, unacknowledged for the durable's whole acknowledgement wait, while the branch's durable delivers one message +// at a time (design 47 H §743). +func restartWaits(t *testing.T, srv *server.Server) func(stop func()) { + t.Helper() + sys, err := nats.Connect(srv.ClientURL(), nats.UserInfo("sys", "pw")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(sys.Close) + return func(stop func()) { + t.Helper() + disconnects, err := sys.SubscribeSync("$SYS.ACCOUNT.MESH.DISCONNECT") + if err != nil { + t.Fatal(err) + } + defer func() { _ = disconnects.Unsubscribe() }() + if err := sys.Flush(); err != nil { + t.Fatal(err) + } + open, err := srv.Connz(&server.ConnzOptions{User: "n1.mesh-delivery-indexer"}) + if err != nil { + t.Fatal(err) + } + stop() + for left := open.NumConns; left > 0; { + m, err := disconnects.NextMsg(waitBound) + if err != nil { + t.Fatalf("the bus closed %d of the stopped indexer's %d connections within %s: %v", open.NumConns-left, open.NumConns, waitBound, err) + } + var ev server.DisconnectEventMsg + if json.Unmarshal(m.Data, &ev) == nil && ev.Client.User == "n1.mesh-delivery-indexer" { + left-- + } + } + } +} + // The build seats and their holders as the mesh has them (design 47:482-485 and 1583: a language's build module is the // module claiming the seat whose `acts` is step build on that language): the seat definitions as modules/mesh-delivery-contracts/seats// // seat.json has them (acts included) and the manifests as the catalog's modules/mesh-build-go and mesh-build-node claim them. @@ -691,6 +740,7 @@ func TestIndexerBuildsARunsCandidateAndDropsItWhenEveryMembersPipelineEnded(t *t func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) { const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381" var assignments, toolchains jetstream.KeyValue + var heard *takeIns place := func(ctx context.Context, t *testing.T, node string, modules ...string) { p := meshmoduleassigner.Assignments{Node: node} for _, m := range modules { @@ -712,6 +762,7 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) { assignerLate = up t.Cleanup(func() { assignerLate = nil }) js, ctx, _, modules, restart := startIndexerRun(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) { + heard = hearTakeIns(t, js.Conn()) registerAvahi(t, ctx, js, "main") kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.ToolchainsBucket("mesh-build-go"), History: 1}) if err != nil { @@ -762,6 +813,10 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) { if !agreed(held) { t.Fatalf("within 20 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27 alone", held.Fp2, held.None2, held.Toolchains) } + // The module is written before its take-in ends: each restart below waits for the source-moved's acknowledgement, the + // take-in's end (design 47 H §743), so the stop cancels no take-in and leaves none unacknowledged ahead of the next one + // on the branch's durable, which delivers one message at a time. + heard.acked(t, "main") place(ctx, t, "n2", "mesh-build-go") key, _ := index.ToolchainKey("n2", "go") value, err := index.EncodeToolchain(nodebuildgo.Toolchain{Language: "go", Version: "go1.28.1", Recipe: "1", At: ptr(time.Now().Add(time.Second))}) @@ -789,6 +844,7 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) { if m := read(disagree); !disagree(m) { t.Fatalf("within 20 s of a merge after n2's assignments avahi has fp2 %q, none2 %q: want the builders to disagree", m.Fp2, m.None2) } + heard.acked(t, "main") // The view moves only at a take-in (design 47 H-A-49, H-A-103): with n2's assignments taken away, the next take-in agrees. place(ctx, t, "n2") restart() diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go index 889e7cdc..98cc851a 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go @@ -457,6 +457,11 @@ func TestIndexerWhole(t *testing.T) { } sayPipeline(t, ctx, js, false, "main", 12) untilKey(t, ctx, pins, index.PinKey("main", 12), pinned("pipeline.main.12")) + // The restart waits for the take-ins' ends, their source-moved acknowledged (design 47 H §743: main 10, 11, 11 given + // again and 12), so the stop cancels none and leaves none unacknowledged ahead of main 13. + for range 4 { + heard.acked(t, "main") + } restart() for _, name := range []string{"main.10", "main.11"} { if _, _, err := index.ReadRevision(ctx, objects, name); err != nil { -- 2.54.0 From 5dba7fe68bf5647c121f4d11607c85174b36991e Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 17:00:55 +0100 Subject: [PATCH 2/3] mesh-catalog's revision names its base; validate's case before reads it, not the number before MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A take-in that moves nothing keeps no revision for its source number, so validate's case before, read as the step's revision minus one, named a revision never kept and could not run. The take-in now writes, on the catalog revision it keeps, the field base: the newest kept revision below it, the one it pins for its pipeline, in the same write; absent on the branch's first take-in. The assigner's validate reads the base from the revision it already asks for; where the revision names none the case before is empty and its end says so, never a could-not-run. - mesh-delivery-contracts: the seat mesh-catalog's CatalogRevision gains base (schema, fixture, generated Go and TypeScript by seatgen). - mesh-delivery-indexer: keepBranchEntries names the base it pins; the pin test's rows check it. - mesh-delivery-assigner: before reads the base; the validate test asks a revision with none, and both stand-ins name the base as the take-in does. Builds: design 47 H-A-151, H-A-154 (hq design/47-H1-answers-59 01842bbe), H-A-147, H-A-108 (3), design 47 H §743 --- .../integration_test.go | 5 +++ .../mesh-delivery-assigner/validatestep.go | 44 ++++++++++++------- .../validatestep_test.go | 21 ++++++++- .../go/seats/meshcatalog/contract.go | 2 + .../go/seats/meshcatalog/contract_test.go | 2 +- .../seats/mesh-catalog/fixtures/states.json | 3 +- .../seats/mesh-catalog/seat.json | 4 ++ .../ts/seats/meshcatalog/index.d.ts | 2 + .../internal/index/catalog_revisions.go | 10 ++++- .../internal/index/pins_test.go | 11 +++-- 10 files changed, 80 insertions(+), 24 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go index 93781e6d..8a26a659 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go @@ -1769,6 +1769,11 @@ func serveCatalog(t *testing.T, b *bed, nc *nats.Conn) { } if kept, ok := b.cat.at[branch][in.Revision]; ok { rev := meshcatalog.CatalogRevision{Kind: "trunk", Branch: in.Branch, Revision: in.Revision} + for k := range b.cat.at[branch] { // its base, the newest kept below it, as the take-in names it (H-A-154) + if k < in.Revision && k > rev.Base { + rev.Base = k + } + } for _, e := range kept { rev.Entries = append(rev.Entries, meshcatalog.EntryRef{Module: e.Module, Digest: e.ManifestDigest}) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep.go index 2b2a4dbb..80082173 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep.go @@ -22,7 +22,7 @@ import ( // `.validate` of the state composition-runs in the CompositionRun form with no machine, which every mover's step // reads: the cycle the revision's entries make (graph.RefusedCycle) ends it at once; else the manifest validator built from // the revision's own entry of the module holding the seat mesh-catalog, on the Go build seat as the composer is, runs in the -// sandbox over the revision's manifests and the branch's previous revision's, its input kept and served on mesh-wants' input. +// sandbox over the revision's manifests and its base's (H-A-154), its input kept and served on mesh-wants' input. // Every mover's step ends the same: failed `cycle` naming the cycle, failed `manifest-refused` naming every problem of a // manifest the revision alters or between manifests one it alters takes part in, or passed. @@ -32,7 +32,7 @@ const ( validatorPackage = "mesh-delivery-indexer/cmd/mesh-judge" // validateKeySuffix ends the key of a revision's validate run, `.validate` (H-A-108). validateKeySuffix = "." + meshwants.StepRunStepValidate - // beforeCase is the validator's case of the branch's previous revision, beside changeCase (H-A-108 (3)). + // beforeCase is the validator's case of the revision's base, beside changeCase (H-A-108 (3), H-A-151, H-A-154). beforeCase = "before" // The reasons a validate step fails with, the first word of its said (§745). reasonManifestRefused = "manifest-refused" @@ -112,16 +112,29 @@ func (r *compositionRun) judgeValidate(ctx context.Context, step meshwants.StepR if err != nil { return couldNotRun("the step's catalog revision is not read: " + err.Error()) } - before, err := r.before(ctx, step) + before, err := r.before(ctx, step, half) if err != nil { - return couldNotRun("the branch's previous catalog revision is not read: " + err.Error()) + return couldNotRun("the base of the case before is not read: " + err.Error()) + } + // A revision that names no base (the branch's first take-in) is judged against an empty case before, and says so (H-A-151). + empty := "" + if half.Revision.Base <= 0 { + empty = emptyBeforeWords } if problems := causedProblems(v.Change, v.Before, altered(half, before)); len(problems) > 0 { - return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: reasonManifestRefused + ": " + strings.Join(problems, "; ")} + said := reasonManifestRefused + ": " + strings.Join(problems, "; ") + if empty != "" { + said += " (" + empty + ")" + } + return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: said} } - return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed} + return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed, Said: empty} } +// emptyBeforeWords say, in a validate step's end, that its revision names no base: the branch's first take-in, judged against +// an empty case before (H-A-151, H-A-154). +const emptyBeforeWords = "the case before is empty: the revision names no base, the branch's first take-in" + // validateVerdict is a validate run's verdict as the run keeps it: the cycle the revision makes, or the validator's two cases. type validateVerdict struct { Cycle []meshcatalog.ArtifactRef `json:"cycle,omitempty"` @@ -148,15 +161,16 @@ func cycleWords(cycle []meshcatalog.ArtifactRef) string { return strings.Join(words, " → ") } -// before is the branch's catalog revision before the step's, read through the seat mesh-catalog's verb revision (H-A-108 (3)); -// none before the first. -func (r *compositionRun) before(ctx context.Context, step meshwants.StepRun) (mcc.Half, error) { - if step.Revision <= 1 { +// before is the case before of the step's revision half: the revision its base names, the newest kept revision below it that +// its take-in pinned, read through the seat mesh-catalog's verb revision (H-A-108 (3), H-A-151, H-A-154); empty where the +// revision names no base, the branch's first take-in. +func (r *compositionRun) before(ctx context.Context, step meshwants.StepRun, half mcc.Half) (mcc.Half, error) { + if half.Revision.Base <= 0 { return mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{}}, nil } - prev := step - prev.Revision-- - return r.revision(ctx, prev) + base := step + base.Revision = half.Revision.Base + return r.revision(ctx, base) } // altered are the modules whose manifest the revision adds or changes against the one before it. @@ -270,9 +284,9 @@ func (r *compositionRun) startValidate(ctx context.Context, key string, run mesh end(meshwants.CompositionRunOutcomePassed, "", verdict) return } - before, err := r.before(ctx, step) + before, err := r.before(ctx, step, half) if err != nil { - couldNot("the branch's previous catalog revision is not read: " + err.Error()) + couldNot("the base of the case before is not read: " + err.Error()) return } entry, ok := catalogEntry(half.Entries) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep_test.go index 13e6d409..6ca9cb01 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep_test.go @@ -20,7 +20,9 @@ import ( // run, `.validate` of the state composition-runs; a manifest the revision alters that names an unknown field fails // it `manifest-refused` naming the problem, a problem of a manifest it leaves as it was does not, and the validator is the // revision's own entry of the module holding the seat mesh-catalog, on the Go build seat; a revision whose entries make a cycle -// fails it `cycle` naming the cycle, with no validator asked. +// fails it `cycle` naming the cycle, with no validator asked. The case before is the revision its base names (H-A-151, H-A-154: +// the newest kept below it, which the stand-in names as the indexer does); a revision naming no base, the branch's first take-in, +// is judged against an empty case before, said in its end, and no revision below it is asked. func TestTheValidateStepRefusesAManifestProblemAndACycle(t *testing.T) { indexer := meshcatalog.CatalogEntry{Module: "indexer", Repository: "novox/mesh-delivery", Commit: commitE3, Path: "modules/mesh-delivery-indexer", Fp: "src2:ix", Serves: []string{"seat:" + meshcatalog.Seat + ".revision"}, Manifest: map[string]any{"module": "indexer"}, ManifestDigest: "d-ix"} @@ -31,6 +33,16 @@ func TestTheValidateStepRefusesAManifestProblemAndACycle(t *testing.T) { 6: {indexer, entry("web", "d-web-1"), entry("db", "d-db")}, 7: {indexer, entry("web", "d-web-2"), entry("db", "d-db")}, 9: {indexer, entry("a", "d-a", meshcatalog.Edge{From: "a:x", To: "b:y", Kind: 3}, meshcatalog.Edge{From: "b:y", To: "a:x", Kind: 3}), entry("b", "d-b")}, + 3: {indexer}, + } + // base is the newest kept revision below n, as the take-in names it; none below the first. + base := func(n int64) (b int64) { + for k := range revisions { + if k < n && k > b { + b = k + } + } + return b } call := func(key string, body any) (json.RawMessage, error) { raw, _ := json.Marshal(body) @@ -42,7 +54,7 @@ func TestTheValidateStepRefusesAManifestProblemAndACycle(t *testing.T) { if !ok || in.Branch != "main" { return nil, errors.New("asked another revision") } - return json.Marshal(meshcatalog.RevisionAnswer{Revision: &meshcatalog.CatalogRevision{Revision: in.Revision, + return json.Marshal(meshcatalog.RevisionAnswer{Revision: &meshcatalog.CatalogRevision{Revision: in.Revision, Base: base(in.Revision), Definitions: []meshcatalog.CatalogRevisionDefinitionsItem{{Seat: "s1", Digest: "sd"}}}, Entries: entries}) case seatDefinitionsKey: return json.Marshal(meshcatalog.SeatDefinitionsAnswer{Found: true, Entry: &meshcatalog.SeatDefinitionsAnswerEntry{Seat: "s1", Digest: "sd", Definition: map[string]any{"seat": "s1"}}}) @@ -117,4 +129,9 @@ func TestTheValidateStepRefusesAManifestProblemAndACycle(t *testing.T) { if o.Outcome != meshwants.StepEndedOutcomeFailed || o.Said != reasonCycle+": a:x → b:y → a:x" || len(asks) != 1 { t.Errorf("a revision making a cycle ended %+v (%d validator asks)", o, len(asks)) } + // The stand-in validator answers its one verdict: judged against the empty case before, web's problem is the revision's. + if o := r.validate(ctx, "ask-first", step("indexer", 3)); o.Outcome != meshwants.StepEndedOutcomeFailed || + o.Said != reasonManifestRefused+`: web: unknown field "colour" (`+emptyBeforeWords+")" || len(asks) != 2 || len(requests[1].Cases[1].Manifests) != 0 { + t.Errorf("a revision naming no base ended %+v (%d validator asks), want it judged against an empty case before, said", o, len(asks)) + } } diff --git a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go index 38996dd9..261c6f66 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go @@ -711,6 +711,8 @@ type CatalogRevisionDefinitionsItem struct { type CatalogRevision struct { // when (RFC 3339 time) At *time.Time `json:"at,omitempty"` + // the newest kept revision of its branch below it, which the take-in that wrote it pinned for its pipeline: the base of the validate step's case before (design 47 H-A-147, H-A-151, H-A-154); absent on the branch's first take-in + Base int64 `json:"base,omitempty"` // the watched branch the revision is of Branch string `json:"branch,omitempty"` // the seat definitions it was judged with, each by its digest, read whole through seat-definitions diff --git a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract_test.go b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract_test.go index 88198926..e8f85fa8 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract_test.go +++ b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract_test.go @@ -59,7 +59,7 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v ModulesAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "the module verb (captured: xclip)", subject: "", given: "{\"module\":{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"manifest\":{\"build\":{\"artifacts\":[{\"binary\":\"xclip-tools\",\"from\":\"cmd/xclip-tools\",\"kind\":\"bundle\",\"language\":\"go\",\"loads\":[\"xclip-tools\"],\"name\":\"tools\",\"system\":\"arch\"}]},\"capabilities\":[\"package-manager\"],\"module\":\"xclip\",\"resources\":[{\"id\":\"package\",\"package\":\"xclip\",\"type\":\"package\"}],\"tools\":[\"xclip_copy\",\"xclip_paste\",\"xclip_targets\",\"xclip_session\"],\"version\":\"1\"},\"module\":\"xclip\",\"path\":\"modules/xclip\",\"repository\":\"novox/mesh-catalog\"}}", wire: "{\"module\":{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"manifest\":{\"build\":{\"artifacts\":[{\"binary\":\"xclip-tools\",\"from\":\"cmd/xclip-tools\",\"kind\":\"bundle\",\"language\":\"go\",\"loads\":[\"xclip-tools\"],\"name\":\"tools\",\"system\":\"arch\"}]},\"capabilities\":[\"package-manager\"],\"module\":\"xclip\",\"resources\":[{\"id\":\"package\",\"package\":\"xclip\",\"type\":\"package\"}],\"tools\":[\"xclip_copy\",\"xclip_paste\",\"xclip_targets\",\"xclip_session\"],\"version\":\"1\"},\"module\":\"xclip\",\"path\":\"modules/xclip\",\"repository\":\"novox/mesh-catalog\"}}", decode: func(b []byte) (any, error) { var v ModuleAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, - {name: "a watched branch's catalog graph revision", subject: "", given: "{\"at\":\"2026-11-09T19:00:00Z\",\"branch\":\"main\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1}", wire: "{\"at\":\"2026-11-09T19:00:00Z\",\"branch\":\"main\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1}", + {name: "a watched branch's catalog graph revision", subject: "", given: "{\"at\":\"2026-11-09T19:00:00Z\",\"base\":9,\"branch\":\"main\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1}", wire: "{\"at\":\"2026-11-09T19:00:00Z\",\"base\":9,\"branch\":\"main\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1}", decode: func(b []byte) (any, error) { var v CatalogRevisionsByBranchRevisionValue err := json.Unmarshal(b, &v) diff --git a/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/states.json b/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/states.json index 96c230ce..e537ca82 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/states.json +++ b/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/states.json @@ -208,7 +208,8 @@ "fingerprint": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "repository": "novox/mesh-catalog" } - ] + ], + "base": 9 } }, { diff --git a/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json b/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json index a6d62b1a..e9db48fc 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json @@ -1927,6 +1927,10 @@ "$ref": "#/defs/TakeInEntry" }, "description": "the movers of the take-in that wrote it, so a redelivered source-moved whose revision is kept says take-in.taken again from them, recomputing nothing (design 47 H §743, H-A-92); absent for one with no mover" + }, + "base": { + "type": "integer", + "description": "the newest kept revision of its branch below it, which the take-in that wrote it pinned for its pipeline: the base of the validate step's case before (design 47 H-A-147, H-A-151, H-A-154); absent on the branch's first take-in" } }, "required": [ diff --git a/modules/mesh-delivery-contracts/ts/seats/meshcatalog/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshcatalog/index.d.ts index eaf806c4..b3d0c697 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshcatalog/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshcatalog/index.d.ts @@ -273,6 +273,8 @@ export interface CatalogRevisionDefinitionsItem { export interface CatalogRevision { /** when (RFC 3339 time) */ "at"?: string; + /** the newest kept revision of its branch below it, which the take-in that wrote it pinned for its pipeline: the base of the validate step's case before (design 47 H-A-147, H-A-151, H-A-154); absent on the branch's first take-in */ + "base"?: number; /** the watched branch the revision is of */ "branch"?: string; /** the seat definitions it was judged with, each by its digest, read whole through seat-definitions */ diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index b101c8ea..8addf0a8 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -305,6 +305,12 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int return err } rev.Movers = movers + // The base of the validate step's case before: the newest revision kept before this write, which the take-in pins for its + // pipeline below, named on the revision in this same write (H-A-147, H-A-151, H-A-154); none on the branch's first take-in. + pinsBase := len(movers) > 0 && ix.Pins != nil && newest.Revision > 0 && newest.Revision != src + if pinsBase { + rev.Base = newest.Revision + } changed := changedEntries(newest.Entries, rev.Entries) used, capacity, fillErr := ix.Revisions.Fill(ctx) if fillErr == nil && capacity > 0 && used+newBytes(ctx, ix.Revisions, rev, objects) > capacity { @@ -332,8 +338,8 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int } // The branch's previous revision, the newest kept before this write, pinned for the same pipeline: it is the base of // the validate step's case before (H-A-108 (3), H-A-147), kept as long as the pipeline that needs it. - if newest.Revision > 0 && newest.Revision != src { - if err := ix.pin(ctx, PinKey(branch, newest.Revision), record); err != nil { + if pinsBase { + if err := ix.pin(ctx, PinKey(branch, rev.Base), record); err != nil { ix.logf("pins: the previous revision %s is not pinned for %s: %v", BranchObject(branch, newest.Revision), record, err) return fmt.Errorf("the pin of %s: %w", BranchObject(branch, newest.Revision), err) } diff --git a/modules/mesh-delivery-indexer/internal/index/pins_test.go b/modules/mesh-delivery-indexer/internal/index/pins_test.go index 4bcc9dab..a2f89cc6 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pins_test.go @@ -106,7 +106,8 @@ func mustRecords(t *testing.T, v []byte) []string { // H-A-147 (§743, §749, H-A-108 (3)): a take-in with a mover pins, in its write, the branch's previous revision (the newest kept // before the write) for the same pipeline record as its own, so the validate step's case before can read it while the pipeline -// runs; a take-in with no revision before it pins only its own; the pipeline's end frees both. +// runs, and names it as its base on the revision it writes (H-A-151, H-A-154); a take-in with no revision before it pins only +// its own and names no base; the pipeline's end frees both. func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) { ctx := context.Background() reader := meshcatalog.CatalogEntry{Module: "reader", Repository: "novox/r", Manifest: map[string]any{}} @@ -116,9 +117,10 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) { before []int64 // sources taken in with no mover before the take-in of source 5 whileRuns []string afterTheEnd []string + base int64 // the base main.5 names (H-A-154) }{ - {name: "the previous revision kept and pinned", before: []int64{3, 4}, whileRuns: []string{"main.4", "main.5"}, afterTheEnd: []string{"main.5"}}, - {name: "the newest kept before, whatever its number", before: []int64{2}, whileRuns: []string{"main.2", "main.5"}, afterTheEnd: []string{"main.5"}}, + {name: "the previous revision kept and pinned", before: []int64{3, 4}, whileRuns: []string{"main.4", "main.5"}, afterTheEnd: []string{"main.5"}, base: 4}, + {name: "the newest kept before, whatever its number", before: []int64{2}, whileRuns: []string{"main.2", "main.5"}, afterTheEnd: []string{"main.5"}, base: 2}, {name: "no revision before it pins only its own", whileRuns: []string{"main.5"}, afterTheEnd: []string{"main.5"}}, } { t.Run(tc.name, func(t *testing.T) { @@ -143,6 +145,9 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) { keep(s, nil) } keep(5, mover) + if rev, _, err := ReadRevision(ctx, ix.Revisions, "main.5"); err != nil || rev.Base != tc.base { + t.Errorf("main.5 names the base %d (%v), want %d, the revision its take-in pinned (H-A-154)", rev.Base, err, tc.base) + } if got := kept(); !slices.Equal(got, tc.whileRuns) { t.Errorf("while the pipeline runs: kept %v, want %v", got, tc.whileRuns) } -- 2.54.0 From d281faeeb76a99a1aea0841f64466fee645cfb55 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 17:08:23 +0100 Subject: [PATCH 3/3] mesh-delivery-assigner: the validate test judges a revision whose base is below its number minus one Every revision the validate test judged with a base had it at its number minus one, so the test passed with the case before read as the number before (review round 1). Revision 12 now names base 7 with 8 to 11 never kept: it ends manifest-refused against base 7's manifests, and could not run when the case before is read as the number minus one. No test added. Builds: design 47 H-A-154 (hq design/47-H1-answers-59 01842bbe), H-A-151 --- .../validatestep_test.go | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep_test.go index 6ca9cb01..7d3575d9 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep_test.go @@ -30,13 +30,18 @@ func TestTheValidateStepRefusesAManifestProblemAndACycle(t *testing.T) { return meshcatalog.CatalogEntry{Module: module, Manifest: map[string]any{"module": module, "d": digest}, ManifestDigest: digest, Edges: edges} } revisions := map[int64][]meshcatalog.CatalogEntry{ - 6: {indexer, entry("web", "d-web-1"), entry("db", "d-db")}, - 7: {indexer, entry("web", "d-web-2"), entry("db", "d-db")}, - 9: {indexer, entry("a", "d-a", meshcatalog.Edge{From: "a:x", To: "b:y", Kind: 3}, meshcatalog.Edge{From: "b:y", To: "a:x", Kind: 3}), entry("b", "d-b")}, - 3: {indexer}, + 6: {indexer, entry("web", "d-web-1"), entry("db", "d-db")}, + 7: {indexer, entry("web", "d-web-2"), entry("db", "d-db")}, + 9: {indexer, entry("a", "d-a", meshcatalog.Edge{From: "a:x", To: "b:y", Kind: 3}, meshcatalog.Edge{From: "b:y", To: "a:x", Kind: 3}), entry("b", "d-b")}, + 3: {indexer}, + 12: {indexer, entry("web", "d-web-2"), entry("db", "d-db")}, } - // base is the newest kept revision below n, as the take-in names it; none below the first. + // base is the newest kept revision below n, as the take-in names it; none below the first. Revision 12 names 7 with 8 to 11 + // never kept, so a case before read as the number minus one could not run (H-A-154). base := func(n int64) (b int64) { + if n == 12 { + return 7 + } for k := range revisions { if k < n && k > b { b = k @@ -134,4 +139,12 @@ func TestTheValidateStepRefusesAManifestProblemAndACycle(t *testing.T) { o.Said != reasonManifestRefused+`: web: unknown field "colour" (`+emptyBeforeWords+")" || len(asks) != 2 || len(requests[1].Cases[1].Manifests) != 0 { t.Errorf("a revision naming no base ended %+v (%d validator asks), want it judged against an empty case before, said", o, len(asks)) } + // Revision 12 holds revision 7's manifests and names base 7: judged against it, web's problem is the revision's, db's old one + // is not, and the case before holds base 7's manifests. + if o := r.validate(ctx, "ask-base", step("web", 12)); o.Outcome != meshwants.StepEndedOutcomeFailed || + o.Said != reasonManifestRefused+`: web: unknown field "colour"` || len(asks) != 3 { + t.Errorf("a revision naming base 7 below its number ended %+v (%d validator asks), want it judged against base 7", o, len(asks)) + } else if web := requests[2].Cases[1].Manifests["web"]; len(requests[2].Cases[1].Manifests) != 3 || !strings.Contains(string(web.Raw), "d-web-2") { + t.Errorf("the case before of revision 12 is %+v, want base 7's manifests", requests[2].Cases[1].Manifests) + } } -- 2.54.0