From 9768d0b67d1107cd15abd839c9493050fcabc037 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 20:20:45 +0100 Subject: [PATCH 01/20] A-476: the take-in asks no validation, holds for no indexer build, and moves only its repository's modules (a) validation_wait.go and validate.go go with the validation ask, its end consumer, indexer_validate_again and Pending.Validation; (b) evaluate always takes the revision in with the build the program runs (holdFor and the build comparison go); (c) a merge's recompute writes only the modules registered for its repository. Builds: design 47 G A-476 (a), (b), (c) --- .../cmd/mesh-delivery-indexer-tools/main.go | 4 - .../mesh-delivery-indexer/integration_test.go | 147 +------- .../cmd/mesh-delivery-indexer/main.go | 6 +- .../packages_integration_test.go | 9 + .../cmd/mesh-delivery-indexer/takein.go | 4 - .../cmd/mesh-delivery-indexer/versions.go | 11 - .../internal/index/candidates.go | 7 - .../internal/index/fingerprints.go | 2 +- .../internal/index/modules.go | 2 +- .../internal/index/pending.go | 154 ++------ .../internal/index/pending_test.go | 231 ++++-------- .../internal/index/pending_tools.go | 31 -- .../internal/index/takein.go | 17 - .../internal/index/toolchains.go | 23 +- .../internal/index/toolchains_test.go | 9 + .../internal/index/validate.go | 264 -------------- .../internal/index/validate_test.go | 130 ------- .../internal/index/validation_wait.go | 329 ------------------ .../internal/index/validation_wait_test.go | 113 ------ .../mesh-delivery-indexer/manifest_test.go | 5 +- modules/mesh-delivery-indexer/module.json | 3 +- .../mesh-delivery-indexer/takein/takein.go | 15 - 22 files changed, 147 insertions(+), 1369 deletions(-) delete mode 100644 modules/mesh-delivery-indexer/internal/index/validate.go delete mode 100644 modules/mesh-delivery-indexer/internal/index/validate_test.go delete mode 100644 modules/mesh-delivery-indexer/internal/index/validation_wait.go delete mode 100644 modules/mesh-delivery-indexer/internal/index/validation_wait_test.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index 664447b7..12599d11 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -318,10 +318,6 @@ func main() { })...) tools = append(tools, viewTools(os.Getenv, trunk.State, time.Now)...) tools = append(tools, index.PendingTools(keptRead{stdio.State(index.PendingState)}, keptRead{stdio.State(index.OwnBuildStateName)}, asks, time.Now)...) - tools = append(tools, index.ValidationTools(asks, time.Now, func(what, why string) error { - _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.ValidateAgainTool, Verb: "validate-again"}) - return err - })...) tools = append(tools, index.FingerprintsTools(asks, time.Now, func(what, why string) error { _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.FingerprintsAgainTool, Verb: "fingerprints-again"}) return err diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index b11129a2..95e55326 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -22,7 +22,6 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" @@ -97,11 +96,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 // The forge bridge's tool bridge_merged_message (issue 1585): answers the merged message published for a merge, by // repository@merge, as the bridge does; a restarted indexer holds none in memory and asks it. var bridgeMessages sync.Map - // The build seat node-build-go's stand-in (design 47 G §712, §713): it hears the indexer's validation asks and ends each with - // the manifest validator's verdict; refusing names, by the commit the job runs at, the manifest a revision's change refuses. - var validations *standIn - var refusing sync.Map - js, ctx, candidates, _, restart := startIndexerRun(t, func(ctx context.Context, builds buildLog) { + js, ctx, candidates, _, _ := startIndexerRun(t, func(ctx context.Context, builds buildLog) { // The builder's log entries of two builds' ends (ADR 0571 clause 1), as the builder appends them. builds.end(t, "novox/mesh-delivery", "a", "modules/a") builds.end(t, "novox/mesh-catalog", "c", "modules/c") @@ -128,9 +123,8 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 if _, err := tc.Put(ctx, key, value); err != nil { t.Fatal(err) } - // The trunk holds the indexer's entry, taken in at an earlier commit with its own manifest; the validation names no Go module (A-268). + // The trunk holds the indexer's entry, taken in at an earlier commit with its own manifest; the take-in asks no validation (A-476 a). seedBuildModules(t, ctx, js, map[string]meshcatalog.CatalogEntry{testIndexer: trunkIndexerEntry(t, ctx, js)}) - validations = standInBuildSeat(t, ctx, js, &refusing) // The node-engine's composition on n1 (process.Conn.Call's subjects): the build of the indexer that runs there. answer := func(m *nats.Msg) { raw, _ := json.Marshal(map[string]any{"result": map[string]any{"runs": map[string]string{"fp": *running.Load()}}, "node": "n1"}) @@ -281,17 +275,13 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 } return takein.TakeIn{}, false } - held, ok := said(takein.TakeInPending, m1Key) - if !ok || held.What != takein.TakeInRevision || held.WaitsFor == "" || held.WaitsFor == first || len(held.Merges) != 1 { - t.Fatalf("take-in.pending.%s = %+v (said %v) within 20 s, want the revision waiting for another build than %s", m1Key, held, ok, first) + // A-476 a, b: the revision changing the indexer's own tree is taken in at once, with the build this program runs and no + // validation ask: it is never said pending, and the build the revision names is not waited for. + if taken, ok := said(takein.TakeInTaken, m1Key); !ok || len(taken.Merges) != 1 || taken.Merges[0].Merge != m1.Merge { + t.Fatalf("take-in.taken.%s = %+v (said %v) within 20 s, want the revision taken in at once", m1Key, taken, ok) } - named := held.WaitsFor - // IX5 (§712): the revision was validated first, by one run ask whose job names the indexer entry's repository, path and - // the revision's head, and no heads; accepted, it is never asked again while it waits for the named indexer. - if asks := validations.asked(); len(asks) != 1 || asks[0].Run == nil || asks[0].Kind != sdkstep.AskCheck || - asks[0].Run.Program.Repository != m1.Repository || asks[0].Run.Program.Dir != "modules/"+ix || asks[0].Run.Program.Commit != m1.Merge || - len(asks[0].Run.Heads) != 0 || asks[0].Run.Program.Package != "./cmd/mesh-judge" { - t.Fatalf("validation asks = %+v, want one run of the indexer's manifest validator at %s", asks, m1.Merge) + if held := saidAll(takein.TakeInPending, m1Key); len(held) != 0 { + t.Errorf("the revision was said pending: %+v", held) } get := func(bucket, key string) ([]byte, bool) { t.Helper() @@ -312,11 +302,10 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 } return e.Value(), true } - // Only what building and moving the named indexer needs is written. raw, ok := get(index.ModulesState, ix) own, err := meshcatalogclient.DecodeModule(raw) - if !ok || err != nil || own.Fp2 != named || own.Commit != m1.Merge { - t.Fatalf("modules.%s = %+v (%v), want the named build %s at the merge", ix, own, err, named) + if !ok || err != nil || own.Commit != m1.Merge { + t.Fatalf("modules.%s = %+v (%v), want the indexer at the merge", ix, own, err) } if _, ok := get(index.Bucket(testIndexer, index.ManifestsState), own.Manifest); !ok { t.Errorf("manifests.%s is not written", own.Manifest) @@ -325,16 +314,11 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 t.Errorf("the verdict on the indexer's manifest %s is not written", own.Manifest) } repoKey, _ := trunk.RepositoryKey(m1.Repository) - if _, ok := get(index.ModulesState, "a"); ok { - t.Errorf("modules.a is written while the revision is pending") + if raw, ok := get(index.ModulesState, "a"); !ok || !strings.Contains(string(raw), m1.Merge) { + t.Errorf("modules.a = %s, want it at the merge %s", raw, m1.Merge) } - if _, ok := get(index.Bucket(testIndexer, index.TakenState), repoKey); ok { - t.Errorf("taken.%s is written while the revision is pending", repoKey) - } - if e, err := candidates.Get(ctx, "feat/x"); err != nil { - t.Errorf("the candidate feat/x is gone while the merge of #12 is pending: %v", err) - } else if c, err := index.DecodeCandidate(e.Value()); err != nil || len(c.Pulls) != 2 { - t.Errorf("the candidate feat/x = %+v while the merge of #12 is pending, want its two pull requests", c) + if _, ok := get(index.Bucket(testIndexer, index.TakenState), repoKey); !ok { + t.Errorf("taken.%s is not written", repoKey) } // A group's first member arrives meanwhile; its end is still to come (the knowledge base's remark: it stays across the restart). @@ -365,34 +349,13 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 } var w []takein.PendingWait for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { - if w = waits(); len(w) == 2 && w[0].WaitsFor == named && w[1].Said != "" { + if w = waits(); len(w) == 1 && w[0].Said != "" { break } } - if len(w) != 2 || w[0].Key != m1Key || w[0].WaitsFor != named || !slices.Contains(w[0].Holds, "a") || !slices.Contains(w[0].Holds, ix) || - w[1].Key != groupKey || w[1].What != takein.TakeInGroup || len(w[1].Merges) != 1 || w[1].Said != takein.TakeInPendingEvent { - t.Fatalf("indexer_pending = %+v, want the revision %s waiting for %s holding a and the indexer, then the group gathered", w, m1Key, named) - } - - // The named indexer starts (the node-engine now runs that build): it takes the revision in, and leaves the group. - running.Store(&named) - restart() - if taken, ok := said(takein.TakeInTaken, m1Key); !ok || len(taken.Merges) != 1 || taken.Merges[0].Merge != m1.Merge { - t.Fatalf("take-in.taken.%s = %+v (said %v) within 20 s of the start of %s", m1Key, taken, ok, named) - } - for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { - if w = waits(); len(w) == 1 { - break - } - } - if raw, ok := get(index.ModulesState, "a"); !ok || !strings.Contains(string(raw), m1.Merge) { - t.Errorf("modules.a = %s after the take-in at the start, want it at the merge %s", raw, m1.Merge) - } - if _, ok := get(index.Bucket(testIndexer, index.TakenState), repoKey); !ok { - t.Errorf("taken.%s is not written by the take-in at the start", repoKey) - } - if len(w) != 1 || w[0].Key != groupKey || len(w[0].Keys) != 1 || w[0].Keys[0] != g1Key { - t.Fatalf("indexer_pending after the start = %+v, want only the group gathered, its first member %s", w, g1Key) + if len(w) != 1 || w[0].Key != groupKey || w[0].What != takein.TakeInGroup || len(w[0].Merges) != 1 || len(w[0].Keys) != 1 || w[0].Keys[0] != g1Key || + w[0].Said != takein.TakeInPendingEvent { + t.Fatalf("indexer_pending = %+v, want only the group gathered, its first member %s", w, g1Key) } if _, ok := get(index.ModulesState, "c"); ok { t.Errorf("modules.c is written while its group is gathered") @@ -425,23 +388,6 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 t.Errorf("the group's first member %s was said pending as a revision of its own", g1Key) } - // IX5 (§712, §713): a merge whose changed manifest gains a problem the trunk case lacks stays pending, refused naming it, - // said in take-in.pending; nothing of it is taken in. - m3 := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/y", Number: 13, Head: h1, - Merge: "3333333333333333333333333333333333333333", Root: "r0", Trees: map[string]string{"modules/a": "tree-a-3"}, - Manifests: map[string]string{"modules/a": `{"module":"a","version":"2.0"}`}} - refusing.Store(m3.Merge, "a") - m3Key := takein.PendingKey(merged(m3)) - if refused, ok := said(takein.TakeInPending, m3Key); !ok || !strings.Contains(refused.Refused, "a: the stand-in's problem") || refused.Failed != "" { - t.Fatalf("take-in.pending.%s = %+v (said %v) within 20 s, want the revision refused naming a's problem", m3Key, refused, ok) - } - if w = waits(); len(w) != 1 || w[0].Key != m3Key || w[0].Validation == nil || w[0].Validation.Accepted != 0 || len(validations.asked()) != 3 { - t.Fatalf("indexer_pending = %+v, asks %d: want the refused revision pending with its validation, asked once after m1 and the group", w, len(validations.asked())) - } - if raw, ok := get(index.ModulesState, "a"); !ok || strings.Contains(string(raw), m3.Merge) { - t.Errorf("modules.a = %s: the refused revision was taken in", raw) - } - } // trunkIndexerEntry keeps the indexer's own manifest by its digest, accepted, and answers its trunk entry at an earlier commit. @@ -475,63 +421,6 @@ func trunkIndexerEntry(t *testing.T, ctx context.Context, js jetstream.JetStream return e } -// standIn is the build seat node-build-go as the indexer's validation meets it: the asks it heard, oldest first. -type standIn struct { - mu sync.Mutex - asks []sdkstep.StepAsk -} - -func (s *standIn) asked() []sdkstep.StepAsk { - s.mu.Lock() - defer s.mu.Unlock() - return slices.Clone(s.asks) -} - -// standInBuildSeat keeps the seat's check queue as a stream and hears the indexer's asks on it (accept.check.indexer): each is -// ended at once with step-ended on the seat's family, its answer inline (A-254), the program's last line: the manifest validator's verdict over both cases, -// the change case refusing the manifest refusing names for the job's commit. -func standInBuildSeat(t *testing.T, ctx context.Context, js jetstream.JetStream, refusing *sync.Map) *standIn { - t.Helper() - subject, err := sdkstep.AskSubjectBy(sdkstep.AskGo, sdkstep.AskCheck, index.ValidationAsker) - if err != nil || subject != "mesh.seat.node-build-go.accept.check.mesh-catalog" { - t.Fatalf("the validation's subject %q %v, want the indexer's seat name its asker (A-163, A-232)", subject, err) - } - if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "SEAT_NODE_BUILD_GO_CHECK", Subjects: []string{"mesh.seat.node-build-go.accept.check.>"}}); err != nil { - t.Fatal(err) - } - // The indexer's state inputs and its objects, where the validation's request is kept for the build seat's holder (§354). - if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, index.InputsState), History: 1}); err != nil { - t.Fatal(err) - } - if _, err := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: catalogue.BusName(testIndexer, catalogue.StateObjectsName(index.InputsState))}); err != nil { - t.Fatal(err) - } - s := &standIn{} - if _, err := js.Conn().Subscribe(subject, func(m *nats.Msg) { - a, err := sdkstep.DecodeStepAsk(m.Data) - if err != nil || a.Run == nil { - t.Errorf("the stand-in build seat heard what is no run ask: %v", err) - return - } - s.mu.Lock() - s.asks = append(s.asks, a) - s.mu.Unlock() - change := composer.CaseVerdict{Name: "change", Manifests: map[string]composer.ManifestRead{}} - if module, ok := refusing.Load(a.Run.Program.Commit); ok { - change.Manifests[module.(string)] = composer.ManifestRead{Refused: "the stand-in's problem"} - } - verdict, _ := json.Marshal(composer.Verdict{V: 1, Cases: []composer.CaseVerdict{change, {Name: judge.TrunkCase, Manifests: map[string]composer.ManifestRead{}}}}) - end, _ := json.Marshal(sdkstep.StepEnded{V: 1, ID: a.ID, Ask: a.ID, Asker: index.ValidationAsker, Node: "n1", Outcome: sdkstep.OutcomePassed, - Output: string(verdict), At: time.Now().UTC()}) - if _, err := js.Publish(ctx, nodebuildgo.SubjectStepEnded("n1"), end); err != nil { - t.Errorf("the stand-in build seat could not say step-ended: %v", err) - } - }); err != nil { - t.Fatal(err) - } - return s -} - // jsSource is a bucket as a views.Source. type jsSource struct { ctx context.Context diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index b99d4b94..9bc84e90 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -479,11 +479,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear; // until then every merge is taken in at once, as before. Pending: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.PendingState), "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)}, - SayTakeIn: sayTakeIn(js, cred), - // The validation's run ask (design 47 G §713, A-247, A-248), put under the indexer's own asker. - ValidationAsks: buildrun.Asker{Name: index.ValidationAsker, Publisher: buildrun.JetStreamAsks{JS: js}}, - // The build seat's verb output, for a validation's answer kept as an object (A-254). - AskVerb: calls.CallFile} + SayTakeIn: sayTakeIn(js, cred)} // The registrations and the changes that wait for their warrant (design 47 G §2, §667): their states are bound when they // appear; a warrant is heard on the router's decided event for this module, below. registry := newRegistry(js, conn, cred, calls.Call, diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go index 9eaa74d8..1c9c5fed 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go @@ -206,6 +206,15 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T) if pub.Commit != m2 || len(pub.Packages) != 3 { t.Fatalf("packages.%s = %+v within 20 s, want the merge %s with a digest of go, announce and buildstep", key, pub, m2) } + // A-476 c: mesh-sdk's merge moved no module registered for another repository: importer stands as it was. + if now := module("importer", func(meshcatalog.ModulesByModuleValue) bool { return true }); now.Fp2 != importer.Fp2 || now.Packages[sdkGo] != m1 { + t.Errorf("mesh-sdk's merge moved importer of novox/mesh-catalog: %+v, want it unmoved at %s", now.Packages, m1) + } + // The next merge of its own repository moves it, built against the package published meanwhile. + merged("novox/mesh-catalog", 302, "8888888888888888888888888888888888888888", forgemsg.Forge{Root: "r0", + Trees: map[string]string{"modules/importer": "t1", "modules/plain": "t2"}, + Manifests: map[string]string{"modules/importer": named("importer"), "modules/plain": named("plain")}, + Locks: map[string]map[string]string{"modules/importer": {"go.sum": goSumHash}, "modules/plain": {"go.sum": goSumHash}}}) moved := module("importer", func(m meshcatalog.ModulesByModuleValue) bool { return m.Fp2 != importer.Fp2 }) if moved.Packages[sdkGo] != m2 { t.Errorf("importer moved to %+v, want it built against %s", moved.Packages, m2) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go index c2b5210b..75d37b44 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go @@ -28,7 +28,6 @@ type takerIn interface { NamedDrop(ctx context.Context, fingerprint, member, by, why string) index.TakenIn Unpin(ctx context.Context, revision, record, by, why string) index.TakenIn DropRunByHand(ctx context.Context, run, by, why string) index.TakenIn - ValidateAgainByHand(ctx context.Context, key, by, why string) index.TakenIn RegistryAct(ctx context.Context, a index.TakeInAsk) index.TakenIn } @@ -75,9 +74,6 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream } else if a.Merge != "" { asked, r = "hand take-in of a merge", ix.TakeInMerge(ctx, a.MergeRepository, a.Merge, a.Sequence, a.By, a.Why) a.Module = r.Module - } else if a.Pending != "" && a.ValidateAgain { - asked, r = "validation asked again", ix.ValidateAgainByHand(ctx, a.Pending, a.By, a.Why) - a.Module = a.Pending } else if a.Pending != "" && a.Drop { asked, r = "drop of a pending merge", ix.DropPending(ctx, a.Pending, a.By, a.Why, a.Abandon) a.Module = a.Pending diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go index e50e8423..cb8076b2 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go @@ -7,8 +7,6 @@ import ( "sync/atomic" "time" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" - "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -155,15 +153,6 @@ func heardStepEnded(ctx context.Context, ix *index.Indexer, m jetstream.Msg, log _ = m.Ack() return } - // The end of a validation ask resumes its pending revision (design 47 G §713); its own fault is given again by the bus. - var end buildstep.StepEnded - if err := json.Unmarshal(m.Data(), &end); err == nil { - if err := ix.HearValidationEnded(ctx, end); err != nil { - logf("take-in: the end of the validation ask %s is not taken in, so the bus gives it again: %v", end.Ask, err) - leftForBus(m, "the end of the validation ask "+end.Ask, err, logf) - return - } - } if err := ix.HearInputEnded(ctx, e); err != nil { logf("inputs: the input of the ask %s could not be deleted, so the bus gives it again: %v", e.Ask, err) leftForBus(m, "the step-ended of the ask "+e.Ask, err, logf) diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index eaa2ed47..1390342c 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -6,7 +6,6 @@ import ( "context" "errors" "fmt" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/stateobjects" "reflect" "slices" @@ -234,12 +233,6 @@ type Indexer struct { Pending Store // SayTakeIn publishes a take-in event (take-in.pending, take-in.taken; ADR 0633 §5); nil says none. SayTakeIn func(ctx context.Context, t takein.TakeIn) error - // ValidationAsks puts the validation's run ask (design 47 G §713, A-247: buildrun.Asker, named as the indexer's asker, - // A-248); nil validates nothing, and a revision is taken in as before. - ValidationAsks ValidationAsks - // AskVerb asks a seat verb as the runtime calls it: the build seat's verb output, for a run's answer the seat keeps as an - // object (design 47 G A-254, buildstep.StepEnded.ReadOutput); nil reads none, and such an answer could not validate. - AskVerb buildstep.AskVerb // Runs is the state runs: the runs on request the indexer holds a candidate for (runcandidates.go, A-174); nil keeps none. Runs Store runMu sync.Mutex // one run event at a time (runcandidates.go) diff --git a/modules/mesh-delivery-indexer/internal/index/fingerprints.go b/modules/mesh-delivery-indexer/internal/index/fingerprints.go index a275cffa..14a2be8f 100644 --- a/modules/mesh-delivery-indexer/internal/index/fingerprints.go +++ b/modules/mesh-delivery-indexer/internal/index/fingerprints.go @@ -60,7 +60,7 @@ func (ix *Indexer) FingerprintsAgain(ctx context.Context, module string) TakenIn ix.freshMu.Unlock() // The path of a merge's recompute: the modules, then the candidates their fingerprints reach (recomputeWith), then the // pending merges a changed fingerprint settles (PendingAgain, as after a toolchain report). - err = ix.recomputeWith(ctx, nil, nil) + err = ix.recomputeWith(ctx, nil, nil, "") ix.freshMu.Lock() ix.fresh = nil ix.freshMu.Unlock() diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index 6aacb8ab..8f2ed205 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -144,7 +144,7 @@ func (ix *Indexer) applyMoving(ctx context.Context, f forgemsg.Forge, drop bool, ix.logf("%s at %s: its packages are not published: %v", f.Repository, f.Merge, err) } // What this merge moved reaches the modules standing on it (a base's tree, build-agent's fingerprint), in the graph's order. - return w, ix.recomputeWith(ctx, overlay, w) + return w, ix.recomputeWith(ctx, overlay, w, f.Repository) } // baseFP answers the new-form fingerprint of a base module as modules. holds it, or why it has none. diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index 6a373b1f..a5bc1f15 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -13,7 +13,6 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" @@ -23,8 +22,6 @@ import ( "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-indexer/judge" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph" ) // The take-in step (novox/hq ADR 0633): a source revision is taken in only by the indexer it names. A merged message is @@ -98,8 +95,6 @@ type Pending struct { // Held is why a ready merge is not taken in although its revision names this build: objects full (HeldObjectsFull), // until room is said (RoomAgain). Held string `json:"held,omitempty"` - // Validation is the revision's validation (design 47 G §713, A-247), on the revision's newest value alone. - Validation *takein.Validation `json:"validation,omitempty"` } // PendingEntry is one key of the pending state as read: its key, revision, value and the merged message decoded. For a @@ -786,31 +781,37 @@ func (ix *Indexer) evaluate(ctx context.Context, own string, start bool) (Applie if len(lost) > 0 { return Applied{}, ix.waitingLine(w), ix.holdLost(ctx, w.Ready, lost, reasons, unread) } - if wait, words, err := ix.validationGate(ctx, ready, own); wait || err != nil { - return Applied{}, words, err - } - names, dry, err := ix.computeOver(ctx, ready) + dry, err := ix.computeOver(ctx, ready) if err != nil { return Applied{}, "", err } if words, refused, err := ix.refuseCycle(ctx, ready, dry); refused || err != nil { return Applied{}, words, err } - if names == own { - a, err := ix.takeInWhole(ctx, ready, own, "") - if errors.Is(err, errClaimLost) { - return a, "claimed by another take-in", nil - } - if errors.Is(err, errVerdictsNotYet) { - return a, "the verdicts state is not on the bus yet: the revision waits for it, taken in when it binds", nil - } - if errors.Is(err, ErrObjectsFull) { - ix.holdFull(ctx, ready) - return a, ix.waitingLine(w) + "; held: " + HeldObjectsFull, nil - } - return a, ix.waitingLine(w), err + // The indexer takes in with the build it runs, whatever build of the indexer the revision names (A-476 b). + a, err := ix.takeInWhole(ctx, ready, own, "") + if errors.Is(err, errClaimLost) { + return a, "claimed by another take-in", nil } - return Applied{}, ix.waitingLine(w), ix.holdFor(ctx, ready, names, own, dry) + if errors.Is(err, errVerdictsNotYet) { + return a, "the verdicts state is not on the bus yet: the revision waits for it, taken in when it binds", nil + } + if errors.Is(err, ErrObjectsFull) { + ix.holdFull(ctx, ready) + return a, ix.waitingLine(w) + "; held: " + HeldObjectsFull, nil + } + return a, ix.waitingLine(w), err +} + +// newestOfRevision is the index in ready of the revision's newest merge. +func newestOfRevision(ready []PendingEntry) int { + n := 0 + for i, e := range ready { + if e.Value.Seq > ready[n].Value.Seq { + n = i + } + } + return n } // refuseCycle refuses the revision whose catalog graph would hold a cycle with a build edge the take-in refuses (design 47 G @@ -951,7 +952,6 @@ func (ix *Indexer) dry() *dryRun { x := &Indexer{Module: ix.Module, Manifests: ix.Manifests, Packages: ix.Packages, JudgeID: ix.JudgeID, Files: ix.Files, GoSum: ix.GoSum, GoMod: ix.GoMod, Now: ix.Now, Builds: ix.Builds, Named: ix.Named, Trunk: ix.Trunk, BranchHead: ix.BranchHead, Read: ix.Read, Contains: ix.Contains, ReadOpen: nil, OwnBuild: nil, Message: ix.Message, SayRefused: nil, SayAccepted: nil, SayTakeIn: nil, Pending: nil, - ValidationAsks: nil, AskVerb: nil, // a dry run asks no validation (§713) // The registry milestone's fields: what a take-in would write on the bus stays out of a dry run, so the states of the // requirements, findings, seat definitions, holds and their bounds are not overlaid, nothing is said, and no hold is // weighed (Holds and Counted nil hold nothing); ReadFile only reads. @@ -991,16 +991,15 @@ func (ix *Indexer) dry() *dryRun { } // computeOver applies the ready merges, in merge order, to a dry indexer, as one overlay: each later merge's trees -// govern (ADR 0633 §3). It answers the fingerprint the indexer's own module has once they are applied (none is "", -// with its reason), and the dry run, whose overlays hold what the take-in would write. A merge that can never be read +// govern (ADR 0633 §3). It answers the dry run, whose overlays hold what the take-in would write. A merge that can never be read // is left out, as its take-in will drop it. -func (ix *Indexer) computeOver(ctx context.Context, ready []PendingEntry) (string, *dryRun, error) { +func (ix *Indexer) computeOver(ctx context.Context, ready []PendingEntry) (*dryRun, error) { d := ix.dry() // The manifests are judged against the seat definitions the revision carries (seatsof.go, ADR 0633 §1), so the indexer // it names is not refused for a verb or an event the built-in list lacks. seats, err := ix.revisionSeats(ctx, ready) if err != nil { - return "", d, fmt.Errorf("reading the seat definitions of %s: %w", ready[0].Key, err) + return d, fmt.Errorf("reading the seat definitions of %s: %w", ready[0].Key, err) } var applyErr error judge.WithSeats(seats, func() { @@ -1018,24 +1017,9 @@ func (ix *Indexer) computeOver(ctx context.Context, ready []PendingEntry) (strin } }) if applyErr != nil { - return "", d, applyErr + return d, applyErr } - key, _ := meshcatalogclient.ModulesKey(ix.Module) - value, _, found, err := d.ix.Modules.Get(ctx, key) - if err != nil { - return "", d, err - } - if !found { - return NoBuild + "the indexer holds no record of itself", d, nil - } - m, err := meshcatalogclient.DecodeModule(value) - if err != nil { - return "", d, nil - } - if m.Fp2 == "" { - return NoBuild + m.None2, d, nil - } - return m.Fp2, d, nil + return d, nil } // claim marks the entries as taken in by this take-in, by compare-and-set on the revisions read. A lost one (another @@ -1298,83 +1282,6 @@ func saidWaits(entries []PendingEntry) []takein.TakeIn { return out } -// holdFor keeps the ready merges pending for the indexer build they name and writes only what building and moving -// that indexer needs (ADR 0633 §1): the indexer's own module entry, its key of the module graph and the verdict on its -// manifest, from the dry run; the manifests and the packages are written by the dry run itself. The revision is said -// pending under its oldest merge's key, and a wait it was said under before by another key is said taken. -func (ix *Indexer) holdFor(ctx context.Context, ready []PendingEntry, names, own string, d *dryRun) error { - key, _ := meshcatalogclient.ModulesKey(ix.Module) - if err := copyKey(ctx, d.modules, ix.Modules, key); err != nil { - return fmt.Errorf("the indexer's own module entry: %w", err) - } - if gk, err := graph.Key(ix.Module); err == nil { - if err := copyKey(ctx, d.graph, ix.Graph, gk); err != nil && !errors.Is(err, process.ErrNotYet) { - return fmt.Errorf("the indexer's key of the module graph: %w", err) - } - } - // Why the manifest of the indexer the revision names was refused, when it was (judged with the seat definitions the - // revision carries): said with the wait, as a merge the forge bridge could not give is (holdLost). - refused := "" - if value, _, found, err := d.ix.Modules.Get(ctx, key); err == nil && found { - if m, err := meshcatalogclient.DecodeModule(value); err == nil && m.Manifest != "" { - if vk, err := meshcatalogclient.VerdictsKey(m.Manifest); err == nil { - if err := copyKey(ctx, d.verdicts, ix.Verdicts, vk); err != nil && !errors.Is(err, process.ErrNotYet) { - return fmt.Errorf("the verdict on the indexer's manifest: %w", err) - } - if vv, _, ok, err := d.ix.Verdicts.Get(ctx, vk); err == nil && ok { - if v, err := meshcatalogclient.DecodeVerdict(vv); err == nil && meshcatalogclient.VerdictRefuses(v) { - refused = strings.Join(v.Problems, "; ") - } - } - } - } - } - // The indexer's own entry the held revision names is kept as the trunk revision after the newest, every other entry as - // it was (milestone 47/G W4.7: a merged revision naming a newer indexer replaces only the indexer's entry). - if err := ix.KeepView(ctx); err != nil { - return fmt.Errorf("the revision with the indexer's own entry: %w", err) - } - t := revisionWait(takein.TakeInPending, ready, names) - t.Refused = refused - var ended, unmarked []PendingEntry - said, changed := false, false - for _, e := range ready { - switch { - case e.Value.Said == t.Key && e.Value.SaidKind == takein.TakeInPending: - said = true - changed = changed || e.Value.WaitsFor != names || e.Value.Refused != refused - if e.Value.WaitsFor != names || e.Value.Refused != refused { - unmarked = append(unmarked, e) - } - case e.Value.Said != "" && e.Value.SaidKind == takein.TakeInPending: - ended = append(ended, e) - unmarked = append(unmarked, e) - default: - unmarked = append(unmarked, e) - } - } - for _, old := range saidWaits(ended) { - ix.sayTakeIn(ctx, old) - } - if said && !changed { - // Nothing to say on the bus, but the start decided nothing and a reader of the journal sees why: the build learnt, the build the wait names. - ix.logf("take-in: %d merges, oldest %s, still name the indexer build %s, this program runs %s: they wait, as already said", len(ready), t.Key, names, own) - } - if !said || changed { - ix.logf("take-in: %d merges, oldest %s, name the indexer build %s, not %s, which runs: they wait for it", len(ready), t.Key, names, own) - if refused != "" { - ix.logf("take-in: the manifest of the indexer they name is refused, judged with the seat definitions they carry, so no such indexer runs: %s", refused) - } - if !ix.sayTakeIn(ctx, t) { - ix.markSaid(ctx, unmarked, nil, func(p *Pending) { p.WaitsFor, p.LastWriter, p.Refused = names, own, refused }) - return nil - } - unmarked = ready - } - ix.markSaid(ctx, unmarked, &t, func(p *Pending) { p.WaitsFor, p.LastWriter, p.Refused = names, own, refused }) - return nil -} - // copyKey writes what the overlay holds of key to the store, when it wrote or deleted it. func copyKey(ctx context.Context, from *overlayStore, to Store, key string) error { if from == nil || to == nil { @@ -1439,9 +1346,6 @@ func waitOf(entries []PendingEntry, what, behind string) takein.PendingWait { if e.Value.Refused != "" { pw.Refused = e.Value.Refused } - if e.Value.Validation != nil { - pw.Validation = e.Value.Validation - } if e.Value.Said != "" && e.Value.SaidAt.After(pw.SaidAt) { pw.Said, pw.SaidAt = "take-in."+e.Value.SaidKind, e.Value.SaidAt } diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index 3d001ffe..6f22c27a 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -106,6 +106,16 @@ func (b *takeInBed) merge(number int, c, avahi, indexer string) forgemsg.Forge { return f } +// arriveHeld arrives f while the verdicts state is not on the bus, so the take-in holds it (ADR 0668 §4): the way a merge is +// left pending now that no merge waits for the indexer build it names (A-476 b). +func (b *takeInBed) arriveHeld(t *testing.T, seq uint64, f forgemsg.Forge) { + t.Helper() + bound := b.ix.Verdicts + b.ix.Verdicts = &memStore{unbound: process.ErrNotYet} + defer func() { b.ix.Verdicts = bound }() + b.arrive(t, seq, f) +} + func (b *takeInBed) arrive(t *testing.T, seq uint64, f forgemsg.Forge) { t.Helper() b.publish(t, f) @@ -224,16 +234,6 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) if want := []string{"pending " + takein.PendingKey(7), "taken " + takein.PendingKey(7)}; !slices.Equal(b.events(), want) || len(b.keys()) != 0 { t.Fatalf("said %v, pending %v: want %v and nothing pending", b.events(), b.keys(), want) } - // Two merges said pending under the oldest's key (waiting for a named indexer), then taken in together: one taken, under - // that key, and no entry's key said twice. - b.said = nil - b.arrive(t, 8, b.merge(31, sha('c'), "tree-avahi-3", "tree-indexer-2")) - b.arrive(t, 9, b.merge(32, sha('d'), "tree-avahi-4", "tree-indexer-2")) - b.said = nil - b.arrive(t, 10, b.merge(33, sha('e'), "tree-avahi-5", b.base.Trees["modules/"+testSelf])) - if want := []string{"taken " + takein.PendingKey(8), "taken " + takein.PendingKey(9), "taken " + takein.PendingKey(10)}; !slices.Equal(b.events(), want) || len(b.keys()) != 0 { - t.Errorf("said %v, pending %v: want %v, the wait's key (m.8, said pending) once and each other entry's own", b.events(), b.keys(), want) - } }) t.Run("T7 and T3 a merge alone, an indexer change, a merge behind it and a revert", func(t *testing.T) { b := newTakeInBed(t) @@ -259,42 +259,28 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) t.Fatalf("alone: avahi %s (was %s), pending %v, said %v: want taken in, said taken under its key (1966/AC195-1)", avahi1, avahi0, b.keys(), b.events()) } b.said = nil - avahi1, verdicts := b.fp(t, "avahi"), len(b.ix.Verdicts.(*memStore).vals) - // T4: a change to the indexer writes only its entry (with its graph key and verdict); the rest is pending. + avahi1 := b.fp(t, "avahi") + // A-476 b: a merge naming another indexer build is taken in at once, with the build this program runs: the indexer's entry + // and avahi are written, nothing is held and said pending. b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) named := b.fp(t, testSelf) - if named == own || b.fp(t, "avahi") != avahi1 || b.commit(t, "avahi") != sha('b') { - t.Fatalf("indexer change: indexer %s (own %s), avahi %s at %s: want only the indexer written", named, own, b.fp(t, "avahi"), b.commit(t, "avahi")) + if named == own || b.fp(t, "avahi") == avahi1 || b.commit(t, "avahi") != sha('c') || len(b.keys()) != 0 { + t.Fatalf("indexer change: indexer %s (own %s), avahi %s at %s, pending %v: want the whole revision taken in at once", named, own, b.fp(t, "avahi"), b.commit(t, "avahi"), b.keys()) } if g := graphOf(t, b.ix, testSelf); g.FP != named { t.Errorf("graph of the indexer names %s, want %s", g.FP, named) } - if n := len(b.ix.Verdicts.(*memStore).vals); n > verdicts+1 { - t.Errorf("verdicts %d, want at most %d: only the new indexer manifest's", n, verdicts+1) + if want := []string{"taken " + takein.PendingKey(2)}; !slices.Equal(b.events(), want) { + t.Errorf("said %v, want %v: nothing said pending", b.events(), want) } - if want := []string{takein.PendingKey(2)}; !reflect.DeepEqual(b.keys(), want) || !slices.Equal(b.events(), []string{"pending " + takein.PendingKey(2)}) || b.said[0].WaitsFor != named { - t.Fatalf("pending %v, said %+v: want m.2 pending for %s", b.keys(), b.said, named) - } - // T5: the merge delivered again writes no second key and says nothing again. - b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) - if len(b.keys()) != 1 || len(b.said) != 1 { - t.Errorf("redelivered: pending %v, said %v", b.keys(), b.events()) - } - // A later merge joins: the newest governs and still names the new indexer. + // A later merge is taken in at once too, and a revert to the first indexer. b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")) - if len(b.keys()) != 2 || b.commit(t, "avahi") != sha('b') || len(b.said) != 1 { - t.Fatalf("behind: pending %v, avahi at %s, said %v: want both pending and nothing said again", b.keys(), b.commit(t, "avahi"), b.events()) - } - // A revert brings the indexer back to the running build: everything is taken in, oldest first. b.arrive(t, 4, b.merge(23, sha('e'), "tree-avahi-5", b.base.Trees["modules/"+testSelf])) if len(b.keys()) != 0 || b.fp(t, testSelf) != own || b.commit(t, "avahi") != sha('e') { t.Fatalf("revert: pending %v, indexer %s, avahi at %s: want all taken in", b.keys(), b.fp(t, testSelf), b.commit(t, "avahi")) } - if want := []string{"pending " + takein.PendingKey(2), "taken " + takein.PendingKey(2), "taken " + takein.PendingKey(3), "taken " + takein.PendingKey(4)}; !slices.Equal(b.events(), want) { - t.Errorf("said %v, want %v", b.events(), want) - } }) - t.Run("IX22 a merged revision naming a newer indexer is the previous catalog graph revision with only the indexer's entry replaced", func(t *testing.T) { // design 47 G §273, §237 + t.Run("IX22 a merged revision naming a newer indexer is a catalog graph revision with its entry and the merge's others replaced", func(t *testing.T) { // design 47 G §273, §237 b := newTakeInBed(t) store := newMemObjects(0) b.ix.Revisions = store @@ -305,14 +291,14 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) if err != nil || before.Revision != 1 { t.Fatalf("the view's revision %+v (%v), want revision 1", before, err) } - // The merge changes avahi and the indexer: only the indexer is written, the revision is made at once. + // The merge changes avahi and the indexer: both are taken in at once (A-476 b), in a revision after the first. b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) after, _, err := newestTrunk(ctx, store) - if err != nil || after.Revision != 2 { - t.Fatalf("after the merge naming a newer indexer the newest revision is %+v (%v), want revision 2", after, err) + if err != nil || after.Revision <= before.Revision { + t.Fatalf("after the merge naming a newer indexer the newest revision is %+v (%v), want one after revision %d", after, err, before.Revision) } - if changed := changedEntries(before.Entries, after.Entries); !slices.Equal(changed, []string{testSelf}) { - t.Errorf("the revision differs from the previous in %v, want the indexer's entry alone", changed) + if changed := changedEntries(before.Entries, after.Entries); !slices.Equal(changed, []string{"avahi", testSelf}) && !slices.Equal(changed, []string{testSelf, "avahi"}) { + t.Errorf("the revision differs from the previous in %v, want avahi and the indexer", changed) } }) t.Run("P1 a revision that adds a Go module its own indexer uses is computed against that module's packages at its merge", func(t *testing.T) { @@ -353,71 +339,13 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) t.Fatal(err) } b.arrive(t, 2, f) - if len(b.said) != 1 || !strings.HasPrefix(b.said[0].WaitsFor, "src2:") { - t.Errorf("said %+v, want m.2 waiting for the indexer build its merge's packages give, not for a version no take-in has published", b.said) + if len(b.said) != 1 || b.said[0].Kind != takein.TakeInTaken || b.said[0].WaitsFor != "" || len(b.keys()) != 0 { + t.Errorf("said %+v, pending %v, want m.2 taken in at once", b.said, b.keys()) } if _, ok := b.ix.publishedAll(ctx)[sibling]; !ok { t.Errorf("the packages the revision publishes at its merge lack %s", sibling) } }) - t.Run("K1 the indexer a revision names is judged with the seat definitions the revision carries", func(t *testing.T) { - // A successor's manifest invokes a verb of the node-engine's seat that the seat list lacks and the revision's - // definitions add (ADR 0633 §1, §6): refused without them, with its reason said on the wait and in indexer_pending; - // accepted with them, and the revision waits for the build it names. - const verb, contracts = "revision-verb", "modules/seat-contracts" - // The third row's definer is the Delivery domain's contracts module defining a Core seat: not taken (ADR 0636 §5). - for _, row := range []struct { - name string - defined bool - refused bool - module string - }{{"definitions the revision does not carry", false, true, ""}, {"definitions the revision carries", true, false, "seat-contracts"}, - {"definitions of a seat outside the carrier's domain", true, true, "mesh-delivery-contracts"}} { - b := newTakeInBed(t) - f := b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2") - f.Manifests = map[string]string{} - for dir, raw := range b.base.Manifests { - f.Manifests[dir] = raw - } - dir := "modules/" + testSelf - f.Manifests[dir] = strings.Replace(f.Manifests[dir], `"module": "`+testSelf+`"`, - `"module": "`+testSelf+`", "invokes": ["seat:node-engine.`+verb+`"]`, 1) - if row.defined { - f.Trees[contracts] = "tree-seat-contracts" - f.Locks = map[string]map[string]string{} - for d, l := range b.base.Locks { - f.Locks[d] = l - } - f.Locks[contracts] = map[string]string{} - f.Manifests[contracts] = `{"module":"` + row.module + `","version":"1.0","build":{"artifacts":[{"name":"go-seats","kind":"package","language":"go","paths":["go"],"series":"0.1","generated-from":"."}]}}` - b.ix.Files = func(_ context.Context, repository, commit string, dirs []string) (map[string]map[string]string, error) { - if repository != f.Repository || commit != f.Merge || !slices.Equal(dirs, []string{contracts + "/seats"}) { - t.Errorf("listed %v of %s at %s", dirs, repository, commit) - } - return map[string]map[string]string{dirs[0]: {"node-engine/seat.json": "1", "node-engine/fixtures/a.json": "2"}}, nil - } - b.ix.ReadFile = func(_ context.Context, _, _, p string) ([]byte, error) { - switch p { - case contracts + "/seats/node-engine/seat.json": - return []byte(`{"format":1,"seat":"node-engine","scope":"node","grain":"seat-list","decision":"d","domain":"Core","verbs":[{"name":"` + verb + `"}]}`), nil - } - return nil, nil - } - } - b.arrive(t, 2, f) - waits := PendingWaits(readPending(t, b)) - refused := "" - if len(b.said) > 0 { - refused = b.said[0].Refused - } - if got := strings.Contains(refused, `serves no verb "`+verb+`"`); got != row.refused || (len(waits) == 1 && waits[0].Refused != refused) { - t.Errorf("%s: said %q, indexer_pending %+v: want a refusal naming the verb %v", row.name, refused, waits, row.refused) - } - if _, ok := catalogue.SeatNamed("node-engine"); !ok || judgeSeatsHave("node-engine", verb) { - t.Errorf("%s: the seats are not put back after the dry run", row.name) - } - } - }) t.Run("A-326 a revision whose catalog graph holds a cycle with a build edge no build seat breaks is refused, naming it", func(t *testing.T) { // avahi's artifact needs the indexer's (kind 2), and the indexer is built on avahi (kind 3): avahi holds no build seat, so // no running version breaks the cycle, and the revision is refused at take-in naming every artifact on it (design 47 G §49). @@ -443,15 +371,15 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) t.Errorf("indexer_pending %+v, want the refusal naming the cycle", waits) } }) - t.Run("T10 the named indexer takes the revision in at its start; a gathered group stays", func(t *testing.T) { + t.Run("T10 a start takes what waits in once its build is known; a gathered group stays", func(t *testing.T) { b := newTakeInBed(t) - b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) + b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) g := b.merge(30, sha('f'), "", "") g.Repository, g.Trees, g.Manifests, g.Locks, g.Group = "novox/mesh-sdk", map[string]string{}, nil, nil, "feat/x" g.Members = []meshdeliveryclient.Head{{Repository: "novox/mesh-sdk", Number: 30, Head: g.Head}, {Repository: "novox/mesh-host", Number: 4, Head: sha('9')}} - b.arrive(t, 5, g) - named := b.fp(t, testSelf) - // The new indexer starts; its build is not known (the engine does not answer at its start): a hand ask is answered + b.arriveHeld(t, 5, g) + own, _ := b.ix.OwnBuild(ctx) + // The indexer starts; its build is not known (the engine does not answer at its start): a hand ask is answered // meanwhile (R5), and the start waits for nothing. It is never asked again, so it stays unknown until it is started again. mk := func(ask func(context.Context) (string, error)) *Indexer { n := &Indexer{Module: testSelf, Store: b.ix.Store, Modules: b.ix.Modules, Manifests: b.ix.Manifests, Graph: b.ix.Graph, Verdicts: b.ix.Verdicts, @@ -462,18 +390,6 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) n.FixOwnBuild(ctx) return n } - // A start that decides nothing says so (issue 1921): the build it learnt, the build the held wait names, also when the - // wait was said before; and indexer_pending names the known build and since when. - oldBuild, _ := b.ix.OwnBuild(ctx) - oldStart := mk(func(context.Context) (string, error) { return oldBuild, nil }) - b.logs.Reset() - if _, words, err := oldStart.TakeInAtStart(ctx); err != nil || len(b.keys()) != 2 || !strings.Contains(b.logs.String(), "still name the indexer build") || - !strings.Contains(b.logs.String(), "at the start, this program runs build") { - t.Fatalf("a start that holds: %q %v, journal %q: want the build learnt and the build the wait names said", words, err, b.logs.String()) - } - if answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{}); err != nil || answer.(takein.PendingAnswer).OwnBuild != oldBuild || answer.(takein.PendingAnswer).OwnBuildSince.IsZero() { - t.Errorf("indexer_pending = %+v %v, want the known build and since", answer, err) - } asked := 0 next := mk(func(context.Context) (string, error) { asked++; return "", errors.New("not yet") }) if r := next.TakeInPendingByHand(ctx, "m.nothing", "lead@shanks", "a test"); r.Stored || !strings.Contains(r.Answer, "nothing waits") { @@ -483,12 +399,15 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) t.Fatalf("a start with its build unknown: %q %v, pending %v: want it to answer at once and take nothing in", words, err, b.keys()) } // The engine answers now, and the same process does not ask it again: it learnt nothing it did not know at its start. - next.OwnBuild = func(context.Context) (string, error) { asked++; return named, nil } + next.OwnBuild = func(context.Context) (string, error) { asked++; return own, nil } if _, words, err := next.TakeInAtStart(ctx); err != nil || asked != 1 || len(b.keys()) != 2 || !strings.Contains(words, "not known") { t.Fatalf("the window of a replacement: %q %v, asked %d, pending %v: want the build fixed at the start, never learnt later", words, err, asked, b.keys()) } + if answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{}); err != nil || answer.(takein.PendingAnswer).OwnBuild != own || answer.(takein.PendingAnswer).OwnBuildSince.IsZero() { + t.Errorf("indexer_pending = %+v %v, want the known build and since", answer, err) + } // The module started again fixes the build at its start and takes the revision in. - next = mk(func(context.Context) (string, error) { return named, nil }) + next = mk(func(context.Context) (string, error) { return own, nil }) if _, _, err := next.TakeInAtStart(ctx); err != nil { t.Fatal(err) } @@ -544,7 +463,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) if _, words, err := b.ix.EndGroup(ctx, end); err != nil || !strings.Contains(words, "taken in already") { t.Errorf("an end again: %q %v", words, err) } - // T11 (the lead's row): a group whose revision changes the indexer: the group's wait is taken, then the revision pending. + // T11: a group whose revision changes the indexer: the group's wait is taken and the revision taken in at once (A-476 b). b.said = nil g := member(catalogueRepo, 40, sha('c')) g.Group, g.Trees = "feat/two", b.merge(0, "", "", "tree-indexer-7").Trees @@ -553,7 +472,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) if _, _, err := b.ix.EndGroup(ctx, forgemsg.ForgeGroupEnd{Group: "feat/two", Members: []forgemsg.GroupMerge{{Repository: catalogueRepo, Number: 40, Merge: sha('c')}}}); err != nil { t.Fatal(err) } - if want := []string{"pending g.feat/two", "taken g.feat/two", "pending " + takein.PendingKey(9)}; !slices.Equal(b.events(), want) { + if want := []string{"pending g.feat/two", "taken g.feat/two", "taken " + takein.PendingKey(9)}; !slices.Equal(b.events(), want) { t.Errorf("a group naming another indexer: said %v, want %v", b.events(), want) } }) @@ -593,11 +512,8 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) } held := func() *takeInBed { b := newTakeInBed(t) - b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) - b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")) - b.ix.ownMu.Lock() - b.ix.ownFP = b.fp(t, testSelf) - b.ix.ownMu.Unlock() + b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) + b.arriveHeld(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")) return b } // A probe that fails keeps the group's wait and stops nothing else: an unrelated merge is still taken in. @@ -719,7 +635,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) }) t.Run("T5 two take-ins of one pending revision: the one that loses the claim writes nothing", func(t *testing.T) { b := newTakeInBed(t) - b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) + b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) named := b.fp(t, testSelf) b.ix.ownMu.Lock() b.ix.ownFP = named @@ -774,8 +690,8 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) {"a take-in that fails", func(b *takeInBed) { b.mods.failWrite = once(func(string) bool { return true }) }}, } { b := newTakeInBed(t) - b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) - b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")) + b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) + b.arriveHeld(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")) named := b.fp(t, testSelf) b.ix.ownMu.Lock() b.ix.ownFP = named @@ -794,7 +710,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) }) t.Run("W4.6 a full store holds the merge with the reason objects full, says full once, and room takes it in", func(t *testing.T) { b := newTakeInBed(t) - b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) + b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) named := b.fp(t, testSelf) b.ix.ownMu.Lock() b.ix.ownFP = named @@ -826,7 +742,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) // The same take-in into a store it fills to 85 % of its cap says filling once, and is taken in. kept := store.used() b2 := newTakeInBed(t) - b2.arrive(t, 2, b2.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) + b2.arriveHeld(t, 2, b2.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) b2.ix.ownMu.Lock() b2.ix.ownFP = b2.fp(t, testSelf) b2.ix.ownMu.Unlock() @@ -845,7 +761,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) t.Run("W4.12 a candidate's half dropped by hand frees its own entries, says room and takes the held merge in; the read tools list it; a recompute that moves nothing is equal", func(t *testing.T) { bed := func(store *memObjects) *takeInBed { b := newTakeInBed(t) - b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) + b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) b.ix.ownMu.Lock() b.ix.ownFP = b.fp(t, testSelf) b.ix.ownMu.Unlock() @@ -930,40 +846,10 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) t.Errorf("indexer_recompute = %+v (%v): want equal to the kept revision 2", got, err) } }) - t.Run("T7 a toolchain report evaluates pending again: the revision then names the running build and is taken in", func(t *testing.T) { - b := newTakeInBed(t) - b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) - entries, err := ReadPending(ctx, b.pending, nil) - if err != nil { - t.Fatal(err) - } - // The build the revision names under the next toolchain, worked out on a dry indexer; this one runs it. - x := b.ix.dry().ix - for _, node := range []string{"novox", "ace"} { - key, _ := ToolchainKey(node, "go") - x.PutToolchain("mesh-build-go", key, toolchainValue(t, "go", "go1.28.0", "1"), false) - } - ready, _, _, _ := b.ix.loadNewest(ctx, WaitsOf(entries).Ready) - next, _, err := x.computeOver(ctx, ready) - if err != nil || next == "" || next == b.said[0].WaitsFor { - t.Fatalf("the build under the next toolchain: %q %v (the revision names %s)", next, err, b.said[0].WaitsFor) - } - b.ix.ownMu.Lock() - b.ix.ownFP = next - b.ix.ownMu.Unlock() - reportToolchain(t, b.ix, "mesh-build-go", "novox", "go", "go1.28.0") - if len(b.keys()) != 1 { - t.Fatalf("one node's report: pending %v, want the revision still held (the nodes disagree)", b.keys()) - } - reportToolchain(t, b.ix, "mesh-build-go", "ace", "go", "go1.28.0") - if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('c') { - t.Errorf("after the toolchain report: pending %v, avahi at %s: want the revision taken in", b.keys(), b.commit(t, "avahi")) - } - }) t.Run("T12 T13 indexer_pending names the wait; a hand take-in and a drop say what they ended", func(t *testing.T) { b := newTakeInBed(t) - b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) - b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")) + b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) + b.arriveHeld(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")) var answer takein.PendingAnswer for _, tool := range PendingTools(b.pending, nil, nil, nil) { if tool.Name == PendingTool { @@ -975,7 +861,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) } } if len(answer.Waits) != 1 || answer.Waits[0].Key != takein.PendingKey(2) || !slices.Equal(answer.Waits[0].Holds, []string{testSelf, "avahi"}) || - answer.Waits[0].Said != takein.TakeInPendingEvent || len(answer.Waits[0].Keys) != 2 { + len(answer.Waits[0].Keys) != 2 { t.Fatalf("indexer_pending = %+v", answer) } // ADR 0668 §6: a later merge covers a pending one only when its commit has the pending merge's as an ancestor. @@ -990,11 +876,14 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) } history = "cd" b.said = nil + bound := b.ix.Verdicts + b.ix.Verdicts = &memStore{unbound: process.ErrNotYet} // m.3 stays held while m.2 is dropped if r := b.ix.DropPending(ctx, takein.PendingKey(2), "lead@shanks", "replaced by m.3", false); !r.Stored { t.Fatalf("drop: %+v", r) } - if len(b.said) < 2 || !b.said[0].Dropped || b.said[0].Key != takein.PendingKey(2) || b.said[0].By != "lead@shanks" || b.said[1].Key != takein.PendingKey(3) { - t.Fatalf("drop said %+v: want m.2 taken (dropped, by) and m.3 pending", b.said) + b.ix.Verdicts = bound + if len(b.keys()) != 1 || b.keys()[0] != takein.PendingKey(3) { + t.Fatalf("pending %v: want m.2 dropped and m.3 pending", b.keys()) } // Issue 1966 (#650), ADR 0633 §5: the newest pending merge of its repository, which no taken merge covers, is not // dropped; the answer names the read again. abandon drops it all the same (below). @@ -1014,8 +903,8 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) // §476, §493: a hand take-in's taken event carries the branch, the source revision it makes and its entries. if len(b.said) == 1 { got := b.said[0] - if got.Branch != TrunkBranch || got.Source <= 0 || len(got.Entries) != 1 || got.Entries[0].Module != "avahi" || - got.Entries[0].Commit != sha('d') || got.Entries[0].Fingerprint == "" { + if got.Branch != TrunkBranch || got.Source <= 0 || len(got.Entries) != 2 || got.Entries[1].Module != "avahi" || + got.Entries[1].Commit != sha('d') || got.Entries[1].Fingerprint == "" { t.Errorf("hand take-in said branch %q source %d entries %+v: want main, a source revision and avahi at its commit", got.Branch, got.Source, got.Entries) } else if newest, any, _ := newestTrunk(ctx, b.ix.Revisions); !any || got.Revision != newest.Revision { t.Errorf("hand take-in said revision %d, want the revision it made, %d (A-448)", got.Revision, newest.Revision) @@ -1025,7 +914,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) t.Errorf("the taken event is refused by its own contract: %v", err) } } - b.arrive(t, 4, b.merge(23, sha('e'), "tree-avahi-5", "tree-indexer-2")) + b.arriveHeld(t, 4, b.merge(23, sha('e'), "tree-avahi-5", "tree-indexer-2")) if r := b.ix.DropPending(ctx, takein.PendingKey(4), "lead@shanks", "abandoned on the operator's word", true); !r.Stored || len(b.keys()) != 0 { t.Errorf("an abandoned drop: %+v, pending %v", r, b.keys()) } @@ -1062,8 +951,8 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) if _, _, found, _ := b.ix.Store.Get(ctx, key); !found { t.Fatal("the group has no candidate") } - b.arrive(t, 2, big(b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))) - b.arrive(t, 3, big(b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))) + b.arriveHeld(t, 2, big(b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))) + b.arriveHeld(t, 3, big(b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))) raw, _, _, _ := b.pending.Get(ctx, takein.PendingKey(2)) if len(b.keys()) != 2 || len(raw) > 4096 || b.commit(t, "avahi") != sha('b') { t.Fatalf("a big merge held: pending %v, value %d bytes, avahi at %s: want two small values", b.keys(), len(raw), b.commit(t, "avahi")) @@ -1139,7 +1028,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) b := newTakeInBed(t) f := b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2") f.Words, f.Branch = strings.Repeat("x", 300*1024), "feat/g" - b.arrive(t, 2, f) // held by reference, for an indexer build that is not this one + b.arriveHeld(t, 2, f) // held by reference, with the verdicts state not on the bus b.ix.Taken = newMem() b.ix.Contains = func(context.Context, string, string, string) (bool, error) { return c.behind, nil } data, _ := takein.EncodeTaken(takein.Taken{Repository: trunk.RepositoryName(catalogueRepo), Merge: sha('d')}) diff --git a/modules/mesh-delivery-indexer/internal/index/pending_tools.go b/modules/mesh-delivery-indexer/internal/index/pending_tools.go index c27abaa7..366654eb 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_tools.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_tools.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "strings" "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" @@ -112,33 +111,3 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T Run: merge}, } } - -// ValidationTools is indexer_validate_again (class act, design 47 G §713): the tools bundle writes an ask in takeins and the -// indexer asks the revision's validation again; record says the hand act in the hand-act log before it is asked. -func ValidationTools(ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time, record func(what, why string) error) []stdio.Tool { - return []stdio.Tool{{Name: ValidateAgainTool, - Description: "Ask again the validation of a pending revision that could not validate (design 47 G §712, §713): the manifest " + - "validator's run on the build seat node-build-go ended could not run, exited other than 0, or answered not both cases. " + - "The revision is asked under its next try; the answer says the validation's state (the ask, its try, accepted, refused, " + - "or why it could not validate). A revision accepted, refused or whose ask is out is asked nothing: a refusal is asked " + - "again only by a later merge of its branch. A hand act, recorded through mesh-hand-acts before it is asked. " + - "indexer_pending names the keys and shows the validation whole.", - Input: map[string]any{"type": "object", "required": []string{"key", "why"}, "properties": map[string]any{ - "key": map[string]any{"type": "string", "description": "the revision's wait key, m., as indexer_pending names it"}, - "why": map[string]any{"type": "string", "description": "why it is asked again, in plain words"}}}, - Run: func(args map[string]any) (any, error) { - str := func(k string) string { s, _ := args[k].(string); return s } - key, value, err := NewValidateAgainAsk(str("key"), str("why"), stdio.CallerOf(args), now()) - if err != nil { - return nil, err - } - if err := record("asked the validation of the pending revision "+strings.TrimSpace(str("key"))+" again by hand", strings.TrimSpace(str("why"))); err != nil { - return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was asked: %w", err) - } - a, err := ask(key, value) - if err != nil { - return nil, err - } - return a.Result, nil - }}} -} diff --git a/modules/mesh-delivery-indexer/internal/index/takein.go b/modules/mesh-delivery-indexer/internal/index/takein.go index ff3f4d88..ae21ccc0 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein.go +++ b/modules/mesh-delivery-indexer/internal/index/takein.go @@ -51,9 +51,6 @@ type TakeInAsk struct { // Abandon, with Drop, drops a pending merge no later merge covers (ADR 0633 §5 asks a later merge; issue 1966, ADR // 0668 §5): the hand act says why it is abandoned all the same. Abandon bool `json:"abandon,omitempty"` - // ValidateAgain, with Pending, asks the validation of the revision under that key again (indexer_validate_again, design 47 - // G §713): one that could not validate. - ValidateAgain bool `json:"validate-again,omitempty"` // Merge asks the merge of MergeRepository at this commit to be taken in by hand (indexer_take_in_merge): its merged message // is asked of the forge bridge and run through the take-in as a delivered one, under its stream sequence Sequence. Merge string `json:"merge,omitempty"` @@ -167,20 +164,6 @@ func NewMergeAsk(repository, merge string, sequence uint64, why, by string, at t return "merge." + merge, data, err } -// NewValidateAgainAsk is the value indexer_validate_again writes: its key in takeins, `pending.validate-again.`, and the -// value. key is a revision's wait key, m., as indexer_pending names it. -func NewValidateAgainAsk(key, why, by string, at time.Time) (string, []byte, error) { - key = strings.TrimSpace(key) - if !strings.HasPrefix(key, "m.") { - return "", nil, errors.New("name the revision whose validation is asked again by its key, m., as indexer_pending names it") - } - if strings.TrimSpace(why) == "" { - return "", nil, errors.New("say why (why): a hand act is recorded through mesh-hand-acts") - } - data, err := json.Marshal(TakeInAsk{V: 1, Pending: key, ValidateAgain: true, Why: strings.TrimSpace(why), By: by, At: at.UTC()}) - return "pending.validate-again." + key, data, err -} - // NewPendingAsk is the value indexer_take_in_pending (drop false) or indexer_drop_pending (drop true) writes: its key // in takeins, `pending..`, and the value. key is a wait's key as indexer_pending names it, for a // drop one pending merge's; abandon drops one no later merge covers. diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index 87c225c7..4d80b405 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -13,6 +13,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" "git.novox.be/novox/mesh-sdk/go/catalogue" "mesh-delivery-indexer/judge" @@ -127,24 +128,26 @@ type known struct { // the order the bucket lists them in; modules that stand on each other have a cycle as their reason. A module is written // only when its fingerprint or its reason changes, and only if its manifest is still the one computed from. A module with // no kept inputs (a message that carried no manifest) is left as it is. -func (ix *Indexer) Recompute(ctx context.Context) error { return ix.recomputeWith(ctx, nil, nil) } +func (ix *Indexer) Recompute(ctx context.Context) error { return ix.recomputeWith(ctx, nil, nil, "") } // recomputeWith is Recompute for a merge as well: the records the merge gives (overlay, by module key) stand in place // of what the bucket holds, are computed with the others, and are written once, with their final fingerprint or reason // (issue 1444: a merge writes each module once). A record of the overlay that has no kept inputs is written as it is. w is -// the merge's write (order.go), nil for none; it is told each module written. -func (ix *Indexer) recomputeWith(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite) error { +// the merge's write (order.go), nil for none; it is told each module written. A merge's recompute names its repository +// (A-476 c): only the modules registered for it are moved, none of another repository; "" moves every module (a toolchain +// report, a hand recompute). +func (ix *Indexer) recomputeWith(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite, repository string) error { if err := ix.fillPaths(ctx); err != nil { return err } - if err := ix.recomputeModules(ctx, overlay, w); err != nil { + if err := ix.recomputeModules(ctx, overlay, w, repository); err != nil { return err } // What the modules moved to reaches the candidates' fingerprints too (a base, the builder, a toolchain). return ix.RecomputeCandidates(ctx) } -func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite) error { +func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite, repository string) error { if ix.Modules == nil { return nil } @@ -153,11 +156,11 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]mesh } ix.recompute.Lock() defer ix.recompute.Unlock() - return ix.recomputeLocked(ctx, overlay, w) + return ix.recomputeLocked(ctx, overlay, w, repository) } // recomputeLocked is recomputeModules with ix.recompute held by the caller (a hold that moves, holds.go). -func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite) error { +func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite, repository string) error { if ix.Manifests == nil { return ix.writeOverlay(ctx, overlay, nil, w) } @@ -307,6 +310,12 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc for _, key := range slices.Sorted(maps.Keys(all)) { k := all[key] r := of(key, nil) + if _, merged := overlay[key]; !merged && repository != "" && !trunk.SameRepository(k.cur.Repository, repository) { + // A module registered for another repository is not moved by this repository's commit (A-476 c): it stands as held. + r = result{fp: k.held.Fp2, none: k.held.None2, tcs: k.held.Toolchains, pkgs: sdkstep.PackageVersions(k.held.Packages)} + edges[key] = graphEdges{module: key, cur: k.cur, manifest: k.manifest, fp: r.fp, none: r.none, tcs: r.tcs, pkgs: r.pkgs} + continue + } edges[key] = graphEdges{module: key, cur: k.cur, manifest: k.manifest, fp: r.fp, none: r.none, tcs: r.tcs, pkgs: r.pkgs} if _, merged := overlay[key]; merged { final := overlay[key] diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index 491b153b..a62e7d0d 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -829,9 +829,18 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) { t.Helper() f := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-sdk", Branch: "feat/x", Number: 3, Head: h1, Merge: commit, Root: tree, Trees: map[string]string{"": tree}, Manifests: map[string]string{"": `{"module":"mesh-sdk","version":"1.0"}`}} + prev, _ := moduleOf(t, mods, "avahi") if err := ix.Apply(context.Background(), f); err != nil { t.Fatal(err) } + // A-476 c: a commit of mesh-sdk moves no module registered for another repository; the next recompute (a hand one here) + // moves avahi, which imports its package. + if after, _ := moduleOf(t, mods, "avahi"); after.Fp2 != prev.Fp2 || !reflect.DeepEqual(after.Packages, prev.Packages) { + t.Fatalf("a commit of mesh-sdk moved avahi, registered for another repository: fp2 %q (was %q) packages %v (were %v)", after.Fp2, prev.Fp2, after.Packages, prev.Packages) + } + if err := ix.Recompute(context.Background()); err != nil { + t.Fatal(err) + } a, _ := moduleOf(t, mods, "avahi") if a.Fp2 == "" || !reflect.DeepEqual(a.Packages, map[string]string{sdk: commit}) { t.Fatalf("after mesh-sdk's merge %.8s: fp2 %q none2 %q packages %v", commit, a.Fp2, a.None2, a.Packages) diff --git a/modules/mesh-delivery-indexer/internal/index/validate.go b/modules/mesh-delivery-indexer/internal/index/validate.go deleted file mode 100644 index 7bb23f69..00000000 --- a/modules/mesh-delivery-indexer/internal/index/validate.go +++ /dev/null @@ -1,264 +0,0 @@ -package index - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "maps" - "slices" - "strings" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" - "git.novox.be/novox/mesh-sdk/go/catalogue" - - "mesh-delivery-indexer/judge" -) - -// The validation before take-in (design 47 G §712, A-213, A-232): for a pending revision of a watched branch the running -// indexer asks the seat node-build-go a `run` of its own manifest validator over two cases, change and trunk, and refuses only -// what the revision causes (ADR 0579 §2). The ask rides the check queue under the indexer's name, its kind `run` in the body. -const ( - // validationProgram is the manifest validator's program, by its directory under the indexer entry's Go module: the indexer's - // own code, so no repository or module name is fixed here (ADR 0666 §5). - validationProgram = "cmd/mesh-judge" - // ValidationAsker is the asker the ask is put under (buildrun.Asker.Name, A-248): the indexer's seat name, mesh-catalog - // (design 47 G A-163, A-232). - ValidationAsker = meshcatalog.Seat - // validationCatalogSeat is the seat the indexer entry claims. - validationCatalogSeat = "mesh-catalog" - // The cases of the request, as the manifest validator names them. - validationChange = "change" - validationTrunk = judge.TrunkCase -) - -// validationInputVerb is where the build seat's holder asks the input: the seat mesh-catalog's verb input (§354). -var validationInputVerb = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbInput - -// validationRequest is the manifest validator's request for the ready merges of branch over the branch's current revision -// (§712): case change, every module of the revision by its entry's manifest, each one the merges carry replaced by the merged -// message's bytes (the newest merge governs) and one added for a new directory; case trunk, every module as it stands; the -// seat definitions the merges' trees carry; the packages and the predefined list as the take-in's own validator has them; -// no machines. It answers the indexer entry of the change case too, the one that claims the seat mesh-catalog. -func (ix *Indexer) validationRequest(ctx context.Context, branch string, ready []PendingEntry) (composer.Request, meshcatalog.CatalogEntry, error) { - var indexer meshcatalog.CatalogEntry - if ix.Revisions == nil { - return composer.Request{}, indexer, errors.New("the catalog graph is not kept here: no revision to validate over") - } - rev, found, err := newestOf(ctx, ix.Revisions, branch) - if err != nil { - return composer.Request{}, indexer, err - } - var entries []meshcatalog.CatalogEntry - if found { - if _, entries, err = ReadRevision(ctx, ix.Revisions, RevisionObject(rev)); err != nil { - return composer.Request{}, indexer, err - } - } - trunkCase := composer.Case{Name: validationTrunk, Manifests: map[string]composer.Manifest{}} - byPlace := map[string]string{} // -> module - of := map[string]meshcatalog.CatalogEntry{} - for _, e := range entries { - raw := ix.manifestBytes(ctx, e.ManifestDigest) - if raw == "" { - b, err := json.Marshal(e.Manifest) - if err != nil { - return composer.Request{}, indexer, fmt.Errorf("the manifest of %s: %w", e.Module, err) - } - raw = string(b) - } - trunkCase.Manifests[e.Module] = composer.Manifest{Raw: json.RawMessage(raw)} - byPlace[placeKey(e.Repository, e.Path)] = e.Module - of[e.Module] = e - } - change := composer.Case{Name: validationChange, Manifests: maps.Clone(trunkCase.Manifests)} - for _, e := range ready { - if e.Covered { - continue - } - for _, dir := range slices.Sorted(maps.Keys(e.Forge.Manifests)) { - raw := e.Forge.Manifests[dir] - module, known := byPlace[placeKey(e.Forge.Repository, dir)] - if !known { - m, err := judge.ForInputs([]byte(raw)) - if err != nil || m.Module == "" { - module = e.Forge.Repository + "/" + dir // a new directory whose manifest names no module: judged under its place - } else { - module = m.Module - } - byPlace[placeKey(e.Forge.Repository, dir)] = module - of[module] = meshcatalog.CatalogEntry{Module: module, Repository: e.Forge.Repository, Path: dir} - } - change.Manifests[module] = composer.Manifest{Raw: json.RawMessage(raw)} - } - } - for _, module := range slices.Sorted(maps.Keys(change.Manifests)) { - m, err := judge.ForInputs(change.Manifests[module].Raw) - if err != nil || !slices.ContainsFunc(m.Claims, func(c catalogue.Claim) bool { return c.Name == validationCatalogSeat }) { - continue - } - indexer = of[module] - if indexer.Module == "" { - indexer.Module = module - } - break - } - seats, err := ix.revisionDefinitions(ctx, ready) - if err != nil { - return composer.Request{}, indexer, err - } - req := composer.Request{V: composer.Version, Input: json.RawMessage("{}"), Cases: []composer.Case{change, trunkCase}, Seats: seats, - Predefined: ix.predefinedList(ctx)} - if known := ix.knownPackages(ctx); known != nil { - req.Packages = slices.Sorted(maps.Keys(known)) - } - return req, indexer, nil -} - -func placeKey(repository, path string) string { - return strings.ToLower(trunk.RepositoryName(repository)) + " " + path -} - -// revisionDefinitions are the validation's seat definitions, raw (§712 `seats`, A-241): the branch's revision's seat -// definitions with the definitions the merges' trees carry laid over them, so a revision that touches no definition is still -// judged against the whole seat list. A revision's own definitions are not kept yet (A-234), so the revision's are the -// trunk's, read from the state seat-definitions as the verb seat-definitions answers them. -func (ix *Indexer) revisionDefinitions(ctx context.Context, ready []PendingEntry) ([]json.RawMessage, error) { - defs, err := ix.trunkDefinitions(ctx) - if err != nil { - return nil, err - } - merged, err := ix.revisionSeatDefinitions(ctx, ready) - if err != nil { - return nil, err - } - for seat, d := range merged { - raw, err := json.Marshal(d) - if err != nil { - return nil, fmt.Errorf("the definition of %s: %w", seat, err) - } - defs[seat] = raw - } - var out []json.RawMessage - for _, seat := range slices.Sorted(maps.Keys(defs)) { - out = append(out, defs[seat]) - } - return out, nil -} - -// trunkDefinitions are the trunk's seat definitions of the state seat-definitions, raw, by seat: every `.trunk` entry -// that holds a definition; none where the indexer keeps no such state. -func (ix *Indexer) trunkDefinitions(ctx context.Context) (map[string]json.RawMessage, error) { - defs := map[string]json.RawMessage{} - if ix.SeatDefinitions == nil { - return defs, nil - } - keys, err := ix.SeatDefinitions.Keys(ctx) - if err != nil { - return nil, fmt.Errorf("the %s state cannot be listed: %w", SeatDefinitionsState, err) - } - for _, key := range keys { - seat, ok := strings.CutSuffix(key, "."+SourceTrunk) - if !ok || strings.HasPrefix(key, outOfStepPrefix) { - continue - } - v, _, found, err := ix.SeatDefinitions.Get(ctx, key) - if err != nil { - return nil, fmt.Errorf("%s.%s cannot be read: %w", SeatDefinitionsState, key, err) - } - var e SeatDefinitionEntry - if !found || json.Unmarshal(v, &e) != nil || e.Digest == "" || len(e.Definition) == 0 { - continue // an entry that holds no definition is no seat of the list - } - defs[seat] = e.Definition - } - return defs, nil -} - -// validationJob is the run of the manifest validator for the request (§712): the program cmd/mesh-judge, its directory under -// the Go module the go.mod at the entry's path names (read by the build, A-268), at the entry's path, built at head, the revision's head of the entry's repository; no -// heads and no arguments; the input the request's bytes, served by the seat mesh-catalog's verb input. -func validationJob(indexer meshcatalog.CatalogEntry, head, branch string, seq uint64, input []byte) (buildstep.RunJob, error) { - switch { - case indexer.Repository == "": - return buildstep.RunJob{}, fmt.Errorf("no entry of the revision claims the seat %s", validationCatalogSeat) - case head == "": - return buildstep.RunJob{}, fmt.Errorf("the revision names no head of %s, the repository of the indexer entry %s", indexer.Repository, indexer.Module) - } - in, err := buildstep.RunInputOf(validationInputVerb, input) - if err != nil { - return buildstep.RunJob{}, err - } - dir := indexer.Path - if dir == "" { - dir = "." - } - return buildstep.RunJob{ - Program: buildstep.RunProgram{Repository: indexer.Repository, Commit: head, Dir: dir, Package: "./" + validationProgram}, - Input: in, - Reader: fmt.Sprintf("manifest validator of %s at %d", branchName(branch), seq), - }, nil -} - -func branchName(branch string) string { - if isTrunkBranch(branch) { - return TrunkBranch - } - return branch -} - -// ValidationRefusal is what a validation's verdict refuses (ADR 0579 §2, §712): of each manifest the change case refuses that -// the revision adds or changes (its bytes differ from the trunk case's), each problem the trunk case lacks; and each problem -// between manifests the trunk case lacks. A problem only in unchanged manifests refuses nothing. It is empty when the revision -// is accepted. An answer that holds not both cases is no verdict: it answers an error, so the run could not validate. -func ValidationRefusal(req composer.Request, v composer.Verdict) (map[string][]string, error) { - cases := map[string]composer.CaseVerdict{} - for _, c := range v.Cases { - cases[c.Name] = c - } - change, okChange := cases[validationChange] - trunkV, okTrunk := cases[validationTrunk] - if !okChange || !okTrunk { - return nil, fmt.Errorf("the manifest validator's answer holds not both cases %s and %s", validationChange, validationTrunk) - } - raws := map[string]map[string]string{} - for _, c := range req.Cases { - raws[c.Name] = map[string]string{} - for name, m := range c.Manifests { - raws[c.Name][name] = InputDigest(m.Raw) - } - } - refused := map[string][]string{} - for _, module := range slices.Sorted(maps.Keys(change.Manifests)) { - r := change.Manifests[module] - if r.Accepted || r.Refused == "" { - continue - } - before, had := raws[validationTrunk][module] - if module != judge.Unnamed && had && before == raws[validationChange][module] { - continue // unchanged: its problems refuse nothing, and its entry keeps its verdict (ADR 0579 §1) - } - old := map[string]bool{} - for _, p := range strings.Split(trunkV.Manifests[module].Refused, "\n") { - old[p] = true - } - for _, p := range strings.Split(r.Refused, "\n") { - if p != "" && !old[p] { - refused[module] = append(refused[module], p) - } - } - } - return refused, nil -} - -// RefusedWords is a refusal as the pending value's `refused` says it: each manifest and its problems, one manifest a line. -func RefusedWords(refused map[string][]string) string { - var lines []string - for _, module := range slices.Sorted(maps.Keys(refused)) { - lines = append(lines, module+": "+strings.Join(refused[module], "; ")) - } - return strings.Join(lines, "\n") -} diff --git a/modules/mesh-delivery-indexer/internal/index/validate_test.go b/modules/mesh-delivery-indexer/internal/index/validate_test.go deleted file mode 100644 index 73bd29a3..00000000 --- a/modules/mesh-delivery-indexer/internal/index/validate_test.go +++ /dev/null @@ -1,130 +0,0 @@ -package index - -import ( - "context" - "encoding/json" - "reflect" - "strings" - "testing" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer" - - "mesh-delivery-indexer/judge" -) - -// The validation refuses only what the revision causes (design 47 G §712, ADR 0579 §2), and its run names the indexer entry's -// repository, path and the revision's head, and no heads. -func TestValidationRefusesWhatTheRevisionCauses(t *testing.T) { - manifests := func(m map[string]string) map[string]composer.Manifest { - out := map[string]composer.Manifest{} - for k, v := range m { - out[k] = composer.Manifest{Raw: json.RawMessage(v)} - } - return out - } - req := composer.Request{Cases: []composer.Case{ - {Name: "change", Manifests: manifests(map[string]string{"a": `{"module":"a","v":2}`, "b": `{"module":"b"}`})}, - {Name: "trunk", Manifests: manifests(map[string]string{"a": `{"module":"a"}`, "b": `{"module":"b"}`})}, - }} - verdict := func(change, trunk map[string]string) composer.Verdict { - read := func(m map[string]string) map[string]composer.ManifestRead { - out := map[string]composer.ManifestRead{"a": {Accepted: true}, "b": {Accepted: true}} - for k, v := range m { - out[k] = composer.ManifestRead{Refused: v} - } - return out - } - return composer.Verdict{Cases: []composer.CaseVerdict{{Name: "change", Manifests: read(change)}, {Name: "trunk", Manifests: read(trunk)}}} - } - for _, c := range []struct { - name string - v composer.Verdict - want, wantErr string - }{ - {"a changed manifest gains a problem", verdict(map[string]string{"a": "new\nold"}, map[string]string{"a": "old"}), "a: new", ""}, - {"an unchanged manifest's problem refuses nothing", verdict(map[string]string{"b": "bad"}, nil), "", ""}, - {"a problem between manifests the trunk lacks", verdict(map[string]string{judge.Unnamed: "x\ny"}, map[string]string{judge.Unnamed: "y"}), judge.Unnamed + ": x", ""}, - {"an answer with one case is no verdict", composer.Verdict{Cases: verdict(nil, nil).Cases[:1]}, "", "not both cases"}, - } { - got, err := ValidationRefusal(req, c.v) - if c.wantErr != "" { - if err == nil || !strings.Contains(err.Error(), c.wantErr) { - t.Errorf("%s: %v, want %q", c.name, err, c.wantErr) - } - continue - } - if err != nil || RefusedWords(got) != c.want { - t.Errorf("%s: %q, %v, want %q", c.name, RefusedWords(got), err, c.want) - } - } - head := strings.Repeat("a", 40) - entry := meshcatalog.CatalogEntry{Module: "mesh-delivery-indexer", Repository: "novox/mesh-delivery", Path: "modules/mesh-delivery-indexer", - Artifacts: []meshcatalog.Artifact{{Language: "go", Kind: "bundle"}}} - job, err := validationJob(entry, head, "main", 7, []byte(`{"v":1}`)) - if err != nil { - t.Fatal(err) - } - want := buildstep.RunProgram{Repository: "novox/mesh-delivery", Commit: head, Dir: "modules/mesh-delivery-indexer", Package: "./cmd/mesh-judge"} - if job.Program != want || len(job.Heads) != 0 || len(job.Args) != 0 || job.Input.Verb != "seat:mesh-catalog.input" || job.Reader != "manifest validator of main at 7" { - t.Errorf("the job %+v", job) - } - if _, err := validationJob(meshcatalog.CatalogEntry{}, head, "main", 7, nil); err == nil || !strings.Contains(err.Error(), "claims the seat mesh-catalog") { - t.Errorf("no indexer entry: %v", err) - } -} - -// The validation's seats are the branch's revision's seat definitions with the merges' changed ones laid over them (A-241): a -// revision that touches no definition is judged against the whole seat list, and a merge's definition replaces its seat's. -func TestValidationSeatsAreTheRevisionsWithTheMergesOver(t *testing.T) { - ctx := context.Background() - defs := newMem() - for seat, raw := range map[string]string{"node-engine": `{"seat":"node-engine","verbs":[{"name":"old"}]}`, "git": `{"seat":"git"}`} { - v, _ := json.Marshal(SeatDefinitionEntry{Seat: seat, Source: SourceTrunk, Digest: "d-" + seat, Definition: json.RawMessage(raw)}) - defs.put(SeatDefinitionKey(seat, SourceTrunk, "", 0), v) - } - v, _ := json.Marshal(SeatDefinitionEntry{Seat: "pulled", Source: SourcePull, Pull: 3, Digest: "d", Definition: json.RawMessage(`{"seat":"pulled"}`)}) - defs.put(SeatDefinitionKey("pulled", SourcePull, "r", 3), v) - ix := &Indexer{Module: testSelf, SeatDefinitions: defs} - seatsOf := func(raws []json.RawMessage) map[string]string { - out := map[string]string{} - for _, raw := range raws { - var d struct { - Seat string `json:"seat"` - Verbs []struct { - Name string `json:"name"` - } `json:"verbs"` - } - if err := json.Unmarshal(raw, &d); err != nil { - t.Fatal(err) - } - out[d.Seat] = "" - if len(d.Verbs) > 0 { - out[d.Seat] = d.Verbs[0].Name - } - } - return out - } - got, err := ix.revisionDefinitions(ctx, nil) - if err != nil || !reflect.DeepEqual(seatsOf(got), map[string]string{"node-engine": "old", "git": ""}) { - t.Errorf("a revision touching no definition: %v (%v), want the trunk's whole list and no pull request's", seatsOf(got), err) - } - const contracts = "modules/seat-contracts" - f := forgemsg.Forge{Repository: "r", Merge: strings.Repeat("c", 40), Manifests: map[string]string{ - contracts: `{"module":"seat-contracts","version":"1.0","build":{"artifacts":[{"name":"go-seats","kind":"package","language":"go","paths":["go"],"series":"0.1","generated-from":"."}]}}`}} - ix.Files = func(_ context.Context, _, _ string, dirs []string) (map[string]map[string]string, error) { - return map[string]map[string]string{dirs[0]: {"node-engine/seat.json": "1"}}, nil - } - ix.ReadFile = func(_ context.Context, _, _, p string) ([]byte, error) { - if p == contracts+"/seats/node-engine/seat.json" { - return []byte(`{"format":1,"seat":"node-engine","scope":"node","grain":"seat-list","decision":"d","domain":"Core","verbs":[{"name":"new"}]}`), nil - } - return nil, nil - } - got, err = ix.revisionDefinitions(ctx, []PendingEntry{{Forge: f}}) - if err != nil || !reflect.DeepEqual(seatsOf(got), map[string]string{"node-engine": "new", "git": ""}) { - t.Errorf("a merge changing a definition: %v (%v), want its definition over the trunk's and the rest kept", seatsOf(got), err) - } -} diff --git a/modules/mesh-delivery-indexer/internal/index/validation_wait.go b/modules/mesh-delivery-indexer/internal/index/validation_wait.go deleted file mode 100644 index ec052ce4..00000000 --- a/modules/mesh-delivery-indexer/internal/index/validation_wait.go +++ /dev/null @@ -1,329 +0,0 @@ -package index - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "strings" - "time" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" -) - -// The pending revision's wait for its validation (design 47 G §713, A-232, A-247): the take-in puts the run ask of §712, writes -// `validation` on the revision's newest pending value and ends there; the build seat's step-ended of that ask, heard through the -// indexer's own durable consumer, resumes it: accepted takes the revision in as before, refused keeps it pending with `refused`, -// and an end that could not validate keeps it pending with `failed`, asked again by a later merge of the branch or by hand. - -// ValidationAsks puts a run ask on the build seat (buildrun.Asker). -type ValidationAsks interface { - Put(ctx context.Context, a buildstep.StepAsk) error -} - -// newestOfRevision is the index in ready of the revision's newest merge, the value that carries `validation` (A-247). -func newestOfRevision(ready []PendingEntry) int { - n := 0 - for i, e := range ready { - if e.Value.Seq > ready[n].Value.Seq { - n = i - } - } - return n -} - -// branchOfRevision is the watched branch the revision's merges landed on, its newest merged message's base (design 47 G §714); -// a message that names none is the trunk's. A pending value's own branch is its pull request's, never the branch validated. -func branchOfRevision(ready []PendingEntry) string { - if base := ready[newestOfRevision(ready)].Forge.Base; base != "" { - return base - } - return TrunkBranch -} - -// validationGate says whether the ready revision waits for its validation (§713): it goes on to the take-in once its newest -// value's validation was accepted at that value's sequence; it waits while its ask is out, while it is refused, and while it -// could not validate; otherwise it puts a new ask, the next try, and waits. Without an asker nothing is validated. -func (ix *Indexer) validationGate(ctx context.Context, ready []PendingEntry, own string) (bool, string, error) { - if ix.ValidationAsks == nil { - return false, "", nil - } - newest := ready[newestOfRevision(ready)] - seq := newest.Value.Seq - if v := newest.Value.Validation; v != nil && v.Seq == seq { - switch { - case v.Accepted == seq: - return false, "", nil - case v.Failed != "": - return true, fmt.Sprintf("the validation of the revision at %d could not validate: %s", seq, v.Failed), nil - case newest.Value.Refused != "": - return true, fmt.Sprintf("the revision at %d is refused: %s", seq, newest.Value.Refused), nil - case v.Ask != "": - return true, fmt.Sprintf("the revision at %d waits for its validation, the ask %s", seq, v.Ask), nil - } - } - return true, "", ix.askValidation(ctx, ready, own) -} - -// askValidation puts the revision's validation ask (§712) and writes it on its newest value, the try after every earlier try -// of the revision; an older value's validation goes (A-247: on the newest alone). A request or job that cannot be made could not -// validate: it is written as `failed`, said, and no ask is put. -func (ix *Indexer) askValidation(ctx context.Context, ready []PendingEntry, own string) error { - n := newestOfRevision(ready) - newest := ready[n] - try := 0 - for _, e := range ready { - if v := e.Value.Validation; v != nil && v.Try > try { - try = v.Try - } - } - v := &takein.Validation{Seq: newest.Value.Seq, Try: try + 1, Asked: ix.now()} - branch := branchOfRevision(ready) - ask := buildstep.StepAsk{ID: buildstep.NewID(), Kind: buildstep.AskCheck, Language: buildstep.AskGo, Asker: ValidationAsker, At: ix.now().UTC()} - req, entry, err := ix.validationRequest(ctx, branch, ready) - var raw []byte - if err == nil { - raw, err = json.Marshal(req) - } - var job buildstep.RunJob - if err == nil { - job, err = validationJob(entry, revisionHead(ready, entry.Repository, entry.Commit), branch, v.Seq, raw) - } - if err == nil { - v.Input, err = ix.KeepInput(ctx, ask.ID, raw) - } - if err != nil { - v.Failed = "the validation could not be asked: " + err.Error() - } else { - v.Ask, ask.Run = ask.ID, &job - } - if err := ix.writeValidation(ctx, ready, n, v, own); err != nil { - return err - } - if v.Failed != "" { - ix.sayValidation(ctx, ready, v) - return nil - } - if err := ix.ValidationAsks.Put(ctx, ask); err != nil { - return fmt.Errorf("the validation ask %s of the revision at %d could not be put: %w", ask.ID, v.Seq, err) - } - ix.logf("take-in: the revision at %d is validated by the ask %s (try %d); it waits for its end", v.Seq, ask.ID, v.Try) - return nil -} - -// revisionHead is the revision's head of repository: the newest merge of it the revision covers, else the commit its entry stands at. -func revisionHead(ready []PendingEntry, repository, commit string) string { - var seq uint64 - head := commit - for _, e := range ready { - if e.Forge.Merge != "" && strings.EqualFold(e.Forge.Repository, repository) && e.Value.Seq >= seq { - seq, head = e.Value.Seq, e.Forge.Merge - } - } - return head -} - -// writeValidation writes v on ready[n] and takes it from every other value, each by compare-and-set; a new validation clears the -// value's refusal. A value lost under it is an error: the evaluation is made again. -func (ix *Indexer) writeValidation(ctx context.Context, ready []PendingEntry, n int, v *takein.Validation, own string) error { - for i, e := range ready { - if i != n && e.Value.Validation == nil { - continue - } - p := e.Value - p.Validation, p.LastWriter = nil, own - if i == n { - p.Validation, p.Refused = v, "" - } - data, err := encodePending(p) - if err != nil { - return err - } - if err := ix.Pending.Update(ctx, e.Key, data, e.Revision); err != nil { - return fmt.Errorf("%s.%s: %w", PendingState, e.Key, err) - } - } - return nil -} - -// sayValidation says the revision's wait in take-in.pending with its refusal or why it could not validate. -func (ix *Indexer) sayValidation(ctx context.Context, ready []PendingEntry, v *takein.Validation) { - t := revisionWait(takein.TakeInPending, ready, "") - t.Failed = v.Failed - t.Refused = ready[newestOfRevision(ready)].Value.Refused - ix.sayTakeIn(ctx, t) -} - -// errNoValidationAnswer is an end whose answer is no verdict of both cases. -var errNoValidationAnswer = errors.New("no answer") - -// HearValidationEnded takes in the end of a validation ask (§713), heard on the build seat's step-ended through the indexer's own -// durable consumer: only an end of the indexer's asks outside a pipeline; an end whose ask no pending value names now is dropped, -// written in the journal. The answer is read from the event's own output or output object (A-247). -func (ix *Indexer) HearValidationEnded(ctx context.Context, e buildstep.StepEnded) error { - if e.Asker != ValidationAsker || e.Pipeline != "" || e.Ask == "" || ix.Pending == nil || !ix.pendingOnBus(ctx) { - return nil - } - own := ix.ownBuildNow(ctx) - ix.pendingMu.Lock() - defer ix.pendingMu.Unlock() - entries, err := ReadPending(ctx, ix.Pending, ix.logf) - if err != nil { - return err - } - ready := WaitsOf(entries).Ready - n := -1 - for i, p := range ready { - if p.Value.Validation != nil && p.Value.Validation.Ask == e.Ask { - n = i - } - } - if n < 0 { - ix.logf("take-in: the end of the validation ask %s (%s on %s) is dropped: no pending revision names that ask now", e.Ask, e.Outcome, e.Node) - return nil - } - ready, lost, _, _ := ix.loadNewest(ctx, ready) - if len(lost) > 0 { - return fmt.Errorf("the revision the validation ask %s ended for holds a merge that cannot be read", e.Ask) - } - n = newestOfRevision(ready) - v := *ready[n].Value.Validation - refused, failed := ix.validationAnswer(ctx, ready, e, v) - p := ready[n].Value - switch { - case failed != "": - v.Failed = failed - case refused != "": - p.Refused = refused - default: - v.Accepted = v.Seq - } - p.Validation, p.LastWriter = &v, own - data, err := encodePending(p) - if err != nil { - return err - } - if err := ix.Pending.Update(ctx, ready[n].Key, data, ready[n].Revision); err != nil { - return fmt.Errorf("%s.%s: %w", PendingState, ready[n].Key, err) - } - if v.Accepted == 0 { - ready[n].Value = p - ix.sayValidation(ctx, ready, &v) - ix.logf("take-in: the validation ask %s of the revision at %d ended: refused %q, could not validate %q", e.Ask, v.Seq, refused, failed) - return nil - } - ix.logf("take-in: the validation ask %s accepted the revision at %d", e.Ask, v.Seq) - if own == "" { - return nil // the take-in waits for the build this program runs, as every take-in does - } - _, _, err = ix.evaluate(ctx, own, false) - return err -} - -// validationAnswer reads the end's answer: the refusal of the revision, or why it could not validate (the step did not pass, the -// program exited other than 0, its answer is no verdict of both cases, or its request is no longer the one asked). -func (ix *Indexer) validationAnswer(ctx context.Context, ready []PendingEntry, e buildstep.StepEnded, v takein.Validation) (string, string) { - if e.Outcome != buildstep.OutcomePassed { - return "", strings.TrimSpace(e.Outcome + ": " + e.Said) - } - // The answer is the program's last line of standard output, inline or kept by the seat as an object (A-254); a program that - // exited other than 0 ended the step failed, read above. - out, err := e.ReadOutput(buildstep.AskGo, ix.AskVerb) - if err != nil { - return "", "the run's answer could not be read: " + err.Error() - } - var verdict composer.Verdict - if err := json.Unmarshal(out, &verdict); err != nil { - return "", fmt.Sprintf("%v: the manifest validator's output is no verdict: %v", errNoValidationAnswer, err) - } - req, _, err := ix.validationRequest(ctx, branchOfRevision(ready), ready) - var raw []byte - if err == nil { - raw, err = json.Marshal(req) - } - if err != nil { - return "", "the request cannot be made again to read the verdict: " + err.Error() - } - if runDigest(raw) != v.Input { - return "", "the revision's request is no longer the one the ask " + v.Ask + " was asked with" - } - refused, err := ValidationRefusal(req, verdict) - if err != nil { - return "", err.Error() - } - return RefusedWords(refused), "" -} - -// ValidateAgainTool asks the validation of a revision that could not validate again by hand (§713). -const ValidateAgainTool = "indexer_validate_again" - -// validationWords is the validation's state as the hand ask answers it. -func validationWords(v *takein.Validation, refused string) string { - switch { - case v == nil: - return "no validation is asked yet" - case v.Accepted != 0: - return fmt.Sprintf("accepted at %d (try %d, the ask %s)", v.Accepted, v.Try, v.Ask) - case v.Failed != "": - return fmt.Sprintf("could not validate at %d (try %d): %s", v.Seq, v.Try, v.Failed) - case refused != "": - return fmt.Sprintf("refused at %d (try %d): %s", v.Seq, v.Try, refused) - default: - return fmt.Sprintf("the ask %s is out since %s (try %d at %d)", v.Ask, v.Asked.UTC().Format(time.RFC3339), v.Try, v.Seq) - } -} - -// ValidateAgainByHand is indexer_validate_again answered (§713): the ready revision under key (its oldest merge's m. key, as -// indexer_pending names it) whose validation could not validate is asked again, the next try; its answer says the ask's state. -// A revision accepted, refused or whose ask is out is asked nothing: a refusal is asked again only by a later merge of its branch. -func (ix *Indexer) ValidateAgainByHand(ctx context.Context, key, by, why string) TakenIn { - r := TakenIn{Module: key} - if ix.Pending == nil || ix.ValidationAsks == nil { - r.Answer = "this indexer asks no validation: nothing asked" - return r - } - own := ix.ownBuildNow(ctx) - ix.pendingMu.Lock() - defer ix.pendingMu.Unlock() - entries, err := ReadPending(ctx, ix.Pending, ix.logf) - if err != nil { - r.Answer = err.Error() - return r - } - ready := WaitsOf(entries).Ready - if len(ready) == 0 || ready[0].Key != key { - r.Answer = fmt.Sprintf("no revision waits under %s: indexer_pending names the waits", key) - return r - } - ready, lost, reasons, _ := ix.loadNewest(ctx, ready) - if len(lost) > 0 { - r.Answer = fmt.Sprintf("nothing asked: the message of %s is not read (%s)", lost[0].Key, reasons[lost[0].Key]) - return r - } - newest := ready[newestOfRevision(ready)] - v := newest.Value.Validation - if v == nil || v.Seq != newest.Value.Seq || v.Failed == "" { - r.Answer = "nothing asked: the validation is " + validationWords(v, newest.Value.Refused) + "; only one that could not validate is asked again by hand" - return r - } - ix.logf("take-in: the validation of %s is asked again by hand by %s: %s", key, by, why) - if err := ix.askValidation(ctx, ready, own); err != nil { - r.Answer = "the validation was not asked again: " + err.Error() - return r - } - after, err := ReadPending(ctx, ix.Pending, ix.logf) - if err == nil { - ready = WaitsOf(after).Ready - } - if err != nil || len(ready) == 0 { - r.Answer = "asked again; its state is not read back" - return r - } - n := ready[newestOfRevision(ready)] - r.Stored = n.Value.Validation != nil && n.Value.Validation.Failed == "" - r.Answer = fmt.Sprintf("asked again by %s (%s): the validation is %s", by, why, validationWords(n.Value.Validation, n.Value.Refused)) - return r -} diff --git a/modules/mesh-delivery-indexer/internal/index/validation_wait_test.go b/modules/mesh-delivery-indexer/internal/index/validation_wait_test.go deleted file mode 100644 index 9588293b..00000000 --- a/modules/mesh-delivery-indexer/internal/index/validation_wait_test.go +++ /dev/null @@ -1,113 +0,0 @@ -package index - -import ( - "context" - "encoding/json" - "strings" - "testing" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer" -) - -type askRecorder struct{ asks []buildstep.StepAsk } - -func (r *askRecorder) Put(_ context.Context, a buildstep.StepAsk) error { - r.asks = append(r.asks, a) - return nil -} - -// Design 47 G §713, A-247: a pending revision names its ask on its newest value and resumes on that ask's end; an end of a -// replaced ask is dropped; a run that could not validate is no refusal and never passed; the start asks nothing again while -// the ask is out. -func TestAPendingRevisionWaitsForItsValidation(t *testing.T) { - ctx := context.Background() - b := newTakeInBed(t) - asks := &askRecorder{} - b.ix.ValidationAsks, b.ix.Inputs, b.ix.Revisions = asks, newMem(), newMemObjects(0) - // No entry of the revision claims the seat mesh-catalog: the validation could not be asked, so it is failed, said, and - // nothing is taken in. - b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "")) - entries := readPending(t, b) - if len(entries) != 1 || entries[0].Value.Validation == nil || !strings.Contains(entries[0].Value.Validation.Failed, "claims the seat mesh-catalog") || len(asks.asks) != 0 { - t.Fatalf("pending %+v, asks %d: want m.2 pending with failed and no ask", entries, len(asks.asks)) - } - if last := b.said[len(b.said)-1]; last.Kind != takein.TakeInPending || last.Failed == "" || last.Refused != "" { - t.Errorf("said %+v, want take-in.pending naming failed, never a refusal", last) - } - if w := PendingWaits(entries); len(w) != 1 || w[0].Validation == nil || w[0].Validation.Try != 1 { - t.Errorf("indexer_pending %+v, want the validation whole", w) - } - // An ask out: the newest value names it; the start asks nothing new while it is out. - loaded, _, _, _ := b.ix.loadNewest(ctx, entries) - req, _, err := b.ix.validationRequest(ctx, TrunkBranch, loaded) - if err != nil { - t.Fatal(err) - } - raw, _ := json.Marshal(req) - setValidation := func(v takein.Validation) { - e := readPending(t, b)[0] - p := e.Value - p.Validation = &v - data, _ := encodePending(p) - if err := b.pending.Update(ctx, e.Key, data, e.Revision); err != nil { - t.Fatal(err) - } - } - setValidation(takein.Validation{Ask: "mvl0frv18d1957", Input: runDigest(raw), Seq: 2, Try: 2}) - if _, _, err := b.ix.TakeInAtStart(ctx); err != nil || len(asks.asks) != 0 || len(b.keys()) != 1 { - t.Fatalf("the start: %v, asks %d, pending %v: want it waiting for its ask", err, len(asks.asks), b.keys()) - } - end := func(ask, outcome, said string, answer any) buildstep.StepEnded { - out, _ := json.Marshal(answer) - return buildstep.StepEnded{Ask: ask, ID: ask, Asker: ValidationAsker, Node: "ace", Outcome: outcome, Said: said, Output: string(out)} - } - // The answer is the manifest validator's verdict itself, the program's last line (A-254). - accepted := composer.Verdict{Cases: []composer.CaseVerdict{{Name: "change"}, {Name: "trunk"}}} - // The end of a replaced ask is dropped, written in the journal. - if err := b.ix.HearValidationEnded(ctx, end("mvl0frv18d1950", buildstep.OutcomePassed, "", accepted)); err != nil || len(b.keys()) != 1 || !strings.Contains(b.logs.String(), "is dropped") { - t.Fatalf("a replaced ask's end: %v, pending %v", err, b.keys()) - } - // could not run: pending with failed, never passed. - if err := b.ix.HearValidationEnded(ctx, end("mvl0frv18d1957", buildstep.OutcomeCouldNotRun, "no build node", nil)); err != nil { - t.Fatal(err) - } - if v := readPending(t, b)[0].Value.Validation; v == nil || v.Failed != "could not run: no build node" || v.Accepted != 0 { - t.Fatalf("after could not run: %+v", v) - } - // indexer_validate_again (§713): its hand act recorded first, then the next try asked; the answer says the ask's state. - var recorded []string - tools := ValidationTools(func(key string, value []byte) (TakeInAsk, error) { - a, err := DecodeTakeIn(value) - if err != nil || key != "pending.validate-again."+takein.PendingKey(2) || !a.ValidateAgain { - t.Fatalf("asked %s %+v %v", key, a, err) - } - r := b.ix.ValidateAgainByHand(ctx, a.Pending, a.By, a.Why) - a.Result = &r - return a, nil - }, b.ix.now, func(what, why string) error { recorded = append(recorded, what); return nil }) - if len(tools) != 1 || tools[0].Name != ValidateAgainTool { - t.Fatalf("tools = %v", tools) - } - if _, err := tools[0].Run(map[string]any{"key": takein.PendingKey(2)}); err == nil || len(recorded) != 0 { - t.Fatalf("asked again with no why: %v, %v", err, recorded) - } - got, err := tools[0].Run(map[string]any{"key": takein.PendingKey(2), "why": "the build node is back"}) - if v := readPending(t, b)[0].Value.Validation; err != nil || len(recorded) != 1 || v == nil || v.Try != 3 || v.Failed == "could not run: no build node" || - !strings.Contains(got.(*TakenIn).Answer, "try 3") { - t.Fatalf("asked again: %v, recorded %v, validation %+v, answer %+v: want the try after 2 asked", err, recorded, v, got) - } - setValidation(takein.Validation{Ask: "mvl0frv18d1959", Input: runDigest(raw), Seq: 2, Try: 3}) - if r := b.ix.ValidateAgainByHand(ctx, takein.PendingKey(2), "lead", "again"); r.Stored || !strings.Contains(r.Answer, "is out") || readPending(t, b)[0].Value.Validation.Try != 3 { - t.Fatalf("an ask out was asked again: %+v", r) - } - // Asked again (by hand or by a later merge) and accepted: the revision is taken in, and its keys go. - setValidation(takein.Validation{Ask: "mvl0frv18d1958", Input: runDigest(raw), Seq: 2, Try: 3}) - if err := b.ix.HearValidationEnded(ctx, end("mvl0frv18d1958", buildstep.OutcomePassed, "", accepted)); err != nil { - t.Fatal(err) - } - if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('c') { - t.Fatalf("accepted: pending %v %+v, avahi at %s: want it taken in\n%s", b.keys(), readPending(t, b)[0].Value, b.commit(t, "avahi"), b.logs.String()) - } -} diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index cc051196..ff8aef65 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -204,9 +204,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered. index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool, index.EntryIndexTool, index.NamedDropTool, // design 47 G §675 - index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.UnpinTool, index.ConsumerTool, index.RunsTool, index.RunDropTool, // design 47 G §667, §8, §579, §64 (A-194) - index.ValidateAgainTool) // design 47 G §713 - if !reflect.DeepEqual(m.Tools, want) || len(want) != 52 || !slices.Contains(m.Tools, "modules") { + index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.UnpinTool, index.ConsumerTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194) + if !reflect.DeepEqual(m.Tools, want) || len(want) != 51 || !slices.Contains(m.Tools, "modules") { t.Errorf("tools = %v, want %v", m.Tools, want) } // Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section. diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 6d387c43..e4c88a65 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -77,8 +77,7 @@ "indexer_unpin", "indexer_consumer", "indexer_runs", - "indexer_run_drop", - "indexer_validate_again" + "indexer_run_drop" ], "consumes": [ "go-package-registry.published", diff --git a/modules/mesh-delivery-indexer/takein/takein.go b/modules/mesh-delivery-indexer/takein/takein.go index 66923591..8e5ba0d1 100644 --- a/modules/mesh-delivery-indexer/takein/takein.go +++ b/modules/mesh-delivery-indexer/takein/takein.go @@ -266,21 +266,6 @@ type PendingWait struct { Unread []string `json:"unread,omitempty"` // Refused is why the manifest of the indexer it names was refused (TakeIn.Refused). Refused string `json:"refused,omitempty"` - // Validation is the revision's validation whole, as its newest pending value keeps it (design 47 G §713, A-247). - Validation *Validation `json:"validation,omitempty"` -} - -// Validation is a pending revision's validation (design 47 G §712, §713, A-232, A-247), kept on the revision's newest pending -// value alone: the run ask's id, when it was asked, its input's digest, the sequence of the newest merge it covers and the try -// number for it, why it could not validate, and the sequence it was accepted at (never asked again once set). -type Validation struct { - Ask string `json:"ask,omitempty"` - Asked time.Time `json:"asked,omitzero"` - Input string `json:"input,omitempty"` - Seq uint64 `json:"seq"` - Try int `json:"try"` - Failed string `json:"failed,omitempty"` - Accepted uint64 `json:"accepted,omitempty"` } // PendingAnswer is indexer_pending's answer: every wait, oldest first, and the pending keys that do not read. From 40b6ced375888ced01a44326320c660dcd22fd5c Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 20:22:55 +0100 Subject: [PATCH 02/20] A-476a: a commit moves its own repository's modules and the importers of what it moved; A-471 toolchain test data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The recompute of a merge leaves a module of another repository as held unless its imported package versions changed (ADR 0593 §3 stands). TestIndexerCountsOnlyThePlacedBuildNodesToolchains reports language versions (A-471). Builds: design 47 G A-476a, A-471 --- .../cmd/mesh-delivery-indexer/integration_test.go | 11 ++++++----- .../packages_integration_test.go | 10 +--------- .../internal/index/toolchains.go | 8 +++++--- .../internal/index/toolchains_test.go | 11 ++--------- 4 files changed, 14 insertions(+), 26 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index 95e55326..2e9dde7f 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -1418,7 +1418,8 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) { t.Fatal(err) } toolchains = kv - for node, version := range map[string]string{"n1": "go1.27.3", "n2": "go1.27.2"} { + // A-471: a toolchain is agreed by its language version (go1.27), not its patch; n2 reports another language version. + for node, version := range map[string]string{"n1": "go1.27", "n2": "go1.28.1"} { key, _ := index.ToolchainKey(node, "go") value, err := index.EncodeToolchain(nodebuildgo.Toolchain{Language: "go", Version: version, Recipe: "1", At: ptr(time.Now())}) if err != nil { @@ -1462,15 +1463,15 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) { return got } agreed := func(m meshcatalog.ModulesByModuleValue) bool { - return m.Fp2 != "" && m.Toolchains["go"].Version == "go1.27.3" + return m.Fp2 != "" && m.Toolchains["go"].Version == "go1.27" } held := read(agreed) if !agreed(held) { - t.Fatalf("within 20 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27.3 alone", held.Fp2, held.None2, held.Toolchains) + t.Fatalf("within 20 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27 alone", held.Fp2, held.None2, held.Toolchains) } place(ctx, t, "n2", "mesh-build-go") key, _ := index.ToolchainKey("n2", "go") - value, err := index.EncodeToolchain(nodebuildgo.Toolchain{Language: "go", Version: "go1.27.2", Recipe: "1", At: ptr(time.Now().Add(time.Second))}) + value, err := index.EncodeToolchain(nodebuildgo.Toolchain{Language: "go", Version: "go1.28.1", Recipe: "1", At: ptr(time.Now().Add(time.Second))}) if err != nil { t.Fatal(err) } @@ -1493,7 +1494,7 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) { t.Fatal(err) } disagree := func(m meshcatalog.ModulesByModuleValue) bool { - return m.Fp2 == "" && strings.HasPrefix(m.None2, "builders disagree: go1.27.2 recipe 1 on n2") + return m.Fp2 == "" && strings.HasPrefix(m.None2, "builders disagree: go1.27 recipe 1 on n1; go1.28 recipe 1 on n2") } if m := read(disagree); !disagree(m) { t.Fatalf("within 20 s of a merge after n2's assignments avahi has fp2 %q, none2 %q: want the builders to disagree", m.Fp2, m.None2) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go index 1c9c5fed..7751e791 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go @@ -206,15 +206,7 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T) if pub.Commit != m2 || len(pub.Packages) != 3 { t.Fatalf("packages.%s = %+v within 20 s, want the merge %s with a digest of go, announce and buildstep", key, pub, m2) } - // A-476 c: mesh-sdk's merge moved no module registered for another repository: importer stands as it was. - if now := module("importer", func(meshcatalog.ModulesByModuleValue) bool { return true }); now.Fp2 != importer.Fp2 || now.Packages[sdkGo] != m1 { - t.Errorf("mesh-sdk's merge moved importer of novox/mesh-catalog: %+v, want it unmoved at %s", now.Packages, m1) - } - // The next merge of its own repository moves it, built against the package published meanwhile. - merged("novox/mesh-catalog", 302, "8888888888888888888888888888888888888888", forgemsg.Forge{Root: "r0", - Trees: map[string]string{"modules/importer": "t1", "modules/plain": "t2"}, - Manifests: map[string]string{"modules/importer": named("importer"), "modules/plain": named("plain")}, - Locks: map[string]map[string]string{"modules/importer": {"go.sum": goSumHash}, "modules/plain": {"go.sum": goSumHash}}}) + // A-476a: mesh-sdk's merge moves importer, of another repository, which imports a package it moved. moved := module("importer", func(m meshcatalog.ModulesByModuleValue) bool { return m.Fp2 != importer.Fp2 }) if moved.Packages[sdkGo] != m2 { t.Errorf("importer moved to %+v, want it built against %s", moved.Packages, m2) diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index 4d80b405..d500c088 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -134,7 +134,7 @@ func (ix *Indexer) Recompute(ctx context.Context) error { return ix.recomputeWit // of what the bucket holds, are computed with the others, and are written once, with their final fingerprint or reason // (issue 1444: a merge writes each module once). A record of the overlay that has no kept inputs is written as it is. w is // the merge's write (order.go), nil for none; it is told each module written. A merge's recompute names its repository -// (A-476 c): only the modules registered for it are moved, none of another repository; "" moves every module (a toolchain +// (A-476 c, A-476a): the modules registered for it are moved, and those of another repository only when they import a package it moved; "" moves every module (a toolchain // report, a hand recompute). func (ix *Indexer) recomputeWith(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite, repository string) error { if err := ix.fillPaths(ctx); err != nil { @@ -310,8 +310,10 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc for _, key := range slices.Sorted(maps.Keys(all)) { k := all[key] r := of(key, nil) - if _, merged := overlay[key]; !merged && repository != "" && !trunk.SameRepository(k.cur.Repository, repository) { - // A module registered for another repository is not moved by this repository's commit (A-476 c): it stands as held. + if _, merged := overlay[key]; !merged && repository != "" && !trunk.SameRepository(k.cur.Repository, repository) && + maps.Equal(k.held.Packages, map[string]string(r.pkgs)) { + // A module registered for another repository is moved by this repository's commit only when it imports a package the + // commit moved (A-476a, ADR 0593 §3); any other stands as held (A-476 c). r = result{fp: k.held.Fp2, none: k.held.None2, tcs: k.held.Toolchains, pkgs: sdkstep.PackageVersions(k.held.Packages)} edges[key] = graphEdges{module: key, cur: k.cur, manifest: k.manifest, fp: r.fp, none: r.none, tcs: r.tcs, pkgs: r.pkgs} continue diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index a62e7d0d..c8675341 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -829,18 +829,11 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) { t.Helper() f := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-sdk", Branch: "feat/x", Number: 3, Head: h1, Merge: commit, Root: tree, Trees: map[string]string{"": tree}, Manifests: map[string]string{"": `{"module":"mesh-sdk","version":"1.0"}`}} - prev, _ := moduleOf(t, mods, "avahi") if err := ix.Apply(context.Background(), f); err != nil { t.Fatal(err) } - // A-476 c: a commit of mesh-sdk moves no module registered for another repository; the next recompute (a hand one here) - // moves avahi, which imports its package. - if after, _ := moduleOf(t, mods, "avahi"); after.Fp2 != prev.Fp2 || !reflect.DeepEqual(after.Packages, prev.Packages) { - t.Fatalf("a commit of mesh-sdk moved avahi, registered for another repository: fp2 %q (was %q) packages %v (were %v)", after.Fp2, prev.Fp2, after.Packages, prev.Packages) - } - if err := ix.Recompute(context.Background()); err != nil { - t.Fatal(err) - } + // A-476a: a commit of mesh-sdk moves avahi, of another repository, which imports its package, and leaves mesh-tools, + // which imports nothing it moved. a, _ := moduleOf(t, mods, "avahi") if a.Fp2 == "" || !reflect.DeepEqual(a.Packages, map[string]string{sdk: commit}) { t.Fatalf("after mesh-sdk's merge %.8s: fp2 %q none2 %q packages %v", commit, a.Fp2, a.None2, a.Packages) From a79b8111139520bf878f273830246c5aae977dfb Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 20:33:52 +0100 Subject: [PATCH 03/20] A-476 r1: no merge waits for the build the indexer runs; leftovers of the validation and the hold go; a commit's scope is decided inside the recompute ErrOwnBuildUnknown, its say and the own-build-unknown wait, TakeIn.Unknown and Failed go; the build only names the writer. indexer_take_in_pending takes only a group in. The node-build-go accept.check and output invokes go. A module of another repository stays as held inside of(), so a base of this repository's module is not given a value that is never written (A-476 c, A-476a); one case in the toolchains test proves it. Builds: design 47 G A-476, A-476a --- .../internal/index/behind.go | 2 +- .../internal/index/pending.go | 79 +++++---------- .../internal/index/pending_test.go | 95 ++++++------------- .../internal/index/pending_tools.go | 22 ++--- .../internal/index/takein.go | 4 +- .../internal/index/toolchains.go | 16 ++-- .../internal/index/toolchains_test.go | 14 ++- .../internal/index/unreadcall.go | 2 +- .../internal/index/verdicts.go | 9 +- .../mesh-delivery-indexer/manifest_test.go | 2 - modules/mesh-delivery-indexer/module.json | 2 - .../mesh-delivery-indexer/takein/takein.go | 31 +++--- .../takein/takein_test.go | 5 +- 13 files changed, 103 insertions(+), 180 deletions(-) diff --git a/modules/mesh-delivery-indexer/internal/index/behind.go b/modules/mesh-delivery-indexer/internal/index/behind.go index 81c02951..d7906ae9 100644 --- a/modules/mesh-delivery-indexer/internal/index/behind.go +++ b/modules/mesh-delivery-indexer/internal/index/behind.go @@ -91,7 +91,7 @@ func (ix *Indexer) TakeInBehind(ctx context.Context) []string { } switch { case held[head]: - // A head the pending holds (a member waiting for its group's end or its build) is taken in by that path, whole with + // A head the pending holds (a member waiting for its group's end) is taken in by that path, whole with // its group: it is not behind (A-463a). ix.clearUntaken(ctx, key, "the head of "+repository+" waits in the indexer's pending") case merge == head: diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index a5bc1f15..94b4697a 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -24,9 +24,8 @@ import ( "mesh-delivery-indexer/judge" ) -// The take-in step (novox/hq ADR 0633): a source revision is taken in only by the indexer it names. A merged message is -// kept in the indexer's own state pending, one key per merge, until the take-in that ends it; a revision naming another -// build of the indexer writes only what building and moving that indexer needs, and the rest stays pending for it. +// The take-in step (novox/hq ADR 0633, A-476): a merged message is kept in the indexer's own state pending, one key per +// merge, until the take-in that ends it; a revision is taken in by the running indexer whatever build of the indexer it names. const ( // PendingState is the indexer's state of held merges: the bucket mesh-delivery-indexer_pending, keyed // takein.PendingKey, so the key order is the merge order. An old `state` entry (the indexer is foundation, ADR @@ -45,10 +44,6 @@ var errClaimLost = errors.New("another take-in claimed the merges first, so this // runs such a build, so the revision waits until a later merge gives it one or a hand take-in (indexer_take_in_pending). const NoBuild = "no build: " -// ErrOwnBuildUnknown is what a merged message answers while the indexer cannot learn the build it runs (ADR 0633 §1): -// the message is left unacknowledged, so the bus delivers it again, and nothing is written. -var ErrOwnBuildUnknown = errors.New("the build of the indexer this program runs is not known") - // PendingRef is what a pending value keeps of its merged message instead of a copy (a merge of a repository with many // modules is about 0.5 MB, over the state's 256 KiB value cap; issue 1585): the small fields the waits are worked out // from. The message itself is read from the forge bridge's stream by Pending.Seq when a take-in needs it. @@ -384,26 +379,13 @@ func (ix *Indexer) pendingOnBus(ctx context.Context) bool { } // ownBuildNow is the build this program runs, as FixOwnBuild fixed it at the start: empty while unknown, and never asked again. +// It names only the writer of what is written (LastWriter); nothing waits for it (A-476 b). func (ix *Indexer) ownBuildNow(context.Context) string { ix.ownMu.Lock() defer ix.ownMu.Unlock() return ix.ownFP } -// sayOwnBuildUnknown says, for a merge this process cannot take in because it does not know its own build (ADR 0633 §1), a -// take-in.pending that names why and since when, so the watchman's take-in-waits bound covers it. The merge is not written to -// pending; the stream delivers it again to the process that knows its build. -func (ix *Indexer) sayOwnBuildUnknown(ctx context.Context, seq uint64, f forgemsg.Forge) { - ix.ownMu.Lock() - since, why := ix.ownSince, ix.ownWhy - ix.ownMu.Unlock() - if since.IsZero() { // never fixed: the start did not run (no engine wired) - since, why = ix.now(), "the build this program runs was never asked" - } - ix.sayTakeIn(ctx, takein.TakeIn{Kind: takein.TakeInPending, Key: takein.PendingKey(seq), What: takein.TakeInRevision, - Merges: []takein.Merge{{Repository: f.Repository, Number: f.Number, Merge: f.Merge}}, Since: since, Unknown: why}) -} - // sayTakeIn publishes a take-in event and keeps what it said on the entries, so indexer_pending reads the last event of // each wait from the state. A publish that fails is said in the log: the next evaluation says it again. func (ix *Indexer) sayTakeIn(ctx context.Context, t takein.TakeIn) bool { @@ -451,10 +433,6 @@ func (ix *Indexer) Arrive(ctx context.Context, seq uint64, f forgemsg.Forge) (Ap return a, "", err } own := ix.ownBuildNow(ctx) - if own == "" { - ix.sayOwnBuildUnknown(ctx, seq, f) - return Applied{}, "", ErrOwnBuildUnknown - } ix.pendingMu.Lock() defer ix.pendingMu.Unlock() key := takein.PendingKey(seq) @@ -497,9 +475,6 @@ func (ix *Indexer) EndGroup(ctx context.Context, end forgemsg.ForgeGroupEnd) (Ap return Applied{}, "no pending state: the group's merges were taken in one by one", nil } own := ix.ownBuildNow(ctx) - if own == "" { - return Applied{}, "", ErrOwnBuildUnknown - } ix.pendingMu.Lock() defer ix.pendingMu.Unlock() entries, err := ReadPending(ctx, ix.Pending, ix.logf) @@ -672,21 +647,15 @@ func (ix *Indexer) takenAlready(ctx context.Context, m forgemsg.GroupMerge) (boo } // TakeInAtStart is the take-in at the start (ADR 0633 §2), before the consumer on the forge bridge's stream starts: -// it evaluates pending once with the build this program runs, taking in the whole revisions that name this build; a -// group still being gathered stays, and a revision naming yet another build is held for that one. A claim left by an -// indexer that ran before is void here. When the build is not known (the node-engine did not answer), it waits for -// nothing: it says so and the consumer starts all the same; a merged message is then left unacknowledged until the -// build is known (clause 1), and its redelivery, or a toolchain report (ToolchainReported), asks again. No timer. +// it evaluates pending once, taking in the whole revisions whatever build of the indexer they name (A-476 b); a group +// still being gathered stays. A claim left by an indexer that ran before is void here. The build this program runs, known +// or not, only names the writer. No timer. func (ix *Indexer) TakeInAtStart(ctx context.Context) (Applied, string, error) { if ix.Pending == nil || !ix.pendingOnBus(ctx) { // Nothing can wait in a state not on the bus: the start asks no build and does not wait for one. return Applied{}, "the pending state is not on the bus yet: nothing waits", nil } own := ix.ownBuildNow(ctx) - if own == "" { - ix.logf("take-in: the build this program runs is not known at its start, so nothing pending is taken in now; the forge consumer starts, and a merged message left unacknowledged asks it again") - return Applied{}, "the build this program runs is not known yet: nothing pending is taken in at the start", nil - } ix.pendingMu.Lock() defer ix.pendingMu.Unlock() a, words, err := ix.evaluate(ctx, own, true) @@ -694,8 +663,8 @@ func (ix *Indexer) TakeInAtStart(ctx context.Context) (Applied, string, error) { return a, "the pending state is not on the bus yet: nothing waits", nil } if err == nil && len(a.Written) == 0 { - // A start that takes nothing in says so, with the build it learnt (the wait's own build is in the line holdFor says). - ix.logf("take-in: at the start, this program runs build %s and took nothing in: %s", own, words) + // A start that takes nothing in says so. + ix.logf("take-in: at the start, this program (build %q) took nothing in: %s", own, words) } return a, words, err } @@ -726,16 +695,12 @@ func (ix *Indexer) UnreadAgain(ctx context.Context) error { } // PendingAgain evaluates pending again after something a fingerprint is computed from changed, a toolchain report -// (ADR 0633 §1, §3), or the verdicts state bound (issue 1966): a revision that now names the running build is taken in, and one that names another build is held -// for that one. It does nothing while the pending state is not on the bus or the build this program runs is not known. +// (ADR 0633 §3), or the verdicts state bound (issue 1966): what waits is taken in. It does nothing while the pending state is not on the bus. func (ix *Indexer) PendingAgain(ctx context.Context) error { if ix.Pending == nil || !ix.pendingOnBus(ctx) { return nil } own := ix.ownBuildNow(ctx) - if own == "" { - return nil - } ix.pendingMu.Lock() defer ix.pendingMu.Unlock() _, words, err := ix.evaluate(ctx, own, false) @@ -881,12 +846,12 @@ func (ix *Indexer) holdLost(ctx context.Context, ready, lost []PendingEntry, rea } for _, g := range lost { if unread[g.Key] { - ix.logf("take-in: %s (%s#%d) is not taken in and holds its revision, its message is unread for a transport fault: %s; it is read again on the next message heard on the forge, at the start, and by %s", - g.Key, g.Forge.Repository, g.Forge.Number, reasons[g.Key], TakeInPendingTool) + ix.logf("take-in: %s (%s#%d) is not taken in and holds its revision, its message is unread for a transport fault: %s; it is read again on the next message heard on the forge and at the start", + g.Key, g.Forge.Repository, g.Forge.Number, reasons[g.Key]) continue } - ix.logf("take-in: %s (%s#%d) is not taken in and holds its revision, the forge bridge could not give its message: %s; %s reads it again; indexer_drop_pending drops it once a later merge of %s covers it", - g.Key, g.Forge.Repository, g.Forge.Number, reasons[g.Key], TakeInPendingTool, g.Forge.Repository) + ix.logf("take-in: %s (%s#%d) is not taken in and holds its revision, the forge bridge could not give its message: %s; indexer_drop_pending drops it once a later merge of %s covers it", + g.Key, g.Forge.Repository, g.Forge.Number, reasons[g.Key], g.Forge.Repository) } if !ix.sayTakeIn(ctx, t) { return nil @@ -1222,8 +1187,7 @@ func (ix *Indexer) nextSource(branch string) int64 { // takenWaits are the taken events of a take-in: every wait the entries were said under (saidWaits), every wait // indexer_pending names the claimed entries by now (WaitsOf: a revision by its oldest merge's key, a group by its g. key) and // the key of every claimed entry, since indexer_pending named it by that key earlier when it waited behind a group. So a wait -// the watchman learned of without the indexer marking it (seeded from indexer_pending, or said by the own-build-unknown -// path) ends too (design 47, Taken in by the indexer it names). No key is said twice. +// the watchman learned of without the indexer marking it (seeded from indexer_pending) ends too. No key is said twice. func takenWaits(entries []PendingEntry) []takein.TakeIn { out := saidWaits(entries) said := map[string]bool{} @@ -1354,9 +1318,9 @@ func waitOf(entries []PendingEntry, what, behind string) takein.PendingWait { return pw } -// TakeInPendingByHand is the hand take-in (indexer_take_in_pending, ADR 0633 §5): the wait named by its key, the -// ready revision (its m. key) or a group gathered so far (its g. key), is taken in with this indexer and its judge, -// whatever build it names, for when the named indexer can never run or the group's end never comes. by and why are +// TakeInPendingByHand is the hand take-in (indexer_take_in_pending, ADR 0633 §5, A-476 b): the group gathered so far, named by +// its g. key, is taken in with this indexer and its judge, +// whatever build it names, for when the group's end never comes. by and why are // said in the journal and on the take-in.taken event. What waited behind it is evaluated again after. func (ix *Indexer) TakeInPendingByHand(ctx context.Context, key, by, why string) TakenIn { r := TakenIn{Module: key} @@ -1381,8 +1345,9 @@ func (ix *Indexer) TakeInPendingByHand(ctx context.Context, key, by, why string) take = held } } - case len(w.Ready) > 0 && w.Ready[0].Key == key: - take = w.Ready + default: + r.Answer = fmt.Sprintf("%s names no delivery group: only a group is taken in by hand (a revision is taken in by the indexer itself); indexer_pending names the waits", key) + return r } if len(take) == 0 { r.Answer = fmt.Sprintf("nothing waits under %s: indexer_pending names the waits", key) @@ -1460,8 +1425,8 @@ func (ix *Indexer) DropPending(ctx context.Context, key, by, why string, abandon } if !covered { r.Answer = fmt.Sprintf("%s is not dropped: no later merge of %s, pending or taken in, has %s as an ancestor, so its "+ - "changes would never be taken in (ADR 0633 §5); %s reads its message again, or abandon: true drops it all the same", - key, e.Forge.Repository, e.Forge.Merge, TakeInPendingTool) + "changes would never be taken in (ADR 0633 §5); abandon: true drops it all the same", + key, e.Forge.Repository, e.Forge.Merge) return r } } diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index 6f22c27a..950b0fbf 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -186,53 +186,21 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) }) } }) - t.Run("T6 a merge while the own build is unknown writes nothing", func(t *testing.T) { + t.Run("T6 a merge while the own build is unknown is taken in all the same (A-476 b)", func(t *testing.T) { b := newTakeInBed(t) b.ix.ownFP, b.ix.ownFixed = "", false // this process is started again and asks the engine, which does not answer b.ix.OwnBuild = func(context.Context) (string, error) { return "", errors.New("the engine does not answer") } b.ix.FixOwnBuild(ctx) b.said = nil - _, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", "")) - if !errors.Is(err, ErrOwnBuildUnknown) || len(b.keys()) != 0 || b.commit(t, "avahi") == sha('b') { - t.Errorf("err %v, pending %v, avahi at %s: want ErrOwnBuildUnknown and nothing written", err, b.keys(), b.commit(t, "avahi")) + if _, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", "tree-indexer-2")); err != nil || len(b.keys()) != 0 || b.commit(t, "avahi") != sha('b') { + t.Errorf("err %v, pending %v, avahi at %s: want the merge taken in at once", err, b.keys(), b.commit(t, "avahi")) } - // The merge is said pending with why and since when (so the watchman's bound covers it), and indexer_pending says the - // build is unknown, from the own-build state. - if len(b.said) != 1 || b.said[0].Kind != takein.TakeInPending || b.said[0].Key != takein.PendingKey(1) || !strings.Contains(b.said[0].Unknown, "does not answer") || - b.said[0].Since.IsZero() || b.said[0].WaitsFor != "" { - t.Errorf("said %+v, want one take-in.pending of m.1 saying why the own build is unknown", b.said) - } else if _, err := takein.EncodeTakeIn(b.said[0]); err != nil { - t.Errorf("the unknown-build event is refused by its own contract: %v", err) + if len(b.said) != 1 || b.said[0].Kind != takein.TakeInTaken { + t.Errorf("said %+v, want one take-in.taken and no wait said", b.said) } answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{}) - if a, _ := answer.(takein.PendingAnswer); err != nil || !strings.Contains(a.OwnBuildUnknown, "does not answer") || a.OwnBuildSince.IsZero() { - t.Errorf("indexer_pending = %+v %v, want the build unknown with why and since", answer, err) - } - // Before the pending state is on the bus nothing waits: the start asks no build, and the merge is taken in at once, - // as before ADR 0633, though the build is unknown (TestIndexerWhole found the start waiting on it for ever). - b.ix.Pending = &memStore{unbound: process.ErrNotYet} - if _, _, err := b.ix.TakeInAtStart(ctx); err != nil { - t.Fatal(err) - } - if _, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", "")); err != nil || b.commit(t, "avahi") != sha('b') { - t.Errorf("no pending state: err %v, avahi at %s: want the merge taken in at once", err, b.commit(t, "avahi")) - } - }) - t.Run("1966/AC195-2 AC195-3 a merge said pending with its build unknown and taken in on redelivery ends that wait under its key; a revision wait is said once", func(t *testing.T) { - b := newTakeInBed(t) - ownBuild := b.ix.OwnBuild - b.ix.ownFP, b.ix.ownFixed = "", false - b.ix.OwnBuild = func(context.Context) (string, error) { return "", errors.New("the engine does not answer") } - b.ix.FixOwnBuild(ctx) - b.said = nil - if _, _, err := b.ix.Arrive(ctx, 7, b.merge(30, sha('b'), "tree-avahi-2", "")); !errors.Is(err, ErrOwnBuildUnknown) { - t.Fatal(err) - } - b.ix.OwnBuild, b.ix.ownFP, b.ix.ownFixed = ownBuild, "", false - b.ix.FixOwnBuild(ctx) - b.arrive(t, 7, b.merge(30, sha('b'), "tree-avahi-2", "")) // the stream delivers it again - if want := []string{"pending " + takein.PendingKey(7), "taken " + takein.PendingKey(7)}; !slices.Equal(b.events(), want) || len(b.keys()) != 0 { - t.Fatalf("said %v, pending %v: want %v and nothing pending", b.events(), b.keys(), want) + if a, _ := answer.(takein.PendingAnswer); err != nil || a.OwnBuild != "" || a.OwnBuildSince.IsZero() || len(a.Waits) != 0 { + t.Errorf("indexer_pending = %+v %v, want no wait and no known build", answer, err) } }) t.Run("T7 and T3 a merge alone, an indexer change, a merge behind it and a revert", func(t *testing.T) { @@ -371,7 +339,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) t.Errorf("indexer_pending %+v, want the refusal naming the cycle", waits) } }) - t.Run("T10 a start takes what waits in once its build is known; a gathered group stays", func(t *testing.T) { + t.Run("T10 a start takes what waits in whatever its build; a gathered group stays", func(t *testing.T) { b := newTakeInBed(t) b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) g := b.merge(30, sha('f'), "", "") @@ -390,30 +358,21 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) n.FixOwnBuild(ctx) return n } - asked := 0 - next := mk(func(context.Context) (string, error) { asked++; return "", errors.New("not yet") }) - if r := next.TakeInPendingByHand(ctx, "m.nothing", "lead@shanks", "a test"); r.Stored || !strings.Contains(r.Answer, "nothing waits") { - t.Errorf("a hand ask while the own build is unknown: %+v", r) + // The build this program runs is not known (the engine does not answer at its start): nothing waits for it, so the start + // takes the revision in all the same (A-476 b), and a hand ask names only groups. + next := mk(func(context.Context) (string, error) { return "", errors.New("not yet") }) + if r := next.TakeInPendingByHand(ctx, "m.nothing", "lead@shanks", "a test"); r.Stored || !strings.Contains(r.Answer, "names no delivery group") { + t.Errorf("a hand ask for a revision: %+v", r) } - if _, words, err := next.TakeInAtStart(ctx); err != nil || len(b.keys()) != 2 || !strings.Contains(words, "not known") { - t.Fatalf("a start with its build unknown: %q %v, pending %v: want it to answer at once and take nothing in", words, err, b.keys()) - } - // The engine answers now, and the same process does not ask it again: it learnt nothing it did not know at its start. - next.OwnBuild = func(context.Context) (string, error) { asked++; return own, nil } - if _, words, err := next.TakeInAtStart(ctx); err != nil || asked != 1 || len(b.keys()) != 2 || !strings.Contains(words, "not known") { - t.Fatalf("the window of a replacement: %q %v, asked %d, pending %v: want the build fixed at the start, never learnt later", words, err, asked, b.keys()) - } - if answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{}); err != nil || answer.(takein.PendingAnswer).OwnBuild != own || answer.(takein.PendingAnswer).OwnBuildSince.IsZero() { - t.Errorf("indexer_pending = %+v %v, want the known build and since", answer, err) - } - // The module started again fixes the build at its start and takes the revision in. - next = mk(func(context.Context) (string, error) { return own, nil }) if _, _, err := next.TakeInAtStart(ctx); err != nil { t.Fatal(err) } if !reflect.DeepEqual(b.keys(), []string{takein.PendingKey(5)}) || b.commit(t, "avahi") != sha('c') { t.Errorf("start: pending %v, avahi at %s: want the revision taken in, the group left", b.keys(), b.commit(t, "avahi")) } + if answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{}); err != nil || answer.(takein.PendingAnswer).OwnBuild != own { + t.Errorf("indexer_pending = %+v %v, want the build the original program knows", answer, err) + } }) t.Run("T8 T9 T11 a group gathered until its end, a merge behind it, one alone, an end again", func(t *testing.T) { b := newTakeInBed(t) @@ -886,25 +845,33 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) t.Fatalf("pending %v: want m.2 dropped and m.3 pending", b.keys()) } // Issue 1966 (#650), ADR 0633 §5: the newest pending merge of its repository, which no taken merge covers, is not - // dropped; the answer names the read again. abandon drops it all the same (below). + // dropped; the answer names abandon, which drops it all the same (below). if r := b.ix.DropPending(ctx, takein.PendingKey(3), "lead@shanks", "lost message", false); r.Stored || - !strings.Contains(r.Answer, TakeInPendingTool) || len(b.keys()) != 1 { + !strings.Contains(r.Answer, "abandon") || len(b.keys()) != 1 { t.Fatalf("a drop nothing covers: %+v, pending %v", r, b.keys()) } + if r := b.ix.DropPending(ctx, takein.PendingKey(3), "lead@shanks", "abandoned", true); !r.Stored || len(b.keys()) != 0 { + t.Fatalf("an abandoned drop: %+v, pending %v", r, b.keys()) + } + // A delivery group gathered is taken in by hand (A-476 b: only a group is). b.ix.Pins = newMem() b.ix.Revisions = newMemObjects(0) // A-448: the taken event names the catalog graph revision the take-in made + gm := b.merge(24, sha('f'), "tree-avahi-6", "") + gm.Group, gm.Head = "feat/h", sha('1') + gm.Members = []meshdeliveryclient.Head{{Repository: catalogueRepo, Number: 24, Head: sha('1')}, {Repository: "novox/mesh-sdk", Number: 25, Head: sha('2')}} + b.arrive(t, 5, gm) b.said = nil - if r := b.ix.TakeInPendingByHand(ctx, takein.PendingKey(3), "lead@shanks", "the named indexer can never run"); !r.Stored { + if r := b.ix.TakeInPendingByHand(ctx, takein.GroupTakeInKey("feat/h"), "lead@shanks", "the group's end never comes"); !r.Stored { t.Fatalf("hand take-in: %+v", r) } - if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('d') || len(b.said) != 1 || b.said[0].Kind != takein.TakeInTaken || b.said[0].By != "lead@shanks" { + if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('f') || len(b.said) < 1 || b.said[0].Kind != takein.TakeInTaken || b.said[0].By != "lead@shanks" { t.Errorf("hand take-in: pending %v, avahi at %s, said %+v", b.keys(), b.commit(t, "avahi"), b.said) } // §476, §493: a hand take-in's taken event carries the branch, the source revision it makes and its entries. - if len(b.said) == 1 { + if len(b.said) >= 1 { got := b.said[0] - if got.Branch != TrunkBranch || got.Source <= 0 || len(got.Entries) != 2 || got.Entries[1].Module != "avahi" || - got.Entries[1].Commit != sha('d') || got.Entries[1].Fingerprint == "" { + if got.Branch != TrunkBranch || got.Source <= 0 || len(got.Entries) != 1 || got.Entries[0].Module != "avahi" || + got.Entries[0].Commit != sha('f') || got.Entries[0].Fingerprint == "" { t.Errorf("hand take-in said branch %q source %d entries %+v: want main, a source revision and avahi at its commit", got.Branch, got.Source, got.Entries) } else if newest, any, _ := newestTrunk(ctx, b.ix.Revisions); !any || got.Revision != newest.Revision { t.Errorf("hand take-in said revision %d, want the revision it made, %d (A-448)", got.Revision, newest.Revision) diff --git a/modules/mesh-delivery-indexer/internal/index/pending_tools.go b/modules/mesh-delivery-indexer/internal/index/pending_tools.go index 366654eb..bec7be46 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_tools.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_tools.go @@ -43,15 +43,13 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T return []stdio.Tool{ {Name: PendingTool, Description: "What waits for a take-in (ADR 0633): merges the indexer holds in its pending state instead of taking them in. " + - "A revision (what revision, its key its oldest merge's m.) waits for the build of the indexer it names " + - "(waits-for): the running indexer wrote only what building and moving that indexer needs, and the new indexer takes " + - "it in at its start. A group (what group, key g.) is a delivery group whose members are gathered until the " + + "A revision (what revision, its key its oldest merge's m.) waits only while its message is unread or lost or the " + + "objects are full. A group (what group, key g.) is a delivery group whose members are gathered until the " + "forge bridge says its end; a merge in one of its members' repositories waits behind it (behind). Each names its " + "merges, since when it waits, the modules it holds back (holds), the merges whose message the forge bridge's stream no " + "longer holds (lost: they are not taken in; indexer_drop_pending drops them) and the last take-in event said of it. Empty waits: " + "every merge is taken in. The watchman raises take-in-waits past 60 minutes for a revision, 30 for a group; " + - "indexer_take_in_pending and indexer_drop_pending steer a wait. own-build says which build the indexer knows it runs and own-build-since since when; own-build-unknown, when set, says the indexer cannot learn the build it " + - "runs (since when): it fixes it once at its start, so every merge waits (a take-in.pending that says so) until the module is started again.", + "indexer_take_in_pending and indexer_drop_pending steer a wait. own-build says which build the indexer knows it runs and own-build-since since when; it only names the writer, and no merge waits for it.", Input: map[string]any{"type": "object", "properties": map[string]any{}}, Run: func(map[string]any) (any, error) { var unread []string @@ -62,7 +60,7 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T return nil, err } answer := takein.PendingAnswer{Waits: PendingWaits(entries), Unread: unread} - // Whether this indexer knows the build it runs: when it does not, every merge waits, and this says since when and why. + // The build this indexer knows it runs, and since when. if ownBuild != nil { if v, _, found, err := ownBuild.Get(context.Background(), OwnBuildKey); err == nil && found { var o OwnBuild @@ -70,8 +68,6 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T answer.OwnBuildSince = o.Since if o.Known { answer.OwnBuild = o.FP - } else { - answer.OwnBuildUnknown = o.Why } } } @@ -79,12 +75,12 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T return answer, nil }}, {Name: TakeInPendingTool, - Description: "Take a wait in by hand with the running indexer and its judge (ADR 0633 §5): the revision under its m. key, " + - "whatever build of the indexer it names, or a delivery group under its g. key with the members gathered so far. For " + - "when the named indexer can never run, or a group's end never comes. A hand act: who and why are said in the " + - "indexer's journal and on the take-in.taken event, which clears take-in-waits. indexer_pending names the keys.", + Description: "Take a delivery group in by hand with the running indexer and its judge (ADR 0633 §5): the group under its g. key " + + "with the members gathered so far, for when its end never comes. Only a group is taken in by hand: a revision is taken in " + + "by the indexer itself. A hand act: who and why are said in the indexer's journal and on the take-in.taken event, which " + + "clears take-in-waits. indexer_pending names the keys.", Input: map[string]any{"type": "object", "required": []string{"key", "why"}, "properties": map[string]any{ - "key": map[string]any{"type": "string", "description": "the wait's key, m. or g., as indexer_pending names it"}, + "key": map[string]any{"type": "string", "description": "the group's key, g., as indexer_pending names it"}, "why": map[string]any{"type": "string", "description": "why it is taken in by hand, in plain words"}}}, Run: hand(false)}, {Name: DropPendingTool, diff --git a/modules/mesh-delivery-indexer/internal/index/takein.go b/modules/mesh-delivery-indexer/internal/index/takein.go index ae21ccc0..5ea1004d 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein.go +++ b/modules/mesh-delivery-indexer/internal/index/takein.go @@ -169,11 +169,11 @@ func NewMergeAsk(repository, merge string, sequence uint64, why, by string, at t // drop one pending merge's; abandon drops one no later merge covers. func NewPendingAsk(key, why, by string, drop, abandon bool, at time.Time) (string, []byte, error) { key = strings.TrimSpace(key) - if !strings.HasPrefix(key, "m.") && !(strings.HasPrefix(key, "g.") && !drop) { + if !(strings.HasPrefix(key, "m.") && drop) && !(strings.HasPrefix(key, "g.") && !drop) { if drop { return "", nil, errors.New("name the pending merge to drop by its key, m., as indexer_pending names it") } - return "", nil, errors.New("name the wait to take in by its key, m. or g., as indexer_pending names it") + return "", nil, errors.New("name the group to take in by its key, g., as indexer_pending names it") } if strings.TrimSpace(why) == "" { return "", nil, errors.New("say why (why): a hand act on what waits is said in the journal and on its take-in event") diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index d500c088..c3c1927d 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -301,6 +301,14 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc r.tcs, r.pkgs = toolchainsUsed(toolchains), f.versions } } + if _, merged := overlay[module]; !merged && repository != "" && !trunk.SameRepository(k.cur.Repository, repository) && + maps.Equal(k.held.Packages, map[string]string(r.pkgs)) { + // A module registered for another repository is moved by this repository's commit only when it imports a package the + // commit moved (A-476a, ADR 0593 §3); any other stands as held (A-476 c), and so it is for a base of this repository's. + r = result{fp: k.held.Fp2, none: k.held.None2, tcs: k.held.Toolchains, pkgs: sdkstep.PackageVersions(k.held.Packages)} + done[module] = r + return r + } done[module] = r k.cur.Inputs2 = &kept return r @@ -310,14 +318,6 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc for _, key := range slices.Sorted(maps.Keys(all)) { k := all[key] r := of(key, nil) - if _, merged := overlay[key]; !merged && repository != "" && !trunk.SameRepository(k.cur.Repository, repository) && - maps.Equal(k.held.Packages, map[string]string(r.pkgs)) { - // A module registered for another repository is moved by this repository's commit only when it imports a package the - // commit moved (A-476a, ADR 0593 §3); any other stands as held (A-476 c). - r = result{fp: k.held.Fp2, none: k.held.None2, tcs: k.held.Toolchains, pkgs: sdkstep.PackageVersions(k.held.Packages)} - edges[key] = graphEdges{module: key, cur: k.cur, manifest: k.manifest, fp: r.fp, none: r.none, tcs: r.tcs, pkgs: r.pkgs} - continue - } edges[key] = graphEdges{module: key, cur: k.cur, manifest: k.manifest, fp: r.fp, none: r.none, tcs: r.tcs, pkgs: r.pkgs} if _, merged := overlay[key]; merged { final := overlay[key] diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index c8675341..24898914 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -843,12 +843,24 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) { } return a } + // A-476 c: mesh-tools, of another repository than mesh-sdk, holds a fingerprint a recompute would move (here a stale one); a + // commit of mesh-sdk, which moved nothing it imports, leaves it as held, and only a full recompute moves it. + stale := tools + stale.Fp2 = "src2:" + strings.Repeat("5", 64) + mods.put("mesh-tools", mustEncode(t, stale)) + tools = stale first := sdkMerge(strings.Repeat("1", 40), "tree-sdk-1") second := sdkMerge(strings.Repeat("2", 40), "tree-sdk-2") if second.Fp2 == first.Fp2 { t.Errorf("a mesh-sdk trunk merge kept avahi's fingerprint %q", first.Fp2) } - n = second + if err := ix.Recompute(context.Background()); err != nil { + t.Fatal(err) + } + if o, _ := moduleOf(t, mods, "mesh-tools"); o.Fp2 == stale.Fp2 { + t.Errorf("a full recompute left mesh-tools at %q: the check above proves nothing", o.Fp2) + } + n, _ = moduleOf(t, mods, "avahi") // moved by the recompute too: its base mesh-tools moved // Nodes that disagree keep the fingerprint already computed, with its toolchains (issue 1940, 2026-10-25 20:56: a // stale report blanked every module and the controller's take-in refused modules already built); only a new // fingerprint needs the agreement: a merge that moves the module's tree gives it none, naming the disagreement. diff --git a/modules/mesh-delivery-indexer/internal/index/unreadcall.go b/modules/mesh-delivery-indexer/internal/index/unreadcall.go index d81e49fb..7d02bd10 100644 --- a/modules/mesh-delivery-indexer/internal/index/unreadcall.go +++ b/modules/mesh-delivery-indexer/internal/index/unreadcall.go @@ -47,7 +47,7 @@ func (ix *Indexer) unreadFindings(ctx context.Context, entries []PendingEntry) { "A call whose answer is kept is not answered", fmt.Sprintf("The call %s is made again on every transport fault and on the callee's event; it has not been answered since %s. Its last fault: %s.", UnreadCallee, since.UTC().Format(time.RFC3339), e.Value.Unread), "indexer."+mcc.KindCallUnanswered) - f.Needs = TakeInPendingTool + " reads the merge again now; " + PendingTool + " shows it unread." + f.Needs = "The merge is read again on the next message heard on the forge; " + PendingTool + " shows it unread." f.Evidence = []mcseat.Evidence{{At: since.UTC(), Said: e.Value.Unread}} if err := ix.Findings.Write(ctx, f); err != nil { ix.logf("take-in: the finding %s is not written: %v", views.FindingKey(f.Scope, f.ID, f.Kind), err) diff --git a/modules/mesh-delivery-indexer/internal/index/verdicts.go b/modules/mesh-delivery-indexer/internal/index/verdicts.go index b8d0c6f0..526bb638 100644 --- a/modules/mesh-delivery-indexer/internal/index/verdicts.go +++ b/modules/mesh-delivery-indexer/internal/index/verdicts.go @@ -506,8 +506,8 @@ func (ix *Indexer) writeVerdict(ctx context.Context, digest string, v meshcatalo } // OwnBuildState is the indexer's state of the build it runs (the bucket mesh-delivery-indexer_own-build, one key, OwnBuildKey): -// written once, at the process's start by FixOwnBuild, so indexer_pending says whether this process knows its build and, when it -// does not, since when and why. +// written once, at the process's start by FixOwnBuild, so indexer_pending says which build this process knows it runs, and since +// when. The build only names the writer of what is written (A-476 b). const ( OwnBuildStateName = "own-build" OwnBuildKey = "state" @@ -526,8 +526,7 @@ type OwnBuild struct { // seat (OwnBuild; never the trunk's value of the indexer, which names a build that may be merged and not delivered), before // this process hears a merge. It is never asked again: while the node-engine replaces the indexer the seat already names the // successor, and an old process that learnt it later would take revisions in with its old judge. Where the start's ask is not -// answered the build stays unknown for the life of this process: every merge waits, loudly (a take-in.pending that says so, the -// own-build state, indexer_pending), and the module is started again to learn it. +// answered the build stays unknown for the life of this process, said in the own-build state and the journal; nothing waits for it. func (ix *Indexer) FixOwnBuild(ctx context.Context) { ix.ownMu.Lock() if ix.ownFixed || ix.OwnBuild == nil { @@ -543,7 +542,7 @@ func (ix *Indexer) FixOwnBuild(ctx context.Context) { st := OwnBuild{V: 1, Since: ix.now().UTC()} if err != nil { st.Why = err.Error() - ix.logf("judge: the build this program runs is not known (%v), so every merge waits and a manifest with a field it does not know waits and is not refused; it is learnt only at the start, so the indexer is started again", err) + ix.logf("judge: the build this program runs is not known (%v); merges are taken in all the same, and it is learnt only at the start", err) } else { st.Known, st.FP = true, fp ix.logf("judge: this program runs the indexer build %s, fixed here at its start", fp) diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index ff8aef65..6de6a6e4 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -179,8 +179,6 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a // commit is in another (issue 1900). if want := []string{"seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", - "seat:node-build-go.accept.check", // design 47 G A-231: the validation's run ask (§712) on the check queue - "seat:node-build-go.output", // design 47 G A-254: a validation's answer kept as an object "seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index e4c88a65..6c0a2a72 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -178,8 +178,6 @@ "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", - "seat:node-build-go.accept.check", - "seat:node-build-go.output", "seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions" diff --git a/modules/mesh-delivery-indexer/takein/takein.go b/modules/mesh-delivery-indexer/takein/takein.go index 8e5ba0d1..70b24218 100644 --- a/modules/mesh-delivery-indexer/takein/takein.go +++ b/modules/mesh-delivery-indexer/takein/takein.go @@ -1,5 +1,5 @@ // Package takein is the indexer's take-in contract (novox/hq ADR 0633 §1, §5): the events it publishes on its own stream -// `take-in` while a source revision waits for the indexer it names or a delivery group is gathered, their keys, and the +// `take-in` while a source revision waits (its message unread, lost or held for room) or a delivery group is gathered, their keys, and the // answer of its tool indexer_pending. The indexer writes them; the watchman and the view read them. It is the indexer's // own package: one module owns these types, so they live with it and not in mesh-sdk (the operator, 2026-11-04). package takein @@ -118,7 +118,7 @@ func checkTakeInKey(key string) error { } // TakeIn is the body of a take-in event. Kind is the event's kind and Key its key. What says what waits: a revision -// (Key its oldest merge's PendingKey, WaitsFor the indexer build it names) or a group being gathered (Key its +// (Key its oldest merge's PendingKey) or a group being gathered (Key its // GroupTakeInKey, Members every approved member with its head, Merges the members held so far). Merges are the merges // it holds, oldest first; Since is when the wait began. A `taken` event names the same once the wait ended; By and Why // name a hand act (a hand take-in, or a drop, Dropped), and are carried by nothing else. @@ -138,16 +138,9 @@ type TakeIn struct { // Lost are the keys of merges whose message the forge bridge's stream no longer holds: they are not taken in, and the // revision waits until indexer_drop_pending drops them (issue 1585). Lost []string `json:"lost,omitempty"` - // Refused is why the manifest of the indexer the revision names was refused when the revision was worked out: with - // the seat definitions the revision carries (ADR 0633 §1), the manifest validator's problems. The revision waits for - // an indexer whose manifest is accepted, and says why here, as it says what it lost. + // Refused is why the revision is refused: the cycle with a build edge its catalog graph would hold (design 47 G §49). The + // revision waits, and says why here, as it says what it lost. Refused string `json:"refused,omitempty"` - // Failed is why the revision's validation could not validate (design 47 G §713): the build seat's words, never a refusal - // and never passed; it is asked again by a later merge of the branch and by indexer_validate_again. - Failed string `json:"failed,omitempty"` - // Unknown is why the indexer holds a merge it does not take in although no indexer build is named: it cannot learn the build - // it runs itself (ADR 0633 §1), so Since is when it began not to know. A pending revision names WaitsFor or Unknown. - Unknown string `json:"own-build-unknown,omitempty"` // Branch, Source and Entries are what a take-in took in (design 47 G §476, §493, §648): the watched branch, the source // revision it makes of that branch, and one entry per module it wrote. Every taken event of one take-in carries the same; // a taken event that ends a wait without a take-in (a group's end, a drop) carries none and starts no pipeline. @@ -184,8 +177,8 @@ func (t TakeIn) check() error { if !strings.HasPrefix(t.Key, "m.") { return fmt.Errorf("delivery: a revision's take-in key is its oldest merge's m., not %q", t.Key) } - if t.Kind == TakeInPending && t.WaitsFor == "" && t.Unknown == "" && len(t.Lost) == 0 && t.Refused == "" && t.Failed == "" { - return fmt.Errorf("delivery: a pending revision %s names the indexer build it waits for, why the indexer does not know its own, the merges it lost, its refusal or why it could not validate", t.Key) + if t.Kind == TakeInPending && t.WaitsFor == "" && len(t.Lost) == 0 && t.Refused == "" { + return fmt.Errorf("delivery: a pending revision %s names the indexer build it waits for, the merges it lost or its refusal", t.Key) } case TakeInGroup: if t.Group == "" { @@ -270,13 +263,11 @@ type PendingWait struct { // PendingAnswer is indexer_pending's answer: every wait, oldest first, and the pending keys that do not read. type PendingAnswer struct { - Waits []PendingWait `json:"waits"` - Unread []string `json:"unread,omitempty"` - // OwnBuildUnknown is why the running indexer does not know its own build, and since when: while it is set, every merge waits - // (a take-in.pending with the same words) and the indexer is started again to learn it. Empty when the build is known. - OwnBuildUnknown string `json:"own-build-unknown,omitempty"` - OwnBuildSince time.Time `json:"own-build-since,omitzero"` - // OwnBuild is the build the running indexer knows it runs, fixed at its start; OwnBuildSince is since when. Empty when unknown. + Waits []PendingWait `json:"waits"` + Unread []string `json:"unread,omitempty"` + OwnBuildSince time.Time `json:"own-build-since,omitzero"` + // OwnBuild is the build the running indexer knows it runs, fixed at its start; OwnBuildSince is since when. Empty when unknown: + // it names only the writer of what the indexer writes, and nothing waits for it. OwnBuild string `json:"own-build,omitempty"` } diff --git a/modules/mesh-delivery-indexer/takein/takein_test.go b/modules/mesh-delivery-indexer/takein/takein_test.go index f69c1e49..a2376fe6 100644 --- a/modules/mesh-delivery-indexer/takein/takein_test.go +++ b/modules/mesh-delivery-indexer/takein/takein_test.go @@ -21,10 +21,7 @@ func sampleTakeIns(since time.Time) []TakeIn { Members: []meshdeliveryclient.Head{{Repository: "novox/mesh-delivery", Number: 9, Head: "ffff"}, {Repository: "novox/mesh-sdk", Number: 37, Head: "dddd"}}, Since: since}, {Kind: TakeInTaken, Key: PendingKey(12), What: TakeInRevision, Merges: m, WaitsFor: "src2:abc", Since: since, By: "lead-47@shanks", Why: "the named indexer can never run"}, - {Kind: TakeInPending, Key: PendingKey(13), What: TakeInRevision, Merges: m, Unknown: "the engine does not answer", Since: since}, // the indexer cannot learn its own build - // design 47 G §713: a revision refused by its validation, and one that could not validate. - {Kind: TakeInPending, Key: PendingKey(14), What: TakeInRevision, Merges: m, Refused: "a: names no version", Since: since}, - {Kind: TakeInPending, Key: PendingKey(15), What: TakeInRevision, Merges: m, Failed: "could not run: no build node", Since: since}, + {Kind: TakeInPending, Key: PendingKey(14), What: TakeInRevision, Merges: m, Refused: "a cycle with a build edge", Since: since}, } } From 763d6a107af5555f730ce5a78192c0dbe394f6b4 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 20:34:49 +0100 Subject: [PATCH 04/20] A-476, principle 4: indexer_take_in_pending takes a pending revision in again, and a group A revision whose message is unread or lost, or held for room, has a hand recovery; its description says nothing of an indexer build. Builds: design 47 G A-476 --- .../internal/index/pending.go | 21 +++++++++---------- .../internal/index/pending_test.go | 18 +++++++++------- .../internal/index/pending_tools.go | 10 ++++----- .../internal/index/takein.go | 4 ++-- .../internal/index/unreadcall.go | 2 +- 5 files changed, 29 insertions(+), 26 deletions(-) diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index 94b4697a..afddbe4d 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -846,12 +846,12 @@ func (ix *Indexer) holdLost(ctx context.Context, ready, lost []PendingEntry, rea } for _, g := range lost { if unread[g.Key] { - ix.logf("take-in: %s (%s#%d) is not taken in and holds its revision, its message is unread for a transport fault: %s; it is read again on the next message heard on the forge and at the start", - g.Key, g.Forge.Repository, g.Forge.Number, reasons[g.Key]) + ix.logf("take-in: %s (%s#%d) is not taken in and holds its revision, its message is unread for a transport fault: %s; it is read again on the next message heard on the forge, at the start, and by %s", + g.Key, g.Forge.Repository, g.Forge.Number, reasons[g.Key], TakeInPendingTool) continue } - ix.logf("take-in: %s (%s#%d) is not taken in and holds its revision, the forge bridge could not give its message: %s; indexer_drop_pending drops it once a later merge of %s covers it", - g.Key, g.Forge.Repository, g.Forge.Number, reasons[g.Key], g.Forge.Repository) + ix.logf("take-in: %s (%s#%d) is not taken in and holds its revision, the forge bridge could not give its message: %s; %s reads it again; indexer_drop_pending drops it once a later merge of %s covers it", + g.Key, g.Forge.Repository, g.Forge.Number, reasons[g.Key], TakeInPendingTool, g.Forge.Repository) } if !ix.sayTakeIn(ctx, t) { return nil @@ -1318,8 +1318,8 @@ func waitOf(entries []PendingEntry, what, behind string) takein.PendingWait { return pw } -// TakeInPendingByHand is the hand take-in (indexer_take_in_pending, ADR 0633 §5, A-476 b): the group gathered so far, named by -// its g. key, is taken in with this indexer and its judge, +// TakeInPendingByHand is the hand take-in (indexer_take_in_pending, ADR 0633 §5, A-476, principle 4): the pending revision (its +// m. key) or the group gathered so far (its g. key) is taken in now with this indexer and its judge, // whatever build it names, for when the group's end never comes. by and why are // said in the journal and on the take-in.taken event. What waited behind it is evaluated again after. func (ix *Indexer) TakeInPendingByHand(ctx context.Context, key, by, why string) TakenIn { @@ -1345,9 +1345,8 @@ func (ix *Indexer) TakeInPendingByHand(ctx context.Context, key, by, why string) take = held } } - default: - r.Answer = fmt.Sprintf("%s names no delivery group: only a group is taken in by hand (a revision is taken in by the indexer itself); indexer_pending names the waits", key) - return r + case len(w.Ready) > 0 && w.Ready[0].Key == key: + take = w.Ready } if len(take) == 0 { r.Answer = fmt.Sprintf("nothing waits under %s: indexer_pending names the waits", key) @@ -1425,8 +1424,8 @@ func (ix *Indexer) DropPending(ctx context.Context, key, by, why string, abandon } if !covered { r.Answer = fmt.Sprintf("%s is not dropped: no later merge of %s, pending or taken in, has %s as an ancestor, so its "+ - "changes would never be taken in (ADR 0633 §5); abandon: true drops it all the same", - key, e.Forge.Repository, e.Forge.Merge) + "changes would never be taken in (ADR 0633 §5); %s reads its message again, or abandon: true drops it all the same", + key, e.Forge.Repository, e.Forge.Merge, TakeInPendingTool) return r } } diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index 950b0fbf..07792014 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -359,10 +359,10 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) return n } // The build this program runs is not known (the engine does not answer at its start): nothing waits for it, so the start - // takes the revision in all the same (A-476 b), and a hand ask names only groups. + // takes the revision in all the same (A-476 b), and a hand ask names what waits. next := mk(func(context.Context) (string, error) { return "", errors.New("not yet") }) - if r := next.TakeInPendingByHand(ctx, "m.nothing", "lead@shanks", "a test"); r.Stored || !strings.Contains(r.Answer, "names no delivery group") { - t.Errorf("a hand ask for a revision: %+v", r) + if r := next.TakeInPendingByHand(ctx, "m.nothing", "lead@shanks", "a test"); r.Stored || !strings.Contains(r.Answer, "nothing waits") { + t.Errorf("a hand ask for what does not wait: %+v", r) } if _, _, err := next.TakeInAtStart(ctx); err != nil { t.Fatal(err) @@ -850,8 +850,12 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) !strings.Contains(r.Answer, "abandon") || len(b.keys()) != 1 { t.Fatalf("a drop nothing covers: %+v, pending %v", r, b.keys()) } - if r := b.ix.DropPending(ctx, takein.PendingKey(3), "lead@shanks", "abandoned", true); !r.Stored || len(b.keys()) != 0 { - t.Fatalf("an abandoned drop: %+v, pending %v", r, b.keys()) + // A pending revision is taken in by the tool, by its key (A-476, principle 4). + if _, _, err := NewPendingAsk(takein.PendingKey(3), "x", "lead@shanks", false, false, time.Unix(0, 0)); err != nil { + t.Fatalf("the ask to take a pending revision in: %v", err) + } + if r := b.ix.TakeInPendingByHand(ctx, takein.PendingKey(3), "lead@shanks", "its message was unread"); !r.Stored || len(b.keys()) != 0 || b.commit(t, "avahi") != sha('d') { + t.Fatalf("hand take-in of a revision: %+v, pending %v, avahi at %s", r, b.keys(), b.commit(t, "avahi")) } // A delivery group gathered is taken in by hand (A-476 b: only a group is). b.ix.Pins = newMem() @@ -870,8 +874,8 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) // §476, §493: a hand take-in's taken event carries the branch, the source revision it makes and its entries. if len(b.said) >= 1 { got := b.said[0] - if got.Branch != TrunkBranch || got.Source <= 0 || len(got.Entries) != 1 || got.Entries[0].Module != "avahi" || - got.Entries[0].Commit != sha('f') || got.Entries[0].Fingerprint == "" { + if got.Branch != TrunkBranch || got.Source <= 0 || len(got.Entries) != 2 || got.Entries[1].Module != "avahi" || + got.Entries[1].Commit != sha('f') || got.Entries[1].Fingerprint == "" { t.Errorf("hand take-in said branch %q source %d entries %+v: want main, a source revision and avahi at its commit", got.Branch, got.Source, got.Entries) } else if newest, any, _ := newestTrunk(ctx, b.ix.Revisions); !any || got.Revision != newest.Revision { t.Errorf("hand take-in said revision %d, want the revision it made, %d (A-448)", got.Revision, newest.Revision) diff --git a/modules/mesh-delivery-indexer/internal/index/pending_tools.go b/modules/mesh-delivery-indexer/internal/index/pending_tools.go index bec7be46..ba52e019 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_tools.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_tools.go @@ -75,12 +75,12 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T return answer, nil }}, {Name: TakeInPendingTool, - Description: "Take a delivery group in by hand with the running indexer and its judge (ADR 0633 §5): the group under its g. key " + - "with the members gathered so far, for when its end never comes. Only a group is taken in by hand: a revision is taken in " + - "by the indexer itself. A hand act: who and why are said in the indexer's journal and on the take-in.taken event, which " + - "clears take-in-waits. indexer_pending names the keys.", + Description: "Take in now a pending revision (its m. key), for a merge whose message is unread or lost or that is held " + + "for room, or a delivery group (its g. key) with the members gathered so far, for when its end never comes (ADR 0633 §5). " + + "A hand act: who and why are said in the indexer's journal and on the take-in.taken event, which clears take-in-waits. " + + "indexer_pending names the keys.", Input: map[string]any{"type": "object", "required": []string{"key", "why"}, "properties": map[string]any{ - "key": map[string]any{"type": "string", "description": "the group's key, g., as indexer_pending names it"}, + "key": map[string]any{"type": "string", "description": "the wait's key, m. or g., as indexer_pending names it"}, "why": map[string]any{"type": "string", "description": "why it is taken in by hand, in plain words"}}}, Run: hand(false)}, {Name: DropPendingTool, diff --git a/modules/mesh-delivery-indexer/internal/index/takein.go b/modules/mesh-delivery-indexer/internal/index/takein.go index 5ea1004d..ae21ccc0 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein.go +++ b/modules/mesh-delivery-indexer/internal/index/takein.go @@ -169,11 +169,11 @@ func NewMergeAsk(repository, merge string, sequence uint64, why, by string, at t // drop one pending merge's; abandon drops one no later merge covers. func NewPendingAsk(key, why, by string, drop, abandon bool, at time.Time) (string, []byte, error) { key = strings.TrimSpace(key) - if !(strings.HasPrefix(key, "m.") && drop) && !(strings.HasPrefix(key, "g.") && !drop) { + if !strings.HasPrefix(key, "m.") && !(strings.HasPrefix(key, "g.") && !drop) { if drop { return "", nil, errors.New("name the pending merge to drop by its key, m., as indexer_pending names it") } - return "", nil, errors.New("name the group to take in by its key, g., as indexer_pending names it") + return "", nil, errors.New("name the wait to take in by its key, m. or g., as indexer_pending names it") } if strings.TrimSpace(why) == "" { return "", nil, errors.New("say why (why): a hand act on what waits is said in the journal and on its take-in event") diff --git a/modules/mesh-delivery-indexer/internal/index/unreadcall.go b/modules/mesh-delivery-indexer/internal/index/unreadcall.go index 7d02bd10..d81e49fb 100644 --- a/modules/mesh-delivery-indexer/internal/index/unreadcall.go +++ b/modules/mesh-delivery-indexer/internal/index/unreadcall.go @@ -47,7 +47,7 @@ func (ix *Indexer) unreadFindings(ctx context.Context, entries []PendingEntry) { "A call whose answer is kept is not answered", fmt.Sprintf("The call %s is made again on every transport fault and on the callee's event; it has not been answered since %s. Its last fault: %s.", UnreadCallee, since.UTC().Format(time.RFC3339), e.Value.Unread), "indexer."+mcc.KindCallUnanswered) - f.Needs = "The merge is read again on the next message heard on the forge; " + PendingTool + " shows it unread." + f.Needs = TakeInPendingTool + " reads the merge again now; " + PendingTool + " shows it unread." f.Evidence = []mcseat.Evidence{{At: since.UTC(), Said: e.Value.Unread}} if err := ix.Findings.Write(ctx, f); err != nil { ix.logf("take-in: the finding %s is not written: %v", views.FindingKey(f.Scope, f.ID, f.Kind), err) From d006132c2a54881c20c5ac80407b047fd0c6ae05 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 20:37:32 +0100 Subject: [PATCH 05/20] A-476 (b): the watchman and the view no longer name an indexer build a revision waits for A pending revision now holds a lost or unread message or a refusal; the watchman's finding and the view's wait say so, the take-in event no longer requires WaitsFor (old values still decode), and the hand take-in comment loses the build. Builds: design 47 G A-476 (b) --- .../mesh-delivery-indexer/internal/index/pending.go | 2 +- modules/mesh-delivery-indexer/takein/takein.go | 4 ++-- .../cmd/mesh-delivery-view-tools/tiers_rows_test.go | 2 +- modules/mesh-delivery-view/internal/view/tiers.go | 2 +- .../cmd/mesh-delivery-watchman/main_test.go | 2 +- .../internal/watch/findings_test.go | 12 ++++++------ .../mesh-delivery-watchman/internal/watch/takein.go | 7 ++----- 7 files changed, 14 insertions(+), 17 deletions(-) diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index afddbe4d..fe930595 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -1320,7 +1320,7 @@ func waitOf(entries []PendingEntry, what, behind string) takein.PendingWait { // TakeInPendingByHand is the hand take-in (indexer_take_in_pending, ADR 0633 §5, A-476, principle 4): the pending revision (its // m. key) or the group gathered so far (its g. key) is taken in now with this indexer and its judge, -// whatever build it names, for when the group's end never comes. by and why are +// for a revision whose message is unread or lost or held for room, or a group whose end never comes. by and why are // said in the journal and on the take-in.taken event. What waited behind it is evaluated again after. func (ix *Indexer) TakeInPendingByHand(ctx context.Context, key, by, why string) TakenIn { r := TakenIn{Module: key} diff --git a/modules/mesh-delivery-indexer/takein/takein.go b/modules/mesh-delivery-indexer/takein/takein.go index 70b24218..351635cf 100644 --- a/modules/mesh-delivery-indexer/takein/takein.go +++ b/modules/mesh-delivery-indexer/takein/takein.go @@ -177,8 +177,8 @@ func (t TakeIn) check() error { if !strings.HasPrefix(t.Key, "m.") { return fmt.Errorf("delivery: a revision's take-in key is its oldest merge's m., not %q", t.Key) } - if t.Kind == TakeInPending && t.WaitsFor == "" && len(t.Lost) == 0 && t.Refused == "" { - return fmt.Errorf("delivery: a pending revision %s names the indexer build it waits for, the merges it lost or its refusal", t.Key) + if t.Kind == TakeInPending && len(t.Lost) == 0 && t.Refused == "" { + return fmt.Errorf("delivery: a pending revision %s names the merges it lost or its refusal", t.Key) } case TakeInGroup: if t.Group == "" { diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/tiers_rows_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/tiers_rows_test.go index 8b0057c9..8a497a16 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/tiers_rows_test.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/tiers_rows_test.go @@ -108,7 +108,7 @@ func tiersToolSaysEachModulesWait(t *testing.T) { t.Fatalf("view_tiers with waits = %+v, %v", out, err) } web = held.Tiers[1][0] - if last := web.Waits[len(web.Waits)-1]; last.Step != view.TakeInStep || !last.Since.Equal(heldSince) || !strings.Contains(last.Words, "src2:abc") { + if last := web.Waits[len(web.Waits)-1]; last.Step != view.TakeInStep || !last.Since.Equal(heldSince) || !strings.Contains(last.Words, "unread or lost") { t.Errorf("web = %+v, want its last wait on the edge take-in since %s", web, heldSince) } if lib := held.Tiers[0][1]; lib.Module != "lib" || lib.Waits[len(lib.Waits)-1].Step != view.TakeInStep || !strings.Contains(lib.Waits[len(lib.Waits)-1].Words, "group feat") { diff --git a/modules/mesh-delivery-view/internal/view/tiers.go b/modules/mesh-delivery-view/internal/view/tiers.go index 1d98db2b..721b5da9 100644 --- a/modules/mesh-delivery-view/internal/view/tiers.go +++ b/modules/mesh-delivery-view/internal/view/tiers.go @@ -219,7 +219,7 @@ func (r Reader) heldBack(out *Tiers) (map[string]TierWait, []string) { return held, nil } for _, w := range a.Waits { - words := "the indexer holds back " + w.Key + ", which waits for the indexer build " + w.WaitsFor + words := "the indexer holds back " + w.Key + " (its message is unread or lost, or it is held for room)" if w.What == takein.TakeInGroup { words = "the indexer gathers the group " + w.Group + " and its end has not come" } else if w.Behind != "" { diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go index 75f1ff81..3843658f 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go @@ -132,7 +132,7 @@ func TestRunNeedsItsCredential(t *testing.T) { // 1135/AC6 } out.findings[old.Key()] = true w.SeedFinding(old.Key(), value) - body, err := takein.EncodeTakeIn(takein.TakeIn{Kind: takein.TakeInPending, Key: "m.00000000000000000043", What: takein.TakeInRevision, Since: time.Now(), WaitsFor: "src2:x", + body, err := takein.EncodeTakeIn(takein.TakeIn{Kind: takein.TakeInPending, Key: "m.00000000000000000043", What: takein.TakeInRevision, Since: time.Now(), Lost: []string{"m.00000000000000000043"}, Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}}) if err != nil { t.Fatal(err) diff --git a/modules/mesh-delivery-watchman/internal/watch/findings_test.go b/modules/mesh-delivery-watchman/internal/watch/findings_test.go index 9e292509..896c8807 100644 --- a/modules/mesh-delivery-watchman/internal/watch/findings_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/findings_test.go @@ -136,27 +136,27 @@ func TestTheWatchmansFindings(t *testing.T) { // began ten minutes ago, and the indexer still holds it. since := c.t.Add(-10 * time.Minute) w.SeedTakeIn(takein.PendingAnswer{Waits: []takein.PendingWait{{Key: revKey, What: takein.TakeInRevision, Keys: []string{revKey}, - Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}, WaitsFor: "src2:x", Since: since}}}) + Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}, Lost: []string{"m.00000000000000000042"}, Since: since}}}) w.TakeInCaughtUp() if len(o.findings) != 0 || o.ttls[signalKey("take-in", revKey)] != TakeInRevisionBound-10*time.Minute { t.Fatalf("within its bound: findings %v, ttl %v; want none yet and the rest of the bound", o.findings, o.ttls[signalKey("take-in", revKey)]) } expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound-10*time.Minute) }, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}}, - {"ADR 0633 §1 (issue 1921): a merge held because the indexer cannot learn its own build is raised past the revision bound from its since, naming why", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + {"ADR 0633 §1 (issue 1921): (A-476 b) a revision held because its message is lost or unread is raised past the revision bound from its since, naming why", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { w.TakeInCaughtUp() body, err := takein.EncodeTakeIn(takein.TakeIn{Kind: takein.TakeInPending, Key: revKey, What: takein.TakeInRevision, Since: c.t, - Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}, Unknown: "the engine does not answer"}) + Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}, Lost: []string{revKey}}) if err != nil { t.Fatal(err) } w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), body) if o.ttls[signalKey("take-in", revKey)] != TakeInRevisionBound || len(o.findings) != 0 { - t.Fatalf("an unknown own build: ttls %v, findings %v", o.ttls, o.findings) + t.Fatalf("a lost message: ttls %v, findings %v", o.ttls, o.findings) } expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound) for _, f := range o.findings { - if !strings.Contains(f.Summary, "cannot learn the build it runs") { + if !strings.Contains(f.Summary, "lost or unread") { t.Errorf("finding %q does not say why", f.Summary) } } @@ -573,7 +573,7 @@ func takeInBody(t *testing.T, kind, key string, since time.Time) []byte { if key[0] == 'g' { body.What, body.Group = takein.TakeInGroup, "feat/take-in" } else if kind == takein.TakeInPending { - body.WaitsFor = "src2:abc" + body.Lost = []string{key} } b, err := takein.EncodeTakeIn(body) if err != nil { diff --git a/modules/mesh-delivery-watchman/internal/watch/takein.go b/modules/mesh-delivery-watchman/internal/watch/takein.go index 9120b322..e534ac6c 100644 --- a/modules/mesh-delivery-watchman/internal/watch/takein.go +++ b/modules/mesh-delivery-watchman/internal/watch/takein.go @@ -135,10 +135,7 @@ func (w *Watchman) takeIn(kind, key string, t takein.TakeIn) { w.unfind(finding) return } - bound, what := TakeInRevisionBound, "a source revision waits for the indexer build "+t.WaitsFor - if t.Unknown != "" { - what = "a source revision is held because the indexer cannot learn the build it runs (" + t.Unknown + ")" - } + bound, what := TakeInRevisionBound, "a source revision is held (a message lost or unread, no room for it, or its refusal)" if t.What == takein.TakeInGroup { bound, what = TakeInGroupBound, "a delivery group is gathered and its end has not come" } @@ -152,7 +149,7 @@ func (w *Watchman) takeIn(kind, key string, t takein.TakeIn) { Headline: "A merged change is held back by the indexer", Explanation: "The indexer holds " + what + " and has not taken it in within " + bound.String() + ", so what the merge changes does " + "not reach the machines.", - Needs: "the indexer build it waits for to run, or the take-in by hand (indexer_take_in_pending), or a drop (indexer_drop_pending)"} + Needs: "the take-in by hand (indexer_take_in_pending), or a drop (indexer_drop_pending)"} left := bound - w.now().Sub(t.Since) w.unfind(finding) w.unsignal(takeInRow, key) From ea106ea8b9bf3cdf1f1a0d0a380ccf5eff7ddef0 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 22:43:08 +0100 Subject: [PATCH 06/20] A-479: the seat mesh-catalog's verbs answer from memory, one read of the trunk view per call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tools bundle's verbs module, modules, provides and dependents read the modules and the module graph key by key from the newest trunk revision, and each key listed the whole catalog-revisions bucket again (6 s per call at 11 000 objects, 35 s with 12 callers); seat-definitions read its state once per seat. The view is read once per call (SourceValues), the seat definitions come from a watched copy, a manifest is read once by its digest. Builds: design 47 G (the seat mesh-catalog's verbs served by the indexer's tools bundle; ADR 0604 §2, ADR 0580 clause 1) --- .../mesh-delivery-indexer-tools/catalog.go | 62 +++++++++------- .../cmd/mesh-delivery-indexer-tools/main.go | 5 +- .../internal/index/catalog_calls_test.go | 72 +++++++++++++++++++ .../internal/index/catalog_revisions.go | 51 +++++++++++++ 4 files changed, 161 insertions(+), 29 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/catalog_calls_test.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go index 52b5aea2..535c8689 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go @@ -15,24 +15,17 @@ import ( // catalogRead reads the indexer's view of the trunk (its modules and module graph, from the newest catalog graph // revision) and its manifests state for the seat's verbs. -func catalogRead(modules views.Source, manifests stdio.KeptState, nodes views.Source, seatDefinitions, marks views.Source) index.CatalogRead { +func catalogRead(modules views.Source, manifests *stdio.WatchedStates, nodes views.Source, seatDefinitions, marks views.Source) index.CatalogRead { return index.CatalogRead{ SeatDefinitions: seatDefinitions, Marks: marks, Modules: func() (map[string]meshcatalog.ModulesByModuleValue, error) { - keys, err := modules.Keys() + values, err := index.SourceValues(modules) if err != nil { return nil, err } out := map[string]meshcatalog.ModulesByModuleValue{} - for _, k := range keys { - e, ok, err := modules.Get(k) - if err != nil { - return nil, fmt.Errorf("modules.%s: %w", k, err) - } - if !ok { - continue - } - if m, err := meshcatalogclient.DecodeModule(e.Value); err == nil { + for k, v := range values { + if m, err := meshcatalogclient.DecodeModule(v); err == nil { out[k] = m } } @@ -43,31 +36,25 @@ func catalogRead(modules views.Source, manifests stdio.KeptState, nodes views.So if err != nil { return "", err } - e, err := manifests.Get(key) + // A manifest never changes under its digest: read once from the state and kept. + value, ok, err := manifests.Once("manifests", key) if err != nil { return "", err } - if e == nil { + if !ok { return "", fmt.Errorf("the manifests state holds no %s", digest) } - kept, err := meshcatalogclient.DecodeManifest(e.Value) + kept, err := meshcatalogclient.DecodeManifest(value) return kept.Manifest, err }, Graph: func() (graph.Graph, error) { - keys, err := nodes.Keys() + values, err := index.SourceValues(nodes) if err != nil { return nil, err } g := graph.Graph{} - for _, k := range keys { - e, ok, err := nodes.Get(k) - if err != nil { - return nil, fmt.Errorf("graph.%s: %w", k, err) - } - if !ok { - continue - } - if n, err := graph.Decode(e.Value); err == nil { + for k, v := range values { + if n, err := graph.Decode(v); err == nil { g[k] = n } } @@ -77,7 +64,28 @@ func catalogRead(modules views.Source, manifests stdio.KeptState, nodes views.So } // catalogTools are the seat's verbs over the indexer's own states, served beside the indexer's own tools, since the indexer's manifest claims the seat (ADR 0604 §2). -func catalogTools(state func(string) views.Source) []stdio.Tool { - return index.CatalogTools(catalogRead(state("modules"), stdio.State("manifests"), state(graph.State), - keptSource(index.SeatDefinitionsState), keptSource(views.MarksState))) +// Every state a verb reads whole is answered from memory (a watched copy, or the view of the trunk read once per call), so +// a call costs no round trip per key: four node-engines ask every module at once (the verb module read the catalog-revisions +// bucket's listing once per module, and seat-definitions the state once per seat). +func catalogTools(state func(string) views.Source, watched *stdio.WatchedStates) []stdio.Tool { + return index.CatalogTools(catalogRead(state("modules"), watched, state(graph.State), + watchedSource{watched, index.SeatDefinitionsState}, keptSource(views.MarksState))) } + +// watchedSource is a state as its watched copy holds it (stdio.WatchedStates), as a Source; it offers every value in one read. +type watchedSource struct { + w *stdio.WatchedStates + state string +} + +func (s watchedSource) Keys() ([]string, error) { return s.w.Keys(s.state) } + +func (s watchedSource) Get(key string) (*views.Entry, bool, error) { + v, ok, err := s.w.Get(s.state, key) + if err != nil || !ok { + return nil, false, err + } + return &views.Entry{Value: v}, true, nil +} + +func (s watchedSource) Values() (map[string][]byte, error) { return s.w.Values(s.state) } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index 12599d11..33f9b653 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -289,7 +289,8 @@ func main() { trunk := index.TrunkStates{Objects: revisions, Under: keptSource, Kept: &index.KeptTrunk{Say: func(format string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] "+format+"\n", a...) }}} - tools = append(tools, catalogTools(trunk.State)...) + watched := stdio.NewWatchedStates() + tools = append(tools, catalogTools(trunk.State, watched)...) // The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly // the parts whose entry declares it (novox/hq ADR 0663). tools = append(tools, index.RevisionTool(revisions, os.Getenv(stdio.ServedAs)), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}), @@ -322,7 +323,7 @@ func main() { _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.FingerprintsAgainTool, Verb: "fingerprints-again"}) return err }, 10*time.Second)...) - tools = append(tools, configReadersTool(stdio.NewWatchedStates(), trunk.State("modules")), busStateTool(trunk.State), requirementsTool(keptSource), + tools = append(tools, configReadersTool(watched, trunk.State("modules")), busStateTool(trunk.State), requirementsTool(keptSource), packageUsersTool(trunk.State), seatDefinitionTool(keptSource), outOfStepTool(os.Getenv(stdio.ServedAs), keptSource)) if err := stdio.Serve("", tools); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err) diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_calls_test.go b/modules/mesh-delivery-indexer/internal/index/catalog_calls_test.go new file mode 100644 index 00000000..445ebad6 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/catalog_calls_test.go @@ -0,0 +1,72 @@ +package index + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "sync/atomic" + "testing" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/views" +) + +// listingObjects counts the listings of the bucket: each costs the whole bucket (56 ms at 11 000 objects, more through the runtime). +type listingObjects struct { + *memObjects + listings atomic.Int64 +} + +func (l *listingObjects) Names(ctx context.Context) ([]string, error) { + l.listings.Add(1) + return l.memObjects.Names(ctx) +} + +// A seat verb that answers from the whole view of the trunk (modules, provides, dependents, module) reads the catalog-revisions +// bucket's listing once per call, never once per module: the node-engines ask `module` for every module at once, and 33 asks +// in 5 minutes passed the tool runner's 30 s (2026-11-17). +func TestTrunkViewIsReadOnceForAWholeAnswer(t *testing.T) { + objects := &listingObjects{memObjects: newMemObjects(0)} + const modules = 40 + var refs []meshcatalog.EntryRef + for m := 0; m < modules; m++ { + name := fmt.Sprintf("mod%02d", m) + digest := fmt.Sprintf("%064x", m) + e := meshcatalog.CatalogEntry{Module: name, Repository: "o/r", Path: "m/" + name, Commit: "c", Fp: "src2:" + strings.Repeat("a", 64), + Manifest: map[string]any{"name": name}, ManifestDigest: "sha256:" + digest} + b, _ := json.Marshal(e) + if err := objects.Put(context.Background(), EntryObject(digest), b); err != nil { + t.Fatal(err) + } + refs = append(refs, meshcatalog.EntryRef{Module: name, Digest: digest}) + } + b, _ := json.Marshal(meshcatalog.CatalogRevision{Kind: "trunk", Revision: 1, Entries: refs}) + if err := objects.Put(context.Background(), "trunk.1", b); err != nil { + t.Fatal(err) + } + ts := TrunkStates{Objects: objects, Under: func(string) views.Source { return nil }, Kept: &KeptTrunk{}} + for _, state := range []string{"modules", "graph"} { + objects.listings.Store(0) + values, err := SourceValues(ts.State(state)) + if err != nil || len(values) != modules { + t.Fatalf("the %s of the view = %d values (%v), want %d", state, len(values), err, modules) + } + if n := objects.listings.Load(); n > 2 { + t.Errorf("a whole read of the %s listed the bucket %d times, want at most 2 (once per call, not once per module)", state, n) + } + } + if _, err := meshcatalogclient.DecodeModule(mustValue(t, ts)); err != nil { + t.Errorf("a value of the view does not decode: %v", err) + } +} + +func mustValue(t *testing.T, ts TrunkStates) []byte { + t.Helper() + v, err := SourceValues(ts.State("modules")) + if err != nil { + t.Fatal(err) + } + return v["mod00"] +} diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index ca51d308..73dfd69b 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -588,6 +588,57 @@ func (r revisionSource) Get(key string) (*views.Entry, bool, error) { return &views.Entry{Value: value, Revision: uint64(at)}, true, nil } +// Values is every value of the modules (or the module graph) of the newest trunk revision from one read of it: a seat verb +// that answers from the whole view asks which revision is newest once, never once per key (each ask lists the whole +// catalog-revisions bucket: 6 s for the 110 modules of a bucket of 11 000 objects, and 35 s with 12 callers at once). +func (r revisionSource) Values() (map[string][]byte, error) { + s, _, err := r.t.read(r.nodes) + if err != nil { + return nil, err + } + ctx := context.Background() + keys, err := s.Keys(ctx) + if err != nil { + return nil, err + } + out := make(map[string][]byte, len(keys)) + for _, k := range keys { + value, _, found, err := s.Get(ctx, k) + if err != nil { + return nil, err + } + if found { + out[k] = value + } + } + return out, nil +} + +// SourceValues is every key's value of a source: in one read where the source offers it (the view of the trunk, a +// watched copy of a state), else a get per key. +func SourceValues(src views.Source) (map[string][]byte, error) { + if all, ok := src.(interface { + Values() (map[string][]byte, error) + }); ok { + return all.Values() + } + keys, err := src.Keys() + if err != nil { + return nil, err + } + out := make(map[string][]byte, len(keys)) + for _, k := range keys { + e, ok, err := src.Get(k) + if err != nil { + return nil, fmt.Errorf("%s: %w", k, err) + } + if ok && e != nil { + out[k] = e.Value + } + } + return out, nil +} + // Page is the values in key order after a key from one read of the revision, as many as fit maxBytes and at least one // (views.Pager): a first view's page reads the revision once, never once per key. func (r revisionSource) Page(after string, maxBytes int, _ time.Duration) ([]views.KeyedEntry, string, error) { From 3a47cccbee65aedda845d190ab2c650587af0549 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 18 Nov 2026 01:15:29 +0100 Subject: [PATCH 07/20] Judge: a process's own secrets belong to the account it runs as, whatever that account (ADR 0680) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The refusals of root and the operator account as an own secret's owner and as a bus process's account end; the secrets-owner is held to the accounts an own secret's owner is; a process naming an own secret (a word equal to its path) it runs neither as the owner of nor as root is refused, container modules excepted (lead answer A-485). Restic's captured manifest is re-captured from mesh-catalog cd3b8fe8 by CAPTURED's recipe. Builds: ADR 0680 §2, §3, §4; lead answer A-485 --- modules/mesh-delivery-indexer/judge/rules.go | 120 ++++++++--- .../mesh-delivery-indexer/judge/rules_test.go | 104 ++++++++-- .../testdata/beside/CAPTURED | 2 + .../modules/restic/files/put-off-bound | 1 + .../modules/restic/module.json.captured | 190 ++++++++++++------ 5 files changed, 308 insertions(+), 109 deletions(-) create mode 100644 modules/mesh-delivery-indexer/testdata/beside/mesh-catalog/modules/restic/files/put-off-bound diff --git a/modules/mesh-delivery-indexer/judge/rules.go b/modules/mesh-delivery-indexer/judge/rules.go index 1ea632ba..01614c65 100644 --- a/modules/mesh-delivery-indexer/judge/rules.go +++ b/modules/mesh-delivery-indexer/judge/rules.go @@ -2476,24 +2476,14 @@ func invokeProblems(m catalogue.Manifest) []string { return problems } -// ownSecretOwnerProblems is what is wrong with an own secret's declared owner (novox/hq ADR 0436 §1): naming -// an account the module makes — a `user` resource of that name — never root, which a secret file owned by it -// would make readable by anything, and never the operator account's placeholder (`${machine:account}`), which -// the mesh fills per machine and a manifest cannot say safely (novox/hq ADR 0436, settled by the lead). +// ownSecretOwnerProblems is what is wrong with an own secret's declared owner (novox/hq ADR 0680 §1, §2, which +// ends ADR 0436 §1's refusal of root and the operator account here): an account the module makes — a `user` +// resource of that name — the operator account's placeholder (`${machine:account}`) or root. Any other account +// the module does not make is still refused. func ownSecretOwnerProblems(m catalogue.Manifest, name, owner string) []string { - if owner == "" { + if owner == "" || owner == "root" || owner == "${machine:account}" { return nil } - if owner == "root" { - return []string{fmt.Sprintf( - "%s names %q as the owner of its secret %q; an own secret's owner is an account the module "+ - "makes, never root (novox/hq ADR 0436 §1)", m.Module, owner, name)} - } - if owner == "${machine:account}" { - return []string{fmt.Sprintf( - "%s names %q as the owner of its secret %q; an own secret's owner is an account the module "+ - "makes, never the operator account (novox/hq ADR 0436 §1)", m.Module, owner, name)} - } for _, r := range m.Resources { if fmt.Sprint(r["type"]) == "user" && fmt.Sprint(r["name"]) == owner { return nil @@ -2501,7 +2491,7 @@ func ownSecretOwnerProblems(m catalogue.Manifest, name, owner string) []string { } return []string{fmt.Sprintf( "%s names %q as the owner of its secret %q, which it does not make: a user resource named %q "+ - "(novox/hq ADR 0436 §1)", m.Module, owner, name, owner)} + "(novox/hq ADR 0680 §2)", m.Module, owner, name, owner)} } // ownSecretPromptProblems is what is wrong with an own secret's declared prompt, in the module check's words: @@ -3218,6 +3208,9 @@ func parseManifest(raw []byte, predefined catalogue.PredefinedList) (catalogue.M problems = append(problems, ownSecretPromptProblems(m, name, own.Prompt, catalogue.IsFamily(name))...) problems = append(problems, ownSecretOwnerProblems(m, name, own.Owner)...) } + // The module's secrets-owner is held to the same accounts as an own secret's owner (novox/hq ADR 0680 §2). + problems = append(problems, ownSecretOwnerProblems(m, "(its secrets-owner)", m.SecretsOwner)...) + problems = append(problems, ownSecretReadProblems(m)...) localOf := map[string]string{} for _, to := range m.SecretRequirements() { if _, plain := m.Secrets[to]; plain { @@ -4204,25 +4197,19 @@ func passableDir(m catalogue.Manifest, id string, depth int) bool { return false } -// processAccountProblems is what is wrong with one process's bus account (novox/hq ADR 0436 §1, §2): an -// account the module makes — a `user` resource of that name — never root and never the operator account, -// which a manifest cannot say safely (a node with no account has nothing to resolve it to, novox/hq ADR -// 0436, settled by the lead). +// processAccountProblems is what is wrong with one process's bus account (novox/hq ADR 0680 §1, §2, which ends ADR +// 0436 §1's refusal of root and the operator account here): an account the module makes — a `user` resource of +// that name — the operator account's placeholder (`${machine:account}`) or root. A process that declares bus +// and names no account, or one the module does not make, is refused. func processAccountProblems(m catalogue.Manifest, processID, user string) []string { - switch { - case user == "": + switch user { + case "": return []string{fmt.Sprintf( "%s's process %s declares bus and names no account; a process account is a user resource the "+ - "module makes, never root and never the operator account (novox/hq ADR 0436 §1, §2)", + "module makes, the operator account or root (novox/hq ADR 0680 §2)", m.Module, processID)} - case user == "root": - return []string{fmt.Sprintf( - "%s's process %s runs as root; a process account is the module's own, never root (novox/hq ADR "+ - "0436 §1, §2)", m.Module, processID)} - case user == "${machine:account}": - return []string{fmt.Sprintf( - "%s's process %s runs as the operator account; a process account is the module's own, never the "+ - "operator's (novox/hq ADR 0436 §1, §2)", m.Module, processID)} + case "root", "${machine:account}": + return nil } for _, r := range m.Resources { if fmt.Sprint(r["type"]) == "user" && fmt.Sprint(r["name"]) == user { @@ -4230,8 +4217,75 @@ func processAccountProblems(m catalogue.Manifest, processID, user string) []stri } } return []string{fmt.Sprintf( - "%s's process %s runs as %s, which it does not make: a user resource named %s (novox/hq ADR 0436 §1, "+ - "§2)", m.Module, processID, user, user)} + "%s's process %s runs as %s, which it does not make: a user resource named %s (novox/hq ADR 0680 §2)", + m.Module, processID, user, user)} +} + +// ownSecretReadProblems is what is wrong with a process that names an own secret it cannot read (novox/hq ADR +// 0680 §3): a process resource whose `run` or `env` names an own secret's path must run as that secret's +// resolved owner (its `owner`, else the module's `secrets-owner`, else root) or as root. A module with a +// container is excepted: a container reads its secrets as its own account. A process with no `user` runs as +// root, as the service manager runs it (lead answer A-485 a, b: a container module is any module with a +// `container` resource). A word names a secret only when it is the secret's path (a family's: begins with the path +// up to its `*`). +func ownSecretReadProblems(m catalogue.Manifest) []string { + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) == "container" { + return nil + } + } + var problems []string + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) != "process" { + continue + } + user, _ := r["user"].(string) + if user == "" || user == "root" { + continue + } + var words []string + if run, ok := r["run"].([]any); ok { + for _, w := range run { + words = append(words, fmt.Sprint(w)) + } + } + if env, ok := r["env"].(map[string]any); ok { + for _, v := range env { + words = append(words, fmt.Sprint(v)) + } + } + for name, own := range m.OwnSecrets { + path, _, _ := strings.Cut(own.Path, "*") + if path == "" { + continue + } + named := false + for _, w := range words { + if w == own.Path || (strings.Contains(own.Path, "*") && strings.HasPrefix(w, path)) { + named = true + } + } + owner := own.Owner + if owner == "" { + owner = m.SecretsOwner + } + if named && owner != user { + problems = append(problems, fmt.Sprintf( + "%s's process %v runs as %s and names its secret %q, which belongs to %s; a process runs as "+ + "the owner of an own secret it names, or as root (novox/hq ADR 0680 §3)", + m.Module, r["id"], user, name, ownerWord(owner))) + } + } + } + sort.Strings(problems) + return problems +} + +func ownerWord(owner string) string { + if owner == "" { + return "root" + } + return owner } // busKeeps says the process resource r's bus declares `keeps` (novox/hq ADR 0626 §1): read from the resource as diff --git a/modules/mesh-delivery-indexer/judge/rules_test.go b/modules/mesh-delivery-indexer/judge/rules_test.go index f5e0c4a5..8d6a0a41 100644 --- a/modules/mesh-delivery-indexer/judge/rules_test.go +++ b/modules/mesh-delivery-indexer/judge/rules_test.go @@ -2113,19 +2113,79 @@ func caseAnOwnSecretSaysWhenAnOutsidePartyIssuesIt(t *testing.T) { } } -// 1170/AC2: an owner naming an account the module does not make, root, or the operator account's placeholder is -// refused, naming the secret and the account. +// ADR 0680 §2, §4: an own secret's owner of root or the operator account's placeholder is accepted; an owner naming an +// account the module does not make is refused, naming the secret and the account. func caseAnOwnSecretsOwnerIsRefusedWhenItIsNotAnAccountTheModuleMakes(t *testing.T) { - for name, c := range map[string]struct{ owner, says string }{ - "root": {"root", "root"}, - "the operator account": {"${machine:account}", "operator account"}, - "an account not declared": {"not-made", "not-made"}, + for name, c := range map[string]struct { + owner string + ok bool + }{ + "root": {"root", true}, + "the operator account": {"${machine:account}", true}, + "an account the module makes": {"mesh-mail-send", true}, + "an account not declared": {"not-made", false}, } { _, err := ParseManifest([]byte(`{"module":"mesh-mail","version":"1.0","own-secrets":{` + `"mail-credential-*":{"path":"/var/lib/mesh-mail/mail-credential-*.secret","issued-by":"outside","owner":"` + c.owner + `"}},"resources":[{"id":"account","type":"user","name":"mesh-mail-send"}]}`)) - if err == nil || !strings.Contains(err.Error(), "mail-credential-*") || !strings.Contains(err.Error(), c.says) { - t.Errorf("%s: %v; want a refusal naming the secret and %q", name, err, c.says) + switch { + case c.ok && err != nil: + t.Errorf("%s: refused: %v", name, err) + case !c.ok && (err == nil || !strings.Contains(err.Error(), "mail-credential-*") || !strings.Contains(err.Error(), c.owner)): + t.Errorf("%s: %v; want a refusal naming the secret and %q", name, err, c.owner) + } + } +} + +// ADR 0680 §2, §4: the module's secrets-owner is held to the same accounts as an own secret's owner. +func caseASecretsOwnerIsRefusedWhenItIsNotAnAccountTheModuleMakes(t *testing.T) { + for name, c := range map[string]struct { + owner string + ok bool + }{ + "root": {"root", true}, + "the operator account": {"${machine:account}", true}, + "an account the module makes": {"mesh-mail-send", true}, + "an account not declared": {"not-made", false}, + } { + _, err := ParseManifest([]byte(`{"module":"mesh-mail","version":"1.0","secrets-owner":"` + c.owner + + `","resources":[{"id":"account","type":"user","name":"mesh-mail-send"}]}`)) + switch { + case c.ok && err != nil: + t.Errorf("%s: refused: %v", name, err) + case !c.ok && (err == nil || !strings.Contains(err.Error(), c.owner)): + t.Errorf("%s: %v; want a refusal naming %q", name, err, c.owner) + } + } +} + +// ADR 0680 §3, §4: a process naming an own secret runs as its resolved owner (its owner, else the module's +// secrets-owner) or as root; any other account is refused; a module with a container is excepted. +func caseAProcessNamingAnOwnSecretMustBeAbleToReadIt(t *testing.T) { + const users = `{"id":"u1","type":"user","name":"alice"},{"id":"u2","type":"user","name":"bob"}` + for name, c := range map[string]struct { + secret, extra, proc string + ok bool + }{ + "the resolved owner": {`"path":"/var/lib/x/k","owner":"alice"`, ``, `"user":"alice","run":["./x","--key","/var/lib/x/k"]`, true}, + "the owner by secrets-owner": {`"path":"/var/lib/x/k"`, `"secrets-owner":"alice",`, `"user":"alice","env":{"KEY":"/var/lib/x/k"}`, true}, + "root": {`"path":"/var/lib/x/k","owner":"alice"`, ``, `"user":"root","run":["./x","/var/lib/x/k"]`, true}, + "the operator account": {`"path":"/var/lib/x/k","owner":"${machine:account}"`, ``, `"user":"${machine:account}","run":["./x","/var/lib/x/k"]`, true}, + "a secret it does not name": {`"path":"/var/lib/x/k","owner":"alice"`, ``, `"user":"bob","run":["./x"]`, true}, + "another account": {`"path":"/var/lib/x/k","owner":"alice"`, ``, `"user":"bob","run":["./x","/var/lib/x/k"]`, false}, + "another account by secrets-owner": {`"path":"/var/lib/x/k"`, `"secrets-owner":"alice",`, `"user":"bob","env":{"KEY":"/var/lib/x/k"}`, false}, + "a secret owned by root": {`"path":"/var/lib/x/k"`, ``, `"user":"bob","run":["./x","/var/lib/x/k"]`, false}, + "a longer word holding the path": {`"path":"/var/lib/x/k","owner":"alice"`, ``, `"user":"bob","env":{"KEY":"/var/lib/x/keyring"}`, true}, + "a container module": {`"path":"/var/lib/x/k","owner":"alice"`, ``, `"user":"bob","run":["./x","/var/lib/x/k"]},{"id":"c","type":"container","image":"x"`, true}, + } { + raw := `{"module":"x","version":"1.0",` + c.extra + `"own-secrets":{"k":{` + c.secret + `}},"resources":[` + users + + `,{"id":"p","type":"process","artifact":"x",` + c.proc + `}]}` + _, err := ParseManifest([]byte(raw)) + switch { + case c.ok && err != nil: + t.Errorf("%s: refused: %v", name, err) + case !c.ok && (err == nil || !strings.Contains(err.Error(), "ADR 0680 §3")): + t.Errorf("%s: %v; want the ADR 0680 §3 refusal", name, err) } } } @@ -2419,13 +2479,18 @@ func caseAPortWithNoSourceIsRefused(t *testing.T) { } } -// 1179/AC2: a bus on a process with no user, user root, or the operator account is refused naming the -// process and the account rule. +// ADR 0680 §2, §4: a bus on a process with no user, or a user the module does not make, is refused naming the +// process; root and the operator account are accepted. func caseAProcessBusAccountMustBeTheModulesOwn(t *testing.T) { - for name, c := range map[string]struct{ user, says string }{ - "no user": {"", "send"}, - "user root": {"root", "root"}, - "the operator account": {"${machine:account}", "operator"}, + for name, c := range map[string]struct { + user string + ok bool + }{ + "no user": {"", false}, + "an account not declared": {"not-made", false}, + "user root": {"root", true}, + "the operator account": {"${machine:account}", true}, + "the module's own": {"mesh-mail-send", true}, } { raw := `{"module":"mesh-mail","version":"1.0","emits":["mail.send.sent"],"state":["sends"],` + `"streams":[{"name":"mail-sends","events":["mail.send.approved"],"retention":"work-queue",` + @@ -2434,10 +2499,13 @@ func caseAProcessBusAccountMustBeTheModulesOwn(t *testing.T) { if c.user != "" { raw += `,"user":"` + c.user + `"` } - raw += `,"bus":{"emits":["mail.send.sent"]}}]}` + raw += `,"bus":{"emits":["mail.send.sent"],"queues":["mail-sends"]}}]}` _, err := ParseManifest([]byte(raw)) - if err == nil || !strings.Contains(err.Error(), "send") || !strings.Contains(err.Error(), c.says) { - t.Errorf("%s: %v; want a refusal naming the process and %q", name, err, c.says) + switch { + case c.ok && err != nil: + t.Errorf("%s: refused: %v", name, err) + case !c.ok && (err == nil || !strings.Contains(err.Error(), "send")): + t.Errorf("%s: %v; want a refusal naming the process", name, err) } } } @@ -6414,6 +6482,8 @@ func TestRulesSecrets(t *testing.T) { t.Run("AnOffersIdentityIsParsedStrictly", caseAnOffersIdentityIsParsedStrictly) t.Run("AnOwnSecretSaysWhenAnOutsidePartyIssuesIt", caseAnOwnSecretSaysWhenAnOutsidePartyIssuesIt) t.Run("AnOwnSecretsOwnerIsRefusedWhenItIsNotAnAccountTheModuleMakes", caseAnOwnSecretsOwnerIsRefusedWhenItIsNotAnAccountTheModuleMakes) + t.Run("ASecretsOwnerIsRefusedWhenItIsNotAnAccountTheModuleMakes", caseASecretsOwnerIsRefusedWhenItIsNotAnAccountTheModuleMakes) + t.Run("AProcessNamingAnOwnSecretMustBeAbleToReadIt", caseAProcessNamingAnOwnSecretMustBeAbleToReadIt) t.Run("AnOwnSecretTakenSomeOtherWayIsRefused", caseAnOwnSecretTakenSomeOtherWayIsRefused) t.Run("AnUnknownAccessReferenceIsRefusedAtParse", caseAnUnknownAccessReferenceIsRefusedAtParse) t.Run("AnUnknownPlaceholderKeyIsRefused", caseAnUnknownPlaceholderKeyIsRefused) diff --git a/modules/mesh-delivery-indexer/testdata/beside/CAPTURED b/modules/mesh-delivery-indexer/testdata/beside/CAPTURED index f4430ea3..2b692aa8 100644 --- a/modules/mesh-delivery-indexer/testdata/beside/CAPTURED +++ b/modules/mesh-delivery-indexer/testdata/beside/CAPTURED @@ -15,6 +15,8 @@ mesh-catalog b5dbfbb784507e211372a7c9640d91f7d6dadd84 modules/*/module.json, m mesh-host 233bb23bbcb1a829ce2a6164a31a5c5cc4f3d950 internal/apply/placement_guard.go mesh-catalog 2d5358b8 (milestone 47/G's AM-K1) modules/zsh/module.json alone: the login shell uses operator-channel for its seat's marked execute (design 46 approval mark §21), which the validator requires (§4) +mesh-catalog cd3b8fe8 (restic's secrets-owner, ADR 0680) modules/restic/module.json and modules/restic/files/** alone: its + processes run as the operator account, so the validator (ADR 0680 §3) needs the secrets-owner it names To move them, from mesh-delivery's modules/mesh-delivery-indexer directory, with the two repositories checked out beside mesh-delivery: diff --git a/modules/mesh-delivery-indexer/testdata/beside/mesh-catalog/modules/restic/files/put-off-bound b/modules/mesh-delivery-indexer/testdata/beside/mesh-catalog/modules/restic/files/put-off-bound new file mode 100644 index 00000000..c33d7f57 --- /dev/null +++ b/modules/mesh-delivery-indexer/testdata/beside/mesh-catalog/modules/restic/files/put-off-bound @@ -0,0 +1 @@ +${config:put-off-bound} diff --git a/modules/mesh-delivery-indexer/testdata/beside/mesh-catalog/modules/restic/module.json.captured b/modules/mesh-delivery-indexer/testdata/beside/mesh-catalog/modules/restic/module.json.captured index 18ee811d..313d3e7d 100644 --- a/modules/mesh-delivery-indexer/testdata/beside/mesh-catalog/modules/restic/module.json.captured +++ b/modules/mesh-delivery-indexer/testdata/beside/mesh-catalog/modules/restic/module.json.captured @@ -1,6 +1,9 @@ { "module": "restic", - "version": "1", + "version": "0.1", + "packages": { + "git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings": "0.1" + }, "moved": true, "claims": [ { @@ -13,13 +16,27 @@ ] } ], + "invokes": [ + "seat:mesh-module-assigner.assignments", + "seat:mesh-broker.show" + ], + "consumes": [ + "mesh-recovery.restore-started", + "mesh-recovery.restored", + "mesh-conditions.findings-asked" + ], + "emits": [ + "backed-up" + ], "tools": [ "backup_health", "backup_delete_retired", "backup_deleted" ], + "secrets-owner": "${machine:account}", "own-secrets": { - "repository": "${dir:state}/repository.secret" + "repository": "${dir:state}/repository.secret", + "broker": "${dir:state}/broker" }, "data": { "own": [ @@ -38,40 +55,48 @@ } ] }, - "config": { - "keep-daily": { - "type": "number", - "description": "How many nightly restore points are kept, a count from 1 to 9999: two weeks undo a mistake noticed within a fortnight. Read by the night, which thins nothing while it does not read (novox/hq issue 416); lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", - "minimum": 1, - "maximum": 9999, - "trusted": true, - "default": 14 - }, - "keep-weekly": { - "type": "number", - "description": "How many weekly restore points are kept behind the dailies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", - "minimum": 1, - "maximum": 9999, - "trusted": true, - "default": 8 - }, - "keep-monthly": { - "type": "number", - "description": "How many monthly restore points are kept behind the weeklies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", - "minimum": 1, - "maximum": 9999, - "trusted": true, - "default": 6 - }, - "keep-hand-taken-days": { - "type": "number", - "description": "How many days a restore point taken by hand (node-backup.now, or the mesh's before a module moves) is kept at least, a count from 1 to 9999; it is kept until a later good night of its module exists, and this many days in any case, so the next thinning never takes the one taken against a risky change (novox/hq issue 453). Lowering it lets go of the older ones taken by hand at the next thinning, and that cannot be undone.", - "minimum": 1, - "maximum": 9999, - "trusted": true, - "default": 7 - } + "config": { + "keep-daily": { + "type": "number", + "description": "How many nightly restore points are kept, a count from 1 to 9999: two weeks undo a mistake noticed within a fortnight. Read by the night, which thins nothing while it does not read (novox/hq issue 416); lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", + "minimum": 1, + "maximum": 9999, + "trusted": true, + "default": 14 }, + "keep-weekly": { + "type": "number", + "description": "How many weekly restore points are kept behind the dailies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", + "minimum": 1, + "maximum": 9999, + "trusted": true, + "default": 8 + }, + "keep-monthly": { + "type": "number", + "description": "How many monthly restore points are kept behind the weeklies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", + "minimum": 1, + "maximum": 9999, + "trusted": true, + "default": 6 + }, + "keep-hand-taken-days": { + "type": "number", + "description": "How many days a restore point taken by hand (node-backup.now, or the mesh's before a module moves) is kept at least, a count from 1 to 9999; it is kept until a later good night of its module exists, and this many days in any case, so the next thinning never takes the one taken against a risky change (novox/hq issue 453). Lowering it lets go of the older ones taken by hand at the next thinning, and that cannot be undone.", + "minimum": 1, + "maximum": 9999, + "trusted": true, + "default": 7 + }, + "put-off-bound": { + "type": "number", + "description": "minutes an item put off for a restore of the recovery module is waited for before the finding backup-waits is written and the wait ends, so the night never holds its lock without a bound; timed by a bound key of the state put-off (design 47 G sections 561, 557, A-155).", + "minimum": 1, + "maximum": 10080, + "trusted": false, + "default": 60 + } + }, "resources": [ { "id": "state", @@ -108,6 +133,13 @@ "trusted": true, "template": "files/retention.conf" }, + { + "id": "put-off-bound", + "type": "file", + "path": "${dir:state}/put-off-bound", + "mode": "0600", + "template": "files/put-off-bound" + }, { "id": "tool", "type": "package", @@ -133,7 +165,32 @@ "MESH_BACKUP_REPOSITORY": "${dir:repository}", "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret", "MESH_BACKUP_STATE": "${dir:state}", - "MESH_BACKUP_DATA": "${dir:state}/data.conf" + "MESH_BACKUP_DATA": "${dir:state}/data.conf", + "MESH_BROKER_FILE": "${dir:state}/broker", + "MESH_BACKUP_PUT_OFF_BOUND_FILE": "${dir:state}/put-off-bound" + }, + "health": { + "kind": "unit" + } + }, + { + "id": "awake", + "type": "process", + "name": "restic-awake", + "artifact": "tools", + "run": [ + "./restic-backups", + "awake" + ], + "user": "${machine:account}", + "env": { + "MESH_BACKUP_DECLARED": "${dir:state}/backups.conf", + "MESH_BACKUP_REPOSITORY": "${dir:repository}", + "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret", + "MESH_BACKUP_STATE": "${dir:state}", + "MESH_BACKUP_DATA": "${dir:state}/data.conf", + "MESH_BROKER_FILE": "${dir:state}/broker", + "MESH_BACKUP_PUT_OFF_BOUND_FILE": "${dir:state}/put-off-bound" }, "health": { "kind": "tool", @@ -143,27 +200,6 @@ "looks": 2, "grace": "60s" } - }, - { - "id": "measure", - "type": "process", - "name": "restic-measure", - "artifact": "tools", - "run": [ - "./restic-backups", - "measure" - ], - "user": "${machine:account}", - "env": { - "MESH_BACKUP_DECLARED": "${dir:state}/backups.conf", - "MESH_BACKUP_REPOSITORY": "${dir:repository}", - "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret", - "MESH_BACKUP_STATE": "${dir:state}", - "MESH_BACKUP_DATA": "${dir:state}/data.conf" - }, - "health": { - "kind": "unit" - } } ], "build": { @@ -183,7 +219,8 @@ "MESH_BACKUP_REPOSITORY": "${dir:repository}", "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret", "MESH_BACKUP_STATE": "${dir:state}", - "MESH_BACKUP_DATA": "${dir:state}/data.conf" + "MESH_BACKUP_DATA": "${dir:state}/data.conf", + "MESH_BACKUP_PUT_OFF_BOUND_FILE": "${dir:state}/put-off-bound" } }, { @@ -192,5 +229,40 @@ "from": "files" } ] - } + }, + "reads": [ + "disk-load.readings" + ], + "state": [ + "good" + ], + "requires": [ + { + "name": "put-off", + "storage": "key-value", + "template": "state", + "options": { + "key-ttl": true + } + }, + { + "name": "findings", + "storage": "key-value", + "template": "state" + }, + { + "name": "finding-events", + "storage": "stream", + "template": "events", + "options": { + "events": [ + "finding-written.*", + "finding-cleared.*", + "findings-stated.*" + ], + "max-age-days": 7, + "max-mib": 16 + } + } + ] } From 59ca36e55b99ca1dbe42642b3d901da35e14b2e8 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 18 Nov 2026 02:15:41 +0100 Subject: [PATCH 08/20] G1: tools runner_start, runner_run, runner_accept, runner_ask_again over the runner's seat verbs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each tool asks its served verb with the same arguments and answers its answer; the verb records the hand act, as for runner_rerun. Declared in the manifest. The seat has no verb release: no tool for it. Lead answer A-486 (G-22-lead-answers-2.md): tools are how a module is spoken to. Builds: design 47 G §83, §84, A-486 --- .../cmd/mesh-delivery-pipeline-tools/main.go | 1 + .../mesh-delivery-pipeline-tools/main_test.go | 5 +- .../mesh-delivery-pipeline-tools/verbtools.go | 81 +++++++++++++++++++ .../verbtools_test.go | 59 ++++++++++++++ .../mesh-delivery-pipeline/manifest_test.go | 1 + modules/mesh-delivery-pipeline/module.json | 4 + 6 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/verbtools.go create mode 100644 modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/verbtools_test.go diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go index 3a4f3e7b..f185e5c4 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go @@ -152,6 +152,7 @@ func main() { tools = append(tools, UnloggedTools(stdio.State(seat.StatePipelines), stdio.Log(seat.StateStepLog), meshhandactsclient.Recorder{Ask: stdio.Ask}, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] "+f+"\n", a...) })...) tools = append(tools, CancelTool(r.List, stdio.Ask), RerunTool(stdio.Ask)) + tools = append(tools, VerbTools(stdio.Ask)...) tools = append(tools, TestRunsTools(stdio.Ask)...) tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, time.Now)) if err := stdio.Serve("", tools); err != nil { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go index 6b708df0..29dfdcc8 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go @@ -32,13 +32,16 @@ func TestTools(t *testing.T) { names = append(names, tl.Name) } names = append(names, CancelTool(nil, nil).Name, RerunTool(nil).Name) + for _, tl := range VerbTools(nil) { + names = append(names, tl.Name) + } // RN25 (§468, §76; A-164): the health check is the bundle's runner_health. names = append(names, HealthTool(none{}, none{}, nil).Name) // §606, §549: the unlogged entries are listed and appended by hand. for _, tl := range UnloggedTools(stdio.State("pipelines"), stdio.Log("step-log"), nil, nil) { names = append(names, tl.Name) } - want := "mesh-pipeline-runner.show mesh-pipeline-runner.list mesh-pipeline-runner.stands mesh-pipeline-runner.why-carried mesh-pipeline-runner.kept runner_first_views runner_first_view_again runner_cancel runner_rerun runner_health runner_unlogged runner_unlogged_append" + want := "mesh-pipeline-runner.show mesh-pipeline-runner.list mesh-pipeline-runner.stands mesh-pipeline-runner.why-carried mesh-pipeline-runner.kept runner_first_views runner_first_view_again runner_cancel runner_rerun runner_start runner_run runner_accept runner_ask_again runner_health runner_unlogged runner_unlogged_append" if got := strings.Join(names, " "); got != want { t.Errorf("tools %q, want %q", got, want) } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/verbtools.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/verbtools.go new file mode 100644 index 00000000..4899f6aa --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/verbtools.go @@ -0,0 +1,81 @@ +package main + +import ( + "encoding/json" + "errors" + "strings" + + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// VerbTools are runner_start, runner_run, runner_accept and runner_ask_again: the runner's seat verbs start, run, accept and +// ask-again, reachable as tools (design 47 G §83, §482), as runner_rerun reaches rerun. Each asks its verb over the bus with +// the arguments it was given and answers the verb's answer; the verb records the hand act and refuses what it refuses. +func VerbTools(ask func(key string, body any) (json.RawMessage, error)) []stdio.Tool { + str := map[string]any{"type": "string"} + obj := func(required []string, props map[string]any) map[string]any { + return map[string]any{"type": "object", "properties": props, "required": required} + } + return []stdio.Tool{ + verbTool[seat.StartInput](ask, "start", seat.VerbStart, + "Start by hand the delivering pipeline of the head of a watched branch for a module, by the seat verb start: the branch, the module and why. A head whose pipeline is written already is refused by the verb, naming it.", + obj([]string{"branch", "module", "why"}, map[string]any{"branch": str, "module": str, "why": str}), + func(in seat.StartInput) string { + if in.Branch == "" || in.Module == "" { + return "start names the watched branch and the module" + } + return whyOf(in.Why) + }), + verbTool[seat.RunInput](ask, "run", seat.VerbRun, + "Start a run on request by the seat verb run: the members (module, branch, commit as the seat's Member) and optionally why.", + obj([]string{"members"}, map[string]any{"members": map[string]any{"type": "array", "items": map[string]any{"type": "object"}}, "why": str}), + func(in seat.RunInput) string { + if len(in.Members) == 0 { + return "run names its members" + } + return "" + }), + verbTool[seat.AcceptInput](ask, "accept", seat.VerbAccept, + "Accept one step of a pipeline by hand by the seat verb accept: the pipeline's key, the step, the artifact where the step is per artifact, and why.", + obj([]string{"pipeline", "step", "why"}, map[string]any{"pipeline": str, "step": str, "artifact": str, "why": str}), + func(in seat.AcceptInput) string { + if in.Pipeline == "" || in.Step == "" { + return "accept names the pipeline and the step" + } + return whyOf(in.Why) + }), + verbTool[seat.AskAgainInput](ask, "ask_again", seat.VerbAskAgain, + "Ask a declined approval step of a pipeline once more by the seat verb ask-again: the pipeline's key and why.", + obj([]string{"pipeline", "why"}, map[string]any{"pipeline": str, "why": str}), + func(in seat.AskAgainInput) string { + if in.Pipeline == "" { + return "ask-again names the pipeline" + } + return whyOf(in.Why) + }), + } +} + +func whyOf(why string) string { + if strings.TrimSpace(why) == "" { + return "a hand act names why" + } + return "" +} + +// verbTool is one tool over one seat verb: it decodes the arguments into the verb's input, refuses what the check refuses +// before asking, and answers the verb's answer. +func verbTool[T any](ask func(string, any) (json.RawMessage, error), name, verb, description string, input map[string]any, check func(T) string) stdio.Tool { + return stdio.Tool{Name: Prefix + "_" + name, Description: description, Input: input, + Run: func(args map[string]any) (any, error) { + in, err := as[T](args) + if err != nil { + return nil, err + } + if msg := check(in); msg != "" { + return nil, errors.New(msg) + } + return ask("seat:"+seat.Seat+"."+verb, in) + }} +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/verbtools_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/verbtools_test.go new file mode 100644 index 00000000..aa13afb8 --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/verbtools_test.go @@ -0,0 +1,59 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" +) + +// Each tool asks its seat verb with the arguments it was given and refuses, before asking, a call the verb would refuse. +func TestVerbTools(t *testing.T) { + for _, c := range []struct { + name, tool, key, refusal string + args map[string]any + body any + }{ + {"start", "runner_start", "seat:mesh-pipeline-runner.start", "", map[string]any{"branch": "main", "module": "i3", "why": "cancelled"}, + seat.StartInput{Branch: "main", Module: "i3", Why: "cancelled"}}, + {"start no why", "runner_start", "", "names why", map[string]any{"branch": "main", "module": "i3"}, nil}, + {"start no module", "runner_start", "", "branch and the module", map[string]any{"branch": "main", "why": "x"}, nil}, + {"run", "runner_run", "seat:mesh-pipeline-runner.run", "", map[string]any{"members": []any{map[string]any{}}, "why": "x"}, + seat.RunInput{Members: []seat.Member{{}}, Why: "x"}}, + {"run no members", "runner_run", "", "members", map[string]any{}, nil}, + {"accept", "runner_accept", "seat:mesh-pipeline-runner.accept", "", map[string]any{"pipeline": "p", "step": "lint", "artifact": "a", "why": "x"}, + seat.AcceptInput{Pipeline: "p", Step: "lint", Artifact: "a", Why: "x"}}, + {"accept no step", "runner_accept", "", "pipeline and the step", map[string]any{"pipeline": "p", "why": "x"}, nil}, + {"ask-again", "runner_ask_again", "seat:mesh-pipeline-runner.ask-again", "", map[string]any{"pipeline": "p", "why": "x"}, + seat.AskAgainInput{Pipeline: "p", Why: "x"}}, + {"ask-again no why", "runner_ask_again", "", "names why", map[string]any{"pipeline": "p"}, nil}, + } { + t.Run(c.name, func(t *testing.T) { + var key string + var body any + ask := func(k string, b any) (json.RawMessage, error) { + key, body = k, b + return json.RawMessage(`{"ok":1}`), nil + } + var run func(map[string]any) (any, error) + for _, tl := range VerbTools(ask) { + if tl.Name == c.tool { + run = tl.Run + } + } + out, err := run(c.args) + if c.refusal != "" { + if err == nil || !strings.Contains(err.Error(), c.refusal) || key != "" { + t.Fatalf("want refusal %q and no ask, got %v (asked %q)", c.refusal, err, key) + } + return + } + want, _ := json.Marshal(c.body) + got, _ := json.Marshal(body) + if err != nil || key != c.key || string(got) != string(want) || out == nil { + t.Fatalf("asked %q with %s, want %s, err %v", key, got, want, err) + } + }) + } +} diff --git a/modules/mesh-delivery-pipeline/manifest_test.go b/modules/mesh-delivery-pipeline/manifest_test.go index 8fd6045e..5f6dd00b 100644 --- a/modules/mesh-delivery-pipeline/manifest_test.go +++ b/modules/mesh-delivery-pipeline/manifest_test.go @@ -114,6 +114,7 @@ func TestManifest(t *testing.T) { {"uses operator-channel", equalSets(m.Uses, []string{"operator-channel"})}, // RN25 (§539): the runner's tools carry the prefix runner_. {"first-view tools", equalSets(m.Tools, []string{"runner_first_views", "runner_first_view_again", "runner_health", "runner_unlogged", "runner_unlogged_append", + "runner_start", "runner_run", "runner_accept", "runner_ask_again", "runner_test_runs", "runner_test_runs_drop"})}, // A-305 (ADR 0227 rule 8, ADR 0240 rule 2): the process is judged by its unit, as every process of mesh-delivery, // never by a tool of its own; runner_health stays a tool. diff --git a/modules/mesh-delivery-pipeline/module.json b/modules/mesh-delivery-pipeline/module.json index 13d86a9a..15910885 100644 --- a/modules/mesh-delivery-pipeline/module.json +++ b/modules/mesh-delivery-pipeline/module.json @@ -30,6 +30,10 @@ "runner_health", "runner_unlogged", "runner_unlogged_append", + "runner_start", + "runner_run", + "runner_accept", + "runner_ask_again", "runner_test_runs", "runner_test_runs_drop" ], From 80cfaa175fbdc4cf4a409e539e69bc8057034672 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 19 Nov 2026 01:02:43 +0100 Subject: [PATCH 09/20] Catalog answers from the newest trunk revision held in memory, and module reads one module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tools bundle's view of the trunk (KeptTrunk) is made from the half of the catalog its own follower of the seat holds (meshcatalogclient.WireCatalog on a views.Reader: the newest revision once at the start, again on the seat's event catalog-revision), once per revision, and says each read with its revision, module count and time. A read no longer lists catalog-revisions or fetches the newest revision object. The verb module reads the one module it is asked (CatalogRead.Module). The manifest consumes mesh-catalog.catalog-revision and invokes seat:mesh-catalog.revision for the follower. Builds: design 47 G §421, lead answer A-487 (03-DESIGN/01-to-be/47-milestones/G-22-lead-answers-2.md) --- .../mesh-delivery-indexer-tools/catalog.go | 12 +++ .../cmd/mesh-delivery-indexer-tools/main.go | 16 +++- .../internal/index/catalog.go | 16 +++- .../internal/index/catalog_revisions.go | 59 ++++++++---- .../internal/index/kept_trunk_test.go | 91 +++++++++++++++++++ .../internal/index/toolchains_test.go | 13 ++- .../mesh-delivery-indexer/manifest_test.go | 4 +- modules/mesh-delivery-indexer/module.json | 2 + 8 files changed, 188 insertions(+), 25 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go index 52b5aea2..d658ea70 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go @@ -38,6 +38,18 @@ func catalogRead(modules views.Source, manifests stdio.KeptState, nodes views.So } return out, nil }, + Module: func(name string) (meshcatalog.ModulesByModuleValue, bool, error) { + key, err := meshcatalogclient.ModulesKey(name) + if err != nil { + return meshcatalog.ModulesByModuleValue{}, false, nil // a name no key holds is a module the trunk holds none of + } + e, ok, err := modules.Get(key) + if err != nil || !ok { + return meshcatalog.ModulesByModuleValue{}, false, err + } + m, err := meshcatalogclient.DecodeModule(e.Value) + return m, err == nil, nil + }, Manifest: func(digest string) (string, error) { key, err := meshcatalogclient.ManifestsKey(digest) if err != nil { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index 664447b7..16cce0a8 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -17,9 +17,11 @@ import ( "time" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/busstate" "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/views" "mesh-delivery-indexer/internal/index" ) @@ -286,9 +288,19 @@ func main() { tools := Tools(asks, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState)), acts) // The modules and the module graph are the newest catalog graph revision's (milestone 47/G): revision.go. revisions := bundleObjects{busstate.ObjectsOf(index.CatalogRevisionsObjects)} - trunk := index.TrunkStates{Objects: revisions, Under: keptSource, Kept: &index.KeptTrunk{Say: func(format string, a ...any) { + // The view of the trunk is the half the bundle's follower of the seat holds (design 47 G A-487): it reads the newest + // revision once when the runtime starts and again on the seat's event catalog-revision, like every part's follower. + say := func(format string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] "+format+"\n", a...) - }}} + } + follow := &views.Reader{Module: os.Getenv(stdio.ServedAs), Logf: say, Acts: acts} + half, err := meshcatalogclient.WireCatalog(follow, stdio.Ask, index.TrunkBranch, nil) + if err != nil { + fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err) + os.Exit(1) + } + follow.Start(context.Background()) + trunk := index.TrunkStates{Objects: revisions, Under: keptSource, Kept: &index.KeptTrunk{View: func() (meshcatalogclient.Half, bool) { return half.View() }, Say: say}} tools = append(tools, catalogTools(trunk.State)...) // The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly // the parts whose entry declares it (novox/hq ADR 0663). diff --git a/modules/mesh-delivery-indexer/internal/index/catalog.go b/modules/mesh-delivery-indexer/internal/index/catalog.go index bca1a826..a6c4a5d0 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog.go @@ -25,6 +25,9 @@ import ( type CatalogRead struct { // Modules is every value of the modules state, by module. Modules func() (map[string]meshcatalog.ModulesByModuleValue, error) + // Module is the one value of the modules state of that module, and whether the state holds it: the verb module reads + // the one module it is asked, never every module (design 47 G A-487). + Module func(name string) (meshcatalog.ModulesByModuleValue, bool, error) // Manifest is the manifest kept under a digest in the manifests state, as its JSON. Manifest func(digest string) (string, error) // Graph is the module graph, by module. @@ -51,17 +54,26 @@ func (c CatalogRead) CatalogModulesAnswer() (catalogue.CatalogModules, error) { return catalogue.CatalogModules{Modules: out}, nil } +// module is one module's value: through Module when set, else out of every module's. +func (c CatalogRead) module(name string) (meshcatalog.ModulesByModuleValue, bool, error) { + if c.Module != nil { + return c.Module(name) + } + mods, err := c.Modules() + m, ok := mods[name] + return m, ok, err +} + // CatalogModuleAnswer is the verb module: one module whole, its manifest included. func (c CatalogRead) CatalogModuleAnswer(name string) (catalogue.CatalogOne, error) { name = strings.TrimSpace(name) if name == "" { return catalogue.CatalogOne{}, fmt.Errorf("name the module") } - mods, err := c.Modules() + m, ok, err := c.module(name) if err != nil { return catalogue.CatalogOne{}, fmt.Errorf("the indexer's modules cannot be read: %w", err) } - m, ok := mods[name] if !ok { return catalogue.CatalogOne{}, fmt.Errorf("the trunk holds no module %s", name) } diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index ca51d308..891486a8 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -438,15 +438,21 @@ func TrunkView(ctx context.Context, o Objects) (modules, nodes Store, at int64, // BranchView is TrunkView of a watched branch: the view read from the newest revision the bucket keeps of that branch // (design 47 G §291, A-93), empty and at 0 when it keeps none. func BranchView(ctx context.Context, o Objects, branch string) (modules, nodes Store, at int64, err error) { - view, held := newOverlay(nil), newOverlay(nil) newest, any, err := newestOf(ctx, o, branch) if err != nil || !any { - return view, held, 0, err + return newOverlay(nil), newOverlay(nil), 0, err } _, entries, err := ReadRevision(ctx, o, RevisionObject(newest)) if err != nil { return nil, nil, 0, err } + return viewOfEntries(newest.Revision, entries) +} + +// viewOfEntries is the view of the entries of one trunk revision, as BranchView reads them. +func viewOfEntries(revision int64, entries []meshcatalog.CatalogEntry) (modules, nodes Store, at int64, err error) { + view, held := newOverlay(nil), newOverlay(nil) + ctx, newest := context.Background(), meshcatalog.CatalogRevision{Revision: revision} for _, e := range entries { key, err := meshcatalogclient.ModulesKey(e.Module) if err != nil { @@ -496,10 +502,16 @@ type TrunkStates struct { Kept *KeptTrunk } -// KeptTrunk is the view of the trunk as last read, at its revision, shared by every read of a TrunkStates; Say, when set, -// says each read of a revision and how long it took. +// KeptTrunk is the view of the newest trunk revision, held in memory (design 47 G A-487): it is made from the half of the +// catalog the bundle's follower of the seat holds (meshcatalogclient's Catalog, which reads the newest revision once when +// its runtime starts and again only when it hears the seat's event catalog-revision), and a read answers from it without +// a call to the bus or a listing of the bucket. Say, when set, says each read of a revision with its number, its module +// count and how long it took. type KeptTrunk struct { + // View is the half the follower holds; false while it has none. + View func() (meshcatalogclient.Half, bool) Say func(format string, a ...any) + mu sync.Mutex at int64 mods Store @@ -518,33 +530,44 @@ func (t TrunkStates) read(nodes bool) (Store, int64, error) { return mods, at, err } -// view is the view of the newest trunk revision: the one kept when it is still the newest, else read whole and kept. +// view is the view of the trunk: the one held while the follower's half is still at its revision, else made from the +// half and held. func (t TrunkStates) view() (modules, nodes Store, at int64, err error) { - ctx := context.Background() if t.Kept == nil { - return TrunkView(ctx, t.Objects) - } - k := t.Kept - newest, any, err := newestOf(ctx, t.Objects, TrunkBranch) - if err != nil || !any { - return newOverlay(nil), newOverlay(nil), 0, err + return TrunkView(context.Background(), t.Objects) } + return t.Kept.current() +} + +// current is the view of the half the follower holds now; a newer half than the view held is made into a view once. +func (k *KeptTrunk) current() (Store, Store, int64, error) { + half, ok := k.View() k.mu.Lock() defer k.mu.Unlock() - if k.mods != nil && k.at == newest.Revision { + if !ok { + if k.mods == nil { + return newOverlay(nil), newOverlay(nil), 0, nil + } return k.mods, k.held, k.at, nil } - started := time.Now() - mods, held, at, err := TrunkView(ctx, t.Objects) + if k.mods != nil && k.at == half.Revision.Revision { + return k.mods, k.held, k.at, nil + } + began := time.Now() + entries := make([]meshcatalog.CatalogEntry, 0, len(half.Entries)) + for _, e := range half.Entries { + entries = append(entries, e) + } + mods, held, at, err := viewOfEntries(half.Revision.Revision, entries) if err != nil { if k.Say != nil { - k.Say("the view of the trunk: revision %d cannot be read after %s: %v", newest.Revision, time.Since(started).Round(time.Millisecond), err) + k.Say("the view of the trunk: revision %d cannot be made: %v", half.Revision.Revision, err) } return nil, nil, 0, err } if k.Say != nil { - n, _ := mods.Keys(ctx) - k.Say("the view of the trunk: read revision %d (was %d), %d modules, in %s", at, k.at, len(n), time.Since(started).Round(time.Millisecond)) + n, _ := mods.Keys(context.Background()) + k.Say("the view of the trunk: read revision %d (was %d), %d modules, in %s", at, k.at, len(n), time.Since(began).Round(time.Millisecond)) } k.mods, k.held, k.at = mods, held, at return mods, held, at, nil diff --git a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go new file mode 100644 index 00000000..508cddeb --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go @@ -0,0 +1,91 @@ +package index + +import ( + "context" + "errors" + "fmt" + "strings" + "sync/atomic" + "testing" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/views" +) + +// listedObjects counts the listings of its bucket. +type listedObjects struct { + *memObjects + n atomic.Int64 +} + +func (l *listedObjects) Names(ctx context.Context) ([]string, error) { + l.n.Add(1) + return l.memObjects.Names(ctx) +} + +// A-487: the view of the trunk is made from the half the follower holds, once per revision it holds; a read never lists +// the bucket or asks the bus. +func TestTheKeptTrunkIsMadeOncePerRevisionOfTheHalfHeld(t *testing.T) { + objects := &listedObjects{memObjects: newMemObjects(0)} + var half meshcatalogclient.Half + held := false + var asks int + said := []string{} + kept := &KeptTrunk{ + View: func() (meshcatalogclient.Half, bool) { asks++; return half, held }, + Say: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }, + } + modules := TrunkStates{Objects: objects, Under: func(string) views.Source { return nil }, Kept: kept}.State("modules") + if _, err := modules.Keys(); !errors.Is(err, ErrNoRevision) { + t.Fatalf("with no half held a read answered %v, want it to wait", err) + } + holds := func(revision int64, names ...string) { + half = meshcatalogclient.Half{Revision: meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk"}, Entries: map[string]meshcatalog.CatalogEntry{}} + for _, name := range names { + half.Entries[name] = meshcatalog.CatalogEntry{Module: name, Repository: catalogueRepo, None: "not built"} + } + held = true + } + holds(1, "avahi") + for i := 0; i < 50; i++ { + if _, _, err := modules.Get("avahi"); err != nil { + t.Fatal(err) + } + if _, err := modules.Keys(); err != nil { + t.Fatal(err) + } + } + if len(said) != 1 || !strings.Contains(said[0], "read revision 1 (was 0), 1 modules, in ") { + t.Errorf("100 reads said %q, want one read of revision 1 with its module count and time", said) + } + holds(2, "avahi", "ca-trust") + if e, ok, err := modules.Get("ca-trust"); err != nil || !ok || e.Revision != 2 { + t.Errorf("after the half moved ca-trust = %+v %v (%v), want it at revision 2", e, ok, err) + } + if len(said) != 2 || !strings.Contains(said[1], "read revision 2 (was 1), 2 modules, in ") { + t.Errorf("said %q, want each read with its revision, module count and time", said) + } + if lists := objects.n.Load(); lists != 0 { + t.Errorf("the reads listed the bucket %d times, want never", lists) + } +} + +// A-487: the verb module reads the one module it is asked. +func TestTheVerbModuleReadsOneModule(t *testing.T) { + c := CatalogRead{ + Modules: func() (map[string]meshcatalog.ModulesByModuleValue, error) { + return nil, errors.New("every module was read") + }, + Module: func(name string) (meshcatalog.ModulesByModuleValue, bool, error) { + return meshcatalog.ModulesByModuleValue{Manifest: "d1"}, name == "avahi", nil + }, + Manifest: func(string) (string, error) { return `{"module":"avahi"}`, nil }, + } + if one, err := c.CatalogModuleAnswer("avahi"); err != nil || one.Module.Module != "avahi" { + t.Errorf("module avahi = %+v (%v)", one, err) + } + if _, err := c.CatalogModuleAnswer("nope"); err == nil || !strings.Contains(err.Error(), "holds no module nope") { + t.Errorf("module nope = %v, want the trunk holds none", err) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index 491b153b..375db333 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -611,7 +611,18 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // // A kept view reads the revision once while it stays the newest, and its first view answers in pages of values, never // one revision read per key (about 11 s a page live, 2026-11-14). reads := 0 - keptView := TrunkStates{Objects: first.Revisions, Under: func(string) views.Source { return nil }, Kept: &KeptTrunk{Say: func(string, ...any) { reads++ }}} + keptView := TrunkStates{Objects: first.Revisions, Under: func(string) views.Source { return nil }, Kept: &KeptTrunk{Say: func(string, ...any) { reads++ }, View: func() (meshcatalogclient.Half, bool) { + newest, _, _ := newestTrunk(context.Background(), first.Revisions) + _, entries, err := ReadRevision(context.Background(), first.Revisions, RevisionObject(newest)) + if err != nil { + t.Fatal(err) + } + half := meshcatalogclient.Half{Revision: newest, Entries: map[string]meshcatalog.CatalogEntry{}} + for _, e := range entries { + half.Entries[e.Module] = e + } + return half, true + }}} if a, err := views.FirstView(testSelf, ModulesState, keptView.State(ModulesState), keptView.State(views.MarksState), "", time.Now()); err != nil || a.Revision != 2 || len(a.Values) != len(builderModules(t)) || a.Next != "" { t.Errorf("the kept first view modules answered %d values at %d, next %q (%v), want every module at revision 2 in one page", len(a.Values), a.Revision, a.Next, err) diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index cc051196..d1003273 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -165,7 +165,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // "Reads FORGE" is a consumes of the indexer's stream: the broker grants only buckets by reads (mesh-control // internal/broker/state.go:62-93). // The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*). - if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-conditions.findings-asked", "mesh-delivery-checker.verdict", "mesh-delivery-forge-bridge.forge.**", + if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-catalog.catalog-revision", "mesh-conditions.findings-asked", "mesh-delivery-checker.verdict", "mesh-delivery-forge-bridge.forge.**", "mesh-delivery.freed", "mesh-delivery.heads-replaced", "mesh-delivery.pinned", "mesh-delivery.pins-stated", "mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.run-started", "mesh-wants.version-delivered", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } @@ -178,7 +178,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths: // ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a // commit is in another (issue 1900). - if want := []string{"seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", + if want := []string{"seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-catalog.revision", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", "seat:node-build-go.accept.check", // design 47 G A-231: the validation's run ask (§712) on the check queue "seat:node-build-go.output", // design 47 G A-254: a validation's answer kept as an object "seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) { diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 6d387c43..6fe600b6 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -94,6 +94,7 @@ "mesh-wants.wait-ended", "mesh-delivery-checker.verdict", "mesh-delivery-forge-bridge.forge.**", + "mesh-catalog.catalog-revision", "mesh-delivery.source-moved", "mesh-delivery.pinned", "mesh-delivery.freed", @@ -176,6 +177,7 @@ "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", + "seat:mesh-catalog.revision", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", From efd8c7b34f399ae6b5d6f1546ec114f1d2d8fc4a Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 19 Nov 2026 01:04:33 +0100 Subject: [PATCH 10/20] Catalog half reads its own objects at the start and on the heard catalog-revision; no invoke of its own seat MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit KeptTrunk reads the newest trunk revision from catalog-revisions once at the start (a follower of the bundle's views.Reader, adopted on the seat's event catalog-revision) and re-reads once for each heard event of a newer trunk revision; reads list nothing. The invoke seat:mesh-catalog.revision and the meshcatalogclient follower are removed; the consume stays. Builds: design 47 G §421, lead answers A-487 and A-487b (03-DESIGN/01-to-be/47-milestones/G-22-lead-answers-2.md) --- .../cmd/mesh-delivery-indexer-tools/main.go | 30 ++++- .../internal/index/catalog_revisions.go | 119 +++++++++++------- .../internal/index/kept_trunk_test.go | 75 ++++++----- .../internal/index/toolchains_test.go | 13 +- .../mesh-delivery-indexer/manifest_test.go | 2 +- modules/mesh-delivery-indexer/module.json | 1 - 6 files changed, 148 insertions(+), 92 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index 16cce0a8..1c957a60 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -17,7 +17,7 @@ import ( "time" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/busstate" "git.novox.be/novox/mesh-sdk/go/process" @@ -288,19 +288,25 @@ func main() { tools := Tools(asks, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState)), acts) // The modules and the module graph are the newest catalog graph revision's (milestone 47/G): revision.go. revisions := bundleObjects{busstate.ObjectsOf(index.CatalogRevisionsObjects)} - // The view of the trunk is the half the bundle's follower of the seat holds (design 47 G A-487): it reads the newest - // revision once when the runtime starts and again on the seat's event catalog-revision, like every part's follower. + // The view of the trunk is read from the indexer's own objects once when the runtime starts and again each time the seat's + // event catalog-revision is heard (design 47 G A-487b), through the bundle's view path. say := func(format string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] "+format+"\n", a...) } + kept := &index.KeptTrunk{Say: say} + trunk := index.TrunkStates{Objects: revisions, Under: keptSource, Kept: kept} follow := &views.Reader{Module: os.Getenv(stdio.ServedAs), Logf: say, Acts: acts} - half, err := meshcatalogclient.WireCatalog(follow, stdio.Ask, index.TrunkBranch, nil) - if err != nil { + if err := follow.Adopt(trunkFollower(kept, revisions), meshcatalog.Seat, meshcatalog.EventCatalogRevision, func(ctx context.Context, data []byte) error { + var ev meshcatalog.CatalogRevisionEvent + if err := json.Unmarshal(data, &ev); err != nil { + return nil // not a revision event: offering it again changes nothing + } + return kept.Heard(ctx, revisions, ev) + }); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err) os.Exit(1) } follow.Start(context.Background()) - trunk := index.TrunkStates{Objects: revisions, Under: keptSource, Kept: &index.KeptTrunk{View: func() (meshcatalogclient.Half, bool) { return half.View() }, Say: say}} tools = append(tools, catalogTools(trunk.State)...) // The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly // the parts whose entry declares it (novox/hq ADR 0663). @@ -345,3 +351,15 @@ func main() { os.Exit(1) } } + +// trunkFollower is the bundle's view of the trunk as a follower of the reader: its first view is the one read of the +// indexer's objects the start makes, and answers no value (the picture is KeptTrunk's, not the reader's). +func trunkFollower(kept *index.KeptTrunk, revisions index.Objects) *views.Follower { + return &views.Follower{Owner: meshcatalog.Seat, State: "trunk-view", Tool: "catalog-revisions", + Ask: func(ctx context.Context, _ string) (views.Answer, error) { + if err := kept.Reread(ctx, revisions); err != nil { + return views.Answer{}, err + } + return views.Answer{V: views.Version, Owner: meshcatalog.Seat, State: "trunk-view", Created: time.Unix(1, 0).UTC(), Values: map[string]json.RawMessage{}, At: time.Now()}, nil + }} +} diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index 891486a8..79dfe40f 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -502,20 +502,63 @@ type TrunkStates struct { Kept *KeptTrunk } -// KeptTrunk is the view of the newest trunk revision, held in memory (design 47 G A-487): it is made from the half of the -// catalog the bundle's follower of the seat holds (meshcatalogclient's Catalog, which reads the newest revision once when -// its runtime starts and again only when it hears the seat's event catalog-revision), and a read answers from it without -// a call to the bus or a listing of the bucket. Say, when set, says each read of a revision with its number, its module -// count and how long it took. +// KeptTrunk is the view of the newest trunk revision, held in memory (design 47 G A-487, A-487b): it reads the newest +// revision from the indexer's own objects catalog-revisions once when the runtime starts (Reread) and again each time it +// hears the seat's event catalog-revision of a newer trunk revision (Heard); a read answers from the view held and never +// lists the bucket. Say, when set, says each read of a revision with its number, its module count and how long it took. type KeptTrunk struct { - // View is the half the follower holds; false while it has none. - View func() (meshcatalogclient.Half, bool) - Say func(format string, a ...any) + Say func(format string, a ...any) - mu sync.Mutex - at int64 - mods Store - held Store + read sync.Mutex // one re-read at a time; a read of the view never holds it + mu sync.Mutex + started bool + at int64 + mods Store + held Store +} + +// Reread reads the newest trunk revision of the bucket and holds its view; a bucket keeping none holds an empty view at 0. +func (k *KeptTrunk) Reread(ctx context.Context, o Objects) error { + k.read.Lock() + defer k.read.Unlock() + return k.reread(ctx, o) +} + +func (k *KeptTrunk) reread(ctx context.Context, o Objects) error { + began := time.Now() + mods, held, at, err := TrunkView(ctx, o) + if err != nil { + if k.Say != nil { + k.Say("the view of the trunk: the newest revision cannot be read after %s: %v", time.Since(began).Round(time.Millisecond), err) + } + return err + } + k.mu.Lock() + was := k.at + k.mods, k.held, k.at, k.started = mods, held, at, true + k.mu.Unlock() + if k.Say != nil { + n, _ := mods.Keys(ctx) + k.Say("the view of the trunk: read revision %d (was %d), %d modules, in %s", at, was, len(n), time.Since(began).Round(time.Millisecond)) + } + return nil +} + +// Heard is the seat's event catalog-revision: one of a trunk revision newer than the one held wakes one re-read of the +// objects; any other event is no change. A failed re-read is an error, so the event is offered again. +func (k *KeptTrunk) Heard(ctx context.Context, o Objects, e meshcatalog.CatalogRevisionEvent) error { + if e.Kind != "trunk" || !isTrunkBranch(e.Branch) { + return nil + } + k.read.Lock() + defer k.read.Unlock() + k.mu.Lock() + held := k.started && e.Revision <= k.at + k.mu.Unlock() + if held { + return nil + } + return k.reread(ctx, o) } // read is the view's modules and nodes, and the revision they were read at; none kept yet is ErrNoRevision. @@ -536,41 +579,31 @@ func (t TrunkStates) view() (modules, nodes Store, at int64, err error) { if t.Kept == nil { return TrunkView(context.Background(), t.Objects) } - return t.Kept.current() + return t.Kept.current(context.Background(), t.Objects) } -// current is the view of the half the follower holds now; a newer half than the view held is made into a view once. -func (k *KeptTrunk) current() (Store, Store, int64, error) { - half, ok := k.View() +// current is the view held; before the start's read it reads the newest revision once. +func (k *KeptTrunk) current(ctx context.Context, o Objects) (Store, Store, int64, error) { + k.mu.Lock() + started := k.started + k.mu.Unlock() + if !started { + k.read.Lock() + k.mu.Lock() + started = k.started + k.mu.Unlock() + var err error + if !started { + err = k.reread(ctx, o) + } + k.read.Unlock() + if err != nil { + return nil, nil, 0, err + } + } k.mu.Lock() defer k.mu.Unlock() - if !ok { - if k.mods == nil { - return newOverlay(nil), newOverlay(nil), 0, nil - } - return k.mods, k.held, k.at, nil - } - if k.mods != nil && k.at == half.Revision.Revision { - return k.mods, k.held, k.at, nil - } - began := time.Now() - entries := make([]meshcatalog.CatalogEntry, 0, len(half.Entries)) - for _, e := range half.Entries { - entries = append(entries, e) - } - mods, held, at, err := viewOfEntries(half.Revision.Revision, entries) - if err != nil { - if k.Say != nil { - k.Say("the view of the trunk: revision %d cannot be made: %v", half.Revision.Revision, err) - } - return nil, nil, 0, err - } - if k.Say != nil { - n, _ := mods.Keys(context.Background()) - k.Say("the view of the trunk: read revision %d (was %d), %d modules, in %s", at, k.at, len(n), time.Since(began).Round(time.Millisecond)) - } - k.mods, k.held, k.at = mods, held, at - return mods, held, at, nil + return k.mods, k.held, k.at, nil } // state is the source of a state by its name: modules and graph from the revision, their marks at its number, every diff --git a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go index 508cddeb..5513c537 100644 --- a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go +++ b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go @@ -2,13 +2,13 @@ package index import ( "context" + "encoding/json" "errors" "fmt" "strings" "sync/atomic" "testing" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-sdk/go/views" ) @@ -24,30 +24,31 @@ func (l *listedObjects) Names(ctx context.Context) ([]string, error) { return l.memObjects.Names(ctx) } -// A-487: the view of the trunk is made from the half the follower holds, once per revision it holds; a read never lists -// the bucket or asks the bus. -func TestTheKeptTrunkIsMadeOncePerRevisionOfTheHalfHeld(t *testing.T) { +// A-487, A-487b: the view of the trunk is read from the bucket once at the start and once for each heard event of a newer +// trunk revision; 100 reads between events list the bucket zero times. +func TestTheKeptTrunkIsReadAtTheStartAndOncePerHeardRevision(t *testing.T) { objects := &listedObjects{memObjects: newMemObjects(0)} - var half meshcatalogclient.Half - held := false - var asks int - said := []string{} - kept := &KeptTrunk{ - View: func() (meshcatalogclient.Half, bool) { asks++; return half, held }, - Say: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }, - } - modules := TrunkStates{Objects: objects, Under: func(string) views.Source { return nil }, Kept: kept}.State("modules") - if _, err := modules.Keys(); !errors.Is(err, ErrNoRevision) { - t.Fatalf("with no half held a read answered %v, want it to wait", err) - } - holds := func(revision int64, names ...string) { - half = meshcatalogclient.Half{Revision: meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk"}, Entries: map[string]meshcatalog.CatalogEntry{}} - for _, name := range names { - half.Entries[name] = meshcatalog.CatalogEntry{Module: name, Repository: catalogueRepo, None: "not built"} + keep := func(revision int64, modules ...string) { + t.Helper() + rev, entries := meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk"}, map[string][]byte{} + for _, name := range modules { + data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: name, Repository: catalogueRepo, None: "not built"}) + digest := fmt.Sprintf("%d-%s", revision, name) + entries[digest] = data + rev.Entries = append(rev.Entries, meshcatalog.EntryRef{Module: name, Digest: digest}) + } + if err := KeepRevision(context.Background(), objects, rev, entries); err != nil { + t.Fatal(err) } - held = true } - holds(1, "avahi") + said := []string{} + kept := &KeptTrunk{Say: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }} + modules := TrunkStates{Objects: objects, Under: func(string) views.Source { return nil }, Kept: kept}.State("modules") + keep(1, "avahi") + if err := kept.Reread(context.Background(), objects); err != nil { // the start's read + t.Fatal(err) + } + listed := objects.n.Load() for i := 0; i < 50; i++ { if _, _, err := modules.Get("avahi"); err != nil { t.Fatal(err) @@ -56,19 +57,35 @@ func TestTheKeptTrunkIsMadeOncePerRevisionOfTheHalfHeld(t *testing.T) { t.Fatal(err) } } - if len(said) != 1 || !strings.Contains(said[0], "read revision 1 (was 0), 1 modules, in ") { - t.Errorf("100 reads said %q, want one read of revision 1 with its module count and time", said) + if got := objects.n.Load() - listed; got != 0 { + t.Errorf("100 reads listed the bucket %d times, want none", got) + } + // a newer revision kept is no reason to read: only the event says it + keep(2, "avahi", "ca-trust") + listed = objects.n.Load() + if keys, _ := modules.Keys(); len(keys) != 1 { + t.Errorf("the view holds %v before the event, want revision 1's one module", keys) + } + for _, e := range []meshcatalog.CatalogRevisionEvent{{Kind: "candidate", Revision: 9}, {Kind: "trunk", Branch: "other", Revision: 9}, {Kind: "trunk", Revision: 1}} { + if err := kept.Heard(context.Background(), objects, e); err != nil { + t.Errorf("%+v: %v", e, err) + } + } + if got := objects.n.Load() - listed; got != 0 { + t.Errorf("events of a candidate, another branch and the revision held listed the bucket %d times, want none", got) + } + if err := kept.Heard(context.Background(), objects, meshcatalog.CatalogRevisionEvent{Kind: "trunk", Branch: "main", Revision: 2}); err != nil { + t.Fatal(err) + } + if got := objects.n.Load() - listed; got != 1 { + t.Errorf("a heard catalog-revision listed the bucket %d times, want exactly one re-read", got) } - holds(2, "avahi", "ca-trust") if e, ok, err := modules.Get("ca-trust"); err != nil || !ok || e.Revision != 2 { - t.Errorf("after the half moved ca-trust = %+v %v (%v), want it at revision 2", e, ok, err) + t.Errorf("after the event ca-trust = %+v %v (%v), want it at revision 2", e, ok, err) } if len(said) != 2 || !strings.Contains(said[1], "read revision 2 (was 1), 2 modules, in ") { t.Errorf("said %q, want each read with its revision, module count and time", said) } - if lists := objects.n.Load(); lists != 0 { - t.Errorf("the reads listed the bucket %d times, want never", lists) - } } // A-487: the verb module reads the one module it is asked. diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index 375db333..491b153b 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -611,18 +611,7 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // // A kept view reads the revision once while it stays the newest, and its first view answers in pages of values, never // one revision read per key (about 11 s a page live, 2026-11-14). reads := 0 - keptView := TrunkStates{Objects: first.Revisions, Under: func(string) views.Source { return nil }, Kept: &KeptTrunk{Say: func(string, ...any) { reads++ }, View: func() (meshcatalogclient.Half, bool) { - newest, _, _ := newestTrunk(context.Background(), first.Revisions) - _, entries, err := ReadRevision(context.Background(), first.Revisions, RevisionObject(newest)) - if err != nil { - t.Fatal(err) - } - half := meshcatalogclient.Half{Revision: newest, Entries: map[string]meshcatalog.CatalogEntry{}} - for _, e := range entries { - half.Entries[e.Module] = e - } - return half, true - }}} + keptView := TrunkStates{Objects: first.Revisions, Under: func(string) views.Source { return nil }, Kept: &KeptTrunk{Say: func(string, ...any) { reads++ }}} if a, err := views.FirstView(testSelf, ModulesState, keptView.State(ModulesState), keptView.State(views.MarksState), "", time.Now()); err != nil || a.Revision != 2 || len(a.Values) != len(builderModules(t)) || a.Next != "" { t.Errorf("the kept first view modules answered %d values at %d, next %q (%v), want every module at revision 2 in one page", len(a.Values), a.Revision, a.Next, err) diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index d1003273..93082750 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -178,7 +178,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths: // ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a // commit is in another (issue 1900). - if want := []string{"seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-catalog.revision", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", + if want := []string{"seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", "seat:node-build-go.accept.check", // design 47 G A-231: the validation's run ask (§712) on the check queue "seat:node-build-go.output", // design 47 G A-254: a validation's answer kept as an object "seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) { diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 6fe600b6..b7324498 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -177,7 +177,6 @@ "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", - "seat:mesh-catalog.revision", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", From 533a9e4ca9ec0510800032e7c93e3e3de647716e Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 19 Nov 2026 01:19:32 +0100 Subject: [PATCH 11/20] Revision, module and provides answer from the held trunk revision; a failed re-read is said as behind MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit KeptTrunk also holds the newest revision and its entries, and the manifests the entries carry by digest. The verb revision asked of the trunk's newest and the manifests the verbs module and provides read come from it, with no read of the bucket or the manifests state; a candidate, a branch and an older number still read the bucket. A failed re-read says the view is BEHIND and the next heard catalog-revision reads again. The follower's state is named catalog-half. Builds: design 47 G §421, lead answers A-487, A-487b, A-487c (03-DESIGN/01-to-be/47-milestones/G-22-lead-answers-2.md) --- .../mesh-delivery-indexer-tools/catalog.go | 22 ++------ .../cmd/mesh-delivery-indexer-tools/main.go | 8 +-- .../internal/index/catalog_branches_test.go | 10 ++-- .../internal/index/catalog_revisions.go | 55 +++++++++++++++++-- .../internal/index/kept_trunk_test.go | 44 ++++++++++++++- .../internal/index/revision_verb.go | 26 ++++++--- .../internal/index/toolchains_test.go | 8 +-- 7 files changed, 128 insertions(+), 45 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go index d658ea70..d1b03a7e 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go @@ -15,7 +15,7 @@ import ( // catalogRead reads the indexer's view of the trunk (its modules and module graph, from the newest catalog graph // revision) and its manifests state for the seat's verbs. -func catalogRead(modules views.Source, manifests stdio.KeptState, nodes views.Source, seatDefinitions, marks views.Source) index.CatalogRead { +func catalogRead(modules views.Source, manifest func(digest string) (string, error), nodes views.Source, seatDefinitions, marks views.Source) index.CatalogRead { return index.CatalogRead{ SeatDefinitions: seatDefinitions, Marks: marks, Modules: func() (map[string]meshcatalog.ModulesByModuleValue, error) { @@ -50,21 +50,7 @@ func catalogRead(modules views.Source, manifests stdio.KeptState, nodes views.So m, err := meshcatalogclient.DecodeModule(e.Value) return m, err == nil, nil }, - Manifest: func(digest string) (string, error) { - key, err := meshcatalogclient.ManifestsKey(digest) - if err != nil { - return "", err - } - e, err := manifests.Get(key) - if err != nil { - return "", err - } - if e == nil { - return "", fmt.Errorf("the manifests state holds no %s", digest) - } - kept, err := meshcatalogclient.DecodeManifest(e.Value) - return kept.Manifest, err - }, + Manifest: manifest, Graph: func() (graph.Graph, error) { keys, err := nodes.Keys() if err != nil { @@ -89,7 +75,7 @@ func catalogRead(modules views.Source, manifests stdio.KeptState, nodes views.So } // catalogTools are the seat's verbs over the indexer's own states, served beside the indexer's own tools, since the indexer's manifest claims the seat (ADR 0604 §2). -func catalogTools(state func(string) views.Source) []stdio.Tool { - return index.CatalogTools(catalogRead(state("modules"), stdio.State("manifests"), state(graph.State), +func catalogTools(state func(string) views.Source, kept *index.KeptTrunk) []stdio.Tool { + return index.CatalogTools(catalogRead(state("modules"), kept.Manifest, state(graph.State), keptSource(index.SeatDefinitionsState), keptSource(views.MarksState))) } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index 1c957a60..9f7f936e 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -307,10 +307,10 @@ func main() { os.Exit(1) } follow.Start(context.Background()) - tools = append(tools, catalogTools(trunk.State)...) + tools = append(tools, catalogTools(trunk.State, kept)...) // The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly // the parts whose entry declares it (novox/hq ADR 0663). - tools = append(tools, index.RevisionTool(revisions, os.Getenv(stdio.ServedAs)), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}), + tools = append(tools, index.RevisionTool(revisions, kept, os.Getenv(stdio.ServedAs)), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}), index.OutdatedTool(&index.Indexer{Versions: keptRead{stdio.State(index.VersionsState)}, Heard: keptRead{stdio.State(index.HeardState)}, Revisions: revisions}), index.InputsReadTool(keptRead{stdio.State(index.InputsState)}, stateobjects.ForState(index.InputsState).Objects)) tools = append(tools, index.CatalogRevisionTools(revisions, asks, time.Now, func(what, why string) error { @@ -355,11 +355,11 @@ func main() { // trunkFollower is the bundle's view of the trunk as a follower of the reader: its first view is the one read of the // indexer's objects the start makes, and answers no value (the picture is KeptTrunk's, not the reader's). func trunkFollower(kept *index.KeptTrunk, revisions index.Objects) *views.Follower { - return &views.Follower{Owner: meshcatalog.Seat, State: "trunk-view", Tool: "catalog-revisions", + return &views.Follower{Owner: meshcatalog.Seat, State: "catalog-half", Tool: "catalog-revisions", Ask: func(ctx context.Context, _ string) (views.Answer, error) { if err := kept.Reread(ctx, revisions); err != nil { return views.Answer{}, err } - return views.Answer{V: views.Version, Owner: meshcatalog.Seat, State: "trunk-view", Created: time.Unix(1, 0).UTC(), Values: map[string]json.RawMessage{}, At: time.Now()}, nil + return views.Answer{V: views.Version, Owner: meshcatalog.Seat, State: "catalog-half", Created: time.Unix(1, 0).UTC(), Values: map[string]json.RawMessage{}, At: time.Now()}, nil }} } diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go b/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go index e5f21613..dd0efd45 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go @@ -45,7 +45,7 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) { t.Errorf("said %+v, want main 1, then the escaped branch 1 and 2", said) } answer := func(branch string, revision int64) meshcatalog.RevisionAnswer { - a, err := Revision(ctx, ix.Revisions, "indexer", "reader", meshcatalog.RevisionInput{Branch: branch, Revision: revision}) + a, err := Revision(ctx, ix.Revisions, nil, "indexer", "reader", meshcatalog.RevisionInput{Branch: branch, Revision: revision}) if err != nil { t.Fatalf("revision of %q: %v", branch, err) } @@ -66,7 +66,7 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) { // A-205: `source` answers the branch's revision taken in from that source revision; a candidate's group alone answers its // newest revision, whatever its heads. bySource := func(branch string, source int64) (meshcatalog.RevisionAnswer, error) { - return Revision(ctx, ix.Revisions, "indexer", "reader", meshcatalog.RevisionInput{Branch: branch, Source: source}) + return Revision(ctx, ix.Revisions, nil, "indexer", "reader", meshcatalog.RevisionInput{Branch: branch, Source: source}) } if a, err := bySource("main", 5); err != nil || a.Revision.Revision != 1 || a.Entries[0].Commit != "c1" { t.Errorf("main's source 5 answered %+v (%v)", a.Revision, err) @@ -89,10 +89,10 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) { t.Fatal(err) } } - if a, err := Revision(ctx, ix.Revisions, "indexer", "reader", meshcatalog.RevisionInput{Group: "run-ab"}); err != nil || a.Revision.Revision != 4 || a.Revision.Heads != "h2" || a.Entries[0].Commit != "c9" { + if a, err := Revision(ctx, ix.Revisions, nil, "indexer", "reader", meshcatalog.RevisionInput{Group: "run-ab"}); err != nil || a.Revision.Revision != 4 || a.Revision.Heads != "h2" || a.Entries[0].Commit != "c9" { t.Errorf("a group alone answered %+v (%v), want its newest revision 4", a.Revision, err) } - if _, err := Revision(ctx, ix.Revisions, "indexer", "reader", meshcatalog.RevisionInput{Group: "run-none"}); err == nil { + if _, err := Revision(ctx, ix.Revisions, nil, "indexer", "reader", meshcatalog.RevisionInput{Group: "run-none"}); err == nil { t.Error("a candidate never kept was answered") } } @@ -123,7 +123,7 @@ func TestTheNodeEngineReadsTheRevisionAsTheModuleHoldingItsSeat(t *testing.T) { if err := ix.KeepBranchEntries(ctx, "main", 5, map[string]meshcatalog.CatalogEntry{"mesh-host": engine}); err != nil { t.Fatal(err) } - _, err := Revision(ctx, ix.Revisions, "indexer", c.caller, meshcatalog.RevisionInput{Branch: "main"}) + _, err := Revision(ctx, ix.Revisions, nil, "indexer", c.caller, meshcatalog.RevisionInput{Branch: "main"}) if (err == nil) != c.want { t.Errorf("caller %q: err %v, want answered %v", c.caller, err, c.want) } diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index 79dfe40f..12f15438 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -515,6 +515,11 @@ type KeptTrunk struct { at int64 mods Store held Store + rev meshcatalog.CatalogRevision + entries []meshcatalog.CatalogEntry + // manifests are the manifests the revision's entries carry whole, as JSON, by their digest as a module's value names it + // (A-487c: the verbs module and provides answer them from here, never from the manifests state). + manifests map[string]string } // Reread reads the newest trunk revision of the bucket and holds its view; a bucket keeping none holds an empty view at 0. @@ -526,16 +531,37 @@ func (k *KeptTrunk) Reread(ctx context.Context, o Objects) error { func (k *KeptTrunk) reread(ctx context.Context, o Objects) error { began := time.Now() - mods, held, at, err := TrunkView(ctx, o) - if err != nil { + fail := func(err error) error { if k.Say != nil { - k.Say("the view of the trunk: the newest revision cannot be read after %s: %v", time.Since(began).Round(time.Millisecond), err) + k.Say("the view of the trunk is BEHIND, its re-read failed after %s (the view held stays, the next heard catalog-revision reads again): %v", time.Since(began).Round(time.Millisecond), err) } return err } + newest, any, err := newestTrunk(ctx, o) + if err != nil { + return fail(err) + } + var entries []meshcatalog.CatalogEntry + mods, held, at := Store(newOverlay(nil)), Store(newOverlay(nil)), int64(0) + if any { + if _, entries, err = ReadRevision(ctx, o, RevisionObject(newest)); err != nil { + return fail(err) + } + if mods, held, at, err = viewOfEntries(newest.Revision, entries); err != nil { + return fail(err) + } + } + manifests := map[string]string{} + for _, e := range entries { + if d := meshcatalog.BareManifestDigest(e.ManifestDigest); d != "" && e.Manifest != nil { + if raw, err := json.Marshal(e.Manifest); err == nil { + manifests[d] = string(raw) + } + } + } k.mu.Lock() was := k.at - k.mods, k.held, k.at, k.started = mods, held, at, true + k.mods, k.held, k.at, k.started, k.rev, k.entries, k.manifests = mods, held, at, true, newest, entries, manifests k.mu.Unlock() if k.Say != nil { n, _ := mods.Keys(ctx) @@ -544,6 +570,27 @@ func (k *KeptTrunk) reread(ctx context.Context, o Objects) error { return nil } +// Newest is the newest trunk revision held, with a copy of its entries; false when the bucket kept none at the last read. +func (k *KeptTrunk) Newest(ctx context.Context, o Objects) (meshcatalog.CatalogRevision, []meshcatalog.CatalogEntry, bool, error) { + if _, _, _, err := k.current(ctx, o); err != nil { + return meshcatalog.CatalogRevision{}, nil, false, err + } + k.mu.Lock() + defer k.mu.Unlock() + return k.rev, slices.Clone(k.entries), k.at != 0, nil +} + +// Manifest is the manifest of that digest (as a module's value names it) the held revision's entries carry. +func (k *KeptTrunk) Manifest(digest string) (string, error) { + k.mu.Lock() + defer k.mu.Unlock() + raw, ok := k.manifests[digest] + if !ok { + return "", fmt.Errorf("the newest trunk revision's entries carry no manifest %s", digest) + } + return raw, nil +} + // Heard is the seat's event catalog-revision: one of a trunk revision newer than the one held wakes one re-read of the // objects; any other event is no change. A failed re-read is an error, so the event is offered again. func (k *KeptTrunk) Heard(ctx context.Context, o Objects, e meshcatalog.CatalogRevisionEvent) error { diff --git a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go index 5513c537..c955644c 100644 --- a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go +++ b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go @@ -16,15 +16,23 @@ import ( // listedObjects counts the listings of its bucket. type listedObjects struct { *memObjects - n atomic.Int64 + n atomic.Int64 + fail atomic.Bool } func (l *listedObjects) Names(ctx context.Context) ([]string, error) { l.n.Add(1) + if l.fail.Load() { + return nil, errors.New("the bucket does not answer") + } return l.memObjects.Names(ctx) } -// A-487, A-487b: the view of the trunk is read from the bucket once at the start and once for each heard event of a newer +func keptDigest(name string) string { + return "sha256:" + fmt.Sprintf("%064x", len(name)) +} + +// A-487, A-487b, A-487c: the view of the trunk is read from the bucket once at the start and once for each heard event of a newer // trunk revision; 100 reads between events list the bucket zero times. func TestTheKeptTrunkIsReadAtTheStartAndOncePerHeardRevision(t *testing.T) { objects := &listedObjects{memObjects: newMemObjects(0)} @@ -32,7 +40,7 @@ func TestTheKeptTrunkIsReadAtTheStartAndOncePerHeardRevision(t *testing.T) { t.Helper() rev, entries := meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk"}, map[string][]byte{} for _, name := range modules { - data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: name, Repository: catalogueRepo, None: "not built"}) + data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: name, Repository: catalogueRepo, None: "not built", ManifestDigest: keptDigest(name), Manifest: map[string]any{"module": name}}) digest := fmt.Sprintf("%d-%s", revision, name) entries[digest] = data rev.Entries = append(rev.Entries, meshcatalog.EntryRef{Module: name, Digest: digest}) @@ -83,9 +91,39 @@ func TestTheKeptTrunkIsReadAtTheStartAndOncePerHeardRevision(t *testing.T) { if e, ok, err := modules.Get("ca-trust"); err != nil || !ok || e.Revision != 2 { t.Errorf("after the event ca-trust = %+v %v (%v), want it at revision 2", e, ok, err) } + // A-487c: the manifests by digest and the verb revision asked of the trunk's newest answer from the view held. + listed = objects.n.Load() + if raw, err := kept.Manifest(meshcatalog.BareManifestDigest(keptDigest("ca-trust"))); err != nil || raw != `{"module":"ca-trust"}` { + t.Errorf("the held manifest = %q (%v), want the entry's manifest", raw, err) + } + if a, err := Revision(context.Background(), objects, kept, "avahi", "avahi", meshcatalog.RevisionInput{}); err != nil || a.Revision.Revision != 2 || len(a.Entries) != 2 { + t.Errorf("revision of the newest answered %+v (%v), want revision 2 whole", a, err) + } + if got := objects.n.Load() - listed; got != 0 { + t.Errorf("the manifest and the verb revision listed the bucket %d times, want none", got) + } if len(said) != 2 || !strings.Contains(said[1], "read revision 2 (was 1), 2 modules, in ") { t.Errorf("said %q, want each read with its revision, module count and time", said) } + // A-487b: a re-read that fails is said loudly, the view held stays, and the next heard event reads again. + keep(3, "avahi", "ca-trust", "dbus") + objects.fail.Store(true) + if err := kept.Heard(context.Background(), objects, meshcatalog.CatalogRevisionEvent{Kind: "trunk", Revision: 3}); err == nil { + t.Error("a re-read that failed was taken") + } + if last := said[len(said)-1]; !strings.Contains(last, "BEHIND") { + t.Errorf("a failed re-read said %q, want it said loudly", last) + } + if keys, _ := modules.Keys(); len(keys) != 2 { + t.Errorf("the view held after a failed re-read has %v, want revision 2's two modules", keys) + } + objects.fail.Store(false) + if err := kept.Heard(context.Background(), objects, meshcatalog.CatalogRevisionEvent{Kind: "trunk", Revision: 3}); err != nil { + t.Fatal(err) + } + if keys, _ := modules.Keys(); len(keys) != 3 { + t.Errorf("the next heard event left the view at %v, want revision 3's three modules", keys) + } } // A-487: the verb module reads the one module it is asked. diff --git a/modules/mesh-delivery-indexer/internal/index/revision_verb.go b/modules/mesh-delivery-indexer/internal/index/revision_verb.go index 13da59fd..c3e4a589 100644 --- a/modules/mesh-delivery-indexer/internal/index/revision_verb.go +++ b/modules/mesh-delivery-indexer/internal/index/revision_verb.go @@ -65,9 +65,20 @@ const ToolRunner = "node-tools" // answered only when the newest trunk revision's entry of its module declares the verb in invokes, and the holder is // refused too. With no trunk revision kept yet the answer is `none-yet` alone (design 47 G §530), never an empty // catalog and never an error. -func Revision(ctx context.Context, o Objects, holder, caller string, in meshcatalog.RevisionInput) (meshcatalog.RevisionAnswer, error) { +// +// kept, when not nil, is the trunk's newest revision held in memory: the trunk's newest answers from it with no read of +// the bucket (A-487c); a candidate, a branch and an older number read the bucket. +func Revision(ctx context.Context, o Objects, kept *KeptTrunk, holder, caller string, in meshcatalog.RevisionInput) (meshcatalog.RevisionAnswer, error) { var out meshcatalog.RevisionAnswer - newest, ok, err := newestTrunk(ctx, o) + var newest meshcatalog.CatalogRevision + var ok bool + var trunkEntries []meshcatalog.CatalogEntry + var err error + if kept != nil { + newest, trunkEntries, ok, err = kept.Newest(ctx, o) + } else { + newest, ok, err = newestTrunk(ctx, o) + } if err != nil { return out, fmt.Errorf("the catalog graph revisions cannot be read: %w", err) } @@ -75,9 +86,10 @@ func Revision(ctx context.Context, o Objects, holder, caller string, in meshcata // An answer is told by its fields (design 47 G §530): no trunk revision kept yet is `none-yet`, no error. return meshcatalog.RevisionAnswer{NoneYet: true}, nil } - _, trunkEntries, err := ReadRevision(ctx, o, RevisionObject(newest)) - if err != nil { - return out, err + if kept == nil { + if _, trunkEntries, err = ReadRevision(ctx, o, RevisionObject(newest)); err != nil { + return out, err + } } caller = CallerModule(trunkEntries, caller) if err := views.Refusal(meshcatalog.Seat+"."+meshcatalog.VerbRevision, RevisionCallers(trunkEntries, holder), caller); err != nil { @@ -152,13 +164,13 @@ func Revision(ctx context.Context, o Objects, holder, caller string, in meshcata // RevisionTool is the verb revision as the tool mesh-catalog.revision of the indexer's own runtime (ADR 0663 §1): the // tool runner describes it with the seat's definition, so it carries no description or input of its own. -func RevisionTool(o Objects, holder string) stdio.Tool { +func RevisionTool(o Objects, kept *KeptTrunk, holder string) stdio.Tool { return stdio.Tool{Name: meshcatalog.Seat + "." + meshcatalog.VerbRevision, Run: func(args map[string]any) (any, error) { var in meshcatalog.RevisionInput if raw, err := json.Marshal(args); err == nil { _ = json.Unmarshal(raw, &in) } - return Revision(context.Background(), o, holder, stdio.CallerOf(args), in) + return Revision(context.Background(), o, kept, holder, stdio.CallerOf(args), in) }} } diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index 491b153b..21804c57 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -645,22 +645,22 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // t.Fatal(err) } } - if got, err := Revision(context.Background(), verbObjects, testSelf, "n1.node-tools", meshcatalog.RevisionInput{}); err != nil || !got.NoneYet || got.Revision != nil || len(got.Entries) != 0 { + if got, err := Revision(context.Background(), verbObjects, nil, testSelf, "n1.node-tools", meshcatalog.RevisionInput{}); err != nil || !got.NoneYet || got.Revision != nil || len(got.Entries) != 0 { t.Errorf("revision with none kept answered %+v, %v, want none-yet alone with no error (design 47 G §530)", got, err) } keep(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, map[string]bool{"node-tools": true, testSelf: true}) keep(meshcatalog.CatalogRevision{Revision: 4, Kind: "candidate", Group: "feat/x", Heads: "0123456789ab"}, map[string]bool{"node-tools": true, "mesh-grants": true}) for caller, answered := range map[string]bool{"n1.node-tools": true, "n1.avahi": true, "n1.mesh-grants": true, "n1." + testSelf: true, "person.someone": false, "unproven": false} { - got, err := Revision(context.Background(), verbObjects, testSelf, caller, meshcatalog.RevisionInput{}) + got, err := Revision(context.Background(), verbObjects, nil, testSelf, caller, meshcatalog.RevisionInput{}) if (err == nil) != answered || (answered && (got.Revision.Revision != 3 || len(got.Entries) != 4 || got.Next != "")) { t.Errorf("revision to %s answered %+v (%v), want answered %v with trunk revision 3 whole", caller, got, err, answered) } } - if got, err := Revision(context.Background(), verbObjects, testSelf, "n1.node-tools", meshcatalog.RevisionInput{Group: "feat/x", Heads: "0123456789ab", After: "mesh-grants"}); err != nil || + if got, err := Revision(context.Background(), verbObjects, nil, testSelf, "n1.node-tools", meshcatalog.RevisionInput{Group: "feat/x", Heads: "0123456789ab", After: "mesh-grants"}); err != nil || got.Revision.Kind != "candidate" || len(got.Entries) != 1 || got.Entries[0].Module != "node-tools" { t.Errorf("a candidate's revision after mesh-grants answered %+v (%v), want its one entry node-tools", got, err) } - if _, err := Revision(context.Background(), verbObjects, testSelf, "n1.node-tools", meshcatalog.RevisionInput{Revision: 9}); err == nil { + if _, err := Revision(context.Background(), verbObjects, nil, testSelf, "n1.node-tools", meshcatalog.RevisionInput{Revision: 9}); err == nil { t.Error("a trunk revision not kept was answered") } // ADR 0614 §2, §3: the graph names what each module invokes and serves, and leaves out, over the whole graph, the From 1dd16bc783e7156ab8ff820e620fdc5507aab1e5 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 19 Nov 2026 12:10:51 +0100 Subject: [PATCH 12/20] The kept trunk reads the newest revision again when the one named is removed under the read The pin judge removes a trunk revision, its list and its entries, as soon as a newer one is kept; take-ins keep one every few seconds, so the read of the newest revision failed with no such object and the first view of the catalog half never came. Builds: design 47 G A-487, A-487b Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- .../internal/index/catalog_revisions.go | 26 ++++++++--- .../internal/index/kept_trunk_test.go | 43 +++++++++++++++++++ 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index 7a104b44..68e0fce5 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -522,6 +522,9 @@ type KeptTrunk struct { manifests map[string]string } +// rereadTries is how many times a read names the newest revision again when the one it named was removed under it. +const rereadTries = 5 + // Reread reads the newest trunk revision of the bucket and holds its view; a bucket keeping none holds an empty view at 0. func (k *KeptTrunk) Reread(ctx context.Context, o Objects) error { k.read.Lock() @@ -537,16 +540,29 @@ func (k *KeptTrunk) reread(ctx context.Context, o Objects) error { } return err } - newest, any, err := newestTrunk(ctx, o) + // The pin judge removes a trunk revision, its list and the entries only it names, as soon as a newer one is kept, and + // revisions are kept seconds apart while take-ins run: the revision named newest can be gone before its entries are read. + // A newer one exists then, so the read names the newest again, a few times, before it fails. + var ( + newest meshcatalog.CatalogRevision + any bool + entries []meshcatalog.CatalogEntry + err error + ) + for try := 0; try < rereadTries; try++ { + entries = nil + if newest, any, err = newestTrunk(ctx, o); err == nil && any { + _, entries, err = ReadRevision(ctx, o, RevisionObject(newest)) + } + if !errors.Is(err, ErrNoObject) { + break + } + } if err != nil { return fail(err) } - var entries []meshcatalog.CatalogEntry mods, held, at := Store(newOverlay(nil)), Store(newOverlay(nil)), int64(0) if any { - if _, entries, err = ReadRevision(ctx, o, RevisionObject(newest)); err != nil { - return fail(err) - } if mods, held, at, err = viewOfEntries(newest.Revision, entries); err != nil { return fail(err) } diff --git a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go index c955644c..e3c6b01c 100644 --- a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go +++ b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go @@ -144,3 +144,46 @@ func TestTheVerbModuleReadsOneModule(t *testing.T) { t.Errorf("module nope = %v, want the trunk holds none", err) } } + +// goneUnderRead is a bucket whose pin judge runs while the first entry of the revision named newest is read: a newer revision +// is kept and the one named is removed, its list and its entries, as the indexer's judge does. +type goneUnderRead struct { + *memObjects + once atomic.Bool + prune func() +} + +func (g *goneUnderRead) Get(ctx context.Context, name string) ([]byte, error) { + if strings.HasPrefix(name, "entry.") && g.once.CompareAndSwap(false, true) { + g.prune() + } + return g.memObjects.Get(ctx, name) +} + +// A revision removed while it is read (take-ins keep one every few seconds, each removing the last) is read again as the newest +// one, never failed: on novox the first view of the catalog half never came while revisions 487 to 493 were kept. +func TestTheKeptTrunkReadsTheNewerRevisionWhenTheOneNamedIsRemovedUnderIt(t *testing.T) { + mem := newMemObjects(0) + keep := func(revision int64) { + data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built"}) + digest := fmt.Sprintf("d%d", revision) + rev := meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: digest}}} + if err := KeepRevision(context.Background(), mem, rev, map[string][]byte{digest: data}); err != nil { + t.Fatal(err) + } + } + keep(1) + objects := &goneUnderRead{memObjects: mem} + objects.prune = func() { + keep(2) + _ = mem.Delete(context.Background(), RevisionObject(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk"})) + _ = mem.Delete(context.Background(), EntryObject("d1")) + } + kept := &KeptTrunk{} + if err := kept.Reread(context.Background(), objects); err != nil { + t.Fatalf("the read failed when the revision named was removed under it: %v", err) + } + if rev, _, ok, err := kept.Newest(context.Background(), objects); err != nil || !ok || rev.Revision != 2 { + t.Errorf("the view holds revision %d (%v, %v), want 2", rev.Revision, ok, err) + } +} From 33143527f29bf904863e46dd0d14e3b2b786dab3 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 19 Nov 2026 12:17:40 +0100 Subject: [PATCH 13/20] A read of the newest revision names the newest again while it is newer than the one that failed Replaces the five tries: progress bounds the loop, the same revision failing twice is the read's failure. One helper, readNewestOf, serves the kept trunk's reads, TrunkView and BranchView; a read by number still fails loudly. Builds: design 47 G A-487d (hq 61bf1215), A-487, A-487b Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- .../internal/index/catalog_revisions.go | 55 ++++++++++--------- .../internal/index/kept_trunk_test.go | 42 ++++++++++++++ 2 files changed, 70 insertions(+), 27 deletions(-) diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index 68e0fce5..4772043a 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -182,7 +182,7 @@ func newestOf(ctx context.Context, o Objects, branch string) (meshcatalog.Catalo name := RevisionObject(meshcatalog.CatalogRevision{Branch: branch, Revision: newest}) data, err := o.Get(ctx, name) if err != nil { - return meshcatalog.CatalogRevision{}, false, err + return meshcatalog.CatalogRevision{Branch: branch, Revision: newest}, false, err // the number that failed, for readNewestOf } var rev meshcatalog.CatalogRevision if err := json.Unmarshal(data, &rev); err != nil { @@ -191,6 +191,31 @@ func newestOf(ctx context.Context, o Objects, branch string) (meshcatalog.Catalo return rev, true, nil } +// readNewestOf is the newest revision of a branch and its entries whole (design 47 G A-487d). The pin judge removes a revision +// as soon as a newer one is kept, so a revision whose list or entry is removed under the read is read again as the newest, +// while the newest is newer than the one that failed; the same revision failing twice is the read's failure. Progress bounds +// the loop. No revision kept is false. +func readNewestOf(ctx context.Context, o Objects, branch string) (meshcatalog.CatalogRevision, []meshcatalog.CatalogEntry, bool, error) { + var failed int64 + for { + newest, any, err := newestOf(ctx, o, branch) + if err == nil && !any { + return newest, nil, false, nil + } + var entries []meshcatalog.CatalogEntry + if err == nil { + _, entries, err = ReadRevision(ctx, o, RevisionObject(newest)) + if err == nil { + return newest, entries, true, nil + } + } + if !errors.Is(err, ErrNoObject) || newest.Revision == 0 || newest.Revision <= failed { + return newest, nil, false, err + } + failed = newest.Revision + } +} + // changedEntries are the modules whose entry differs between two revisions' lists, one gone or added included, by name. func changedEntries(before, after []meshcatalog.EntryRef) []string { was := map[string]string{} @@ -438,14 +463,10 @@ func TrunkView(ctx context.Context, o Objects) (modules, nodes Store, at int64, // BranchView is TrunkView of a watched branch: the view read from the newest revision the bucket keeps of that branch // (design 47 G §291, A-93), empty and at 0 when it keeps none. func BranchView(ctx context.Context, o Objects, branch string) (modules, nodes Store, at int64, err error) { - newest, any, err := newestOf(ctx, o, branch) + newest, entries, any, err := readNewestOf(ctx, o, branch) if err != nil || !any { return newOverlay(nil), newOverlay(nil), 0, err } - _, entries, err := ReadRevision(ctx, o, RevisionObject(newest)) - if err != nil { - return nil, nil, 0, err - } return viewOfEntries(newest.Revision, entries) } @@ -522,9 +543,6 @@ type KeptTrunk struct { manifests map[string]string } -// rereadTries is how many times a read names the newest revision again when the one it named was removed under it. -const rereadTries = 5 - // Reread reads the newest trunk revision of the bucket and holds its view; a bucket keeping none holds an empty view at 0. func (k *KeptTrunk) Reread(ctx context.Context, o Objects) error { k.read.Lock() @@ -540,24 +558,7 @@ func (k *KeptTrunk) reread(ctx context.Context, o Objects) error { } return err } - // The pin judge removes a trunk revision, its list and the entries only it names, as soon as a newer one is kept, and - // revisions are kept seconds apart while take-ins run: the revision named newest can be gone before its entries are read. - // A newer one exists then, so the read names the newest again, a few times, before it fails. - var ( - newest meshcatalog.CatalogRevision - any bool - entries []meshcatalog.CatalogEntry - err error - ) - for try := 0; try < rereadTries; try++ { - entries = nil - if newest, any, err = newestTrunk(ctx, o); err == nil && any { - _, entries, err = ReadRevision(ctx, o, RevisionObject(newest)) - } - if !errors.Is(err, ErrNoObject) { - break - } - } + newest, entries, any, err := readNewestOf(ctx, o, TrunkBranch) if err != nil { return fail(err) } diff --git a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go index e3c6b01c..badac4e4 100644 --- a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go +++ b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go @@ -187,3 +187,45 @@ func TestTheKeptTrunkReadsTheNewerRevisionWhenTheOneNamedIsRemovedUnderIt(t *tes t.Errorf("the view holds revision %d (%v, %v), want 2", rev.Revision, ok, err) } } + +// A-487d: the same revision failing twice is the read's failure, said as it is; a read by number never reads another. +func TestTheKeptTrunkFailsWhenTheSameRevisionIsGoneTwice(t *testing.T) { + mem := newMemObjects(0) + data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built"}) + rev := meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: "d1"}}} + if err := KeepRevision(context.Background(), mem, rev, map[string][]byte{"d1": data}); err != nil { + t.Fatal(err) + } + _ = mem.Delete(context.Background(), EntryObject("d1")) // revision 1 stays listed, its entry lost, no newer revision + for name, read := range map[string]func() error{ + "kept trunk": func() error { return (&KeptTrunk{}).Reread(context.Background(), mem) }, + "TrunkView": func() error { _, _, _, err := TrunkView(context.Background(), mem); return err }, + } { + if err := read(); !errors.Is(err, ErrNoObject) { + t.Errorf("%s: %v, want the lost entry said (no such object)", name, err) + } + } +} + +// A-487d: BranchView and TrunkView name the newest again when the one named is removed under the read. +func TestTrunkViewReadsTheNewerRevisionWhenTheOneNamedIsRemovedUnderIt(t *testing.T) { + mem := newMemObjects(0) + keep := func(revision int64) { + data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built"}) + digest := fmt.Sprintf("d%d", revision) + rev := meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: digest}}} + if err := KeepRevision(context.Background(), mem, rev, map[string][]byte{digest: data}); err != nil { + t.Fatal(err) + } + } + keep(1) + objects := &goneUnderRead{memObjects: mem} + objects.prune = func() { + keep(2) + _ = mem.Delete(context.Background(), RevisionObject(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk"})) + _ = mem.Delete(context.Background(), EntryObject("d1")) + } + if _, _, at, err := TrunkView(context.Background(), objects); err != nil || at != 2 { + t.Errorf("TrunkView read revision %d (%v), want 2", at, err) + } +} From 00b452e57bb107bdc6f3fcb3726fdaa06d671dcd Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 19 Nov 2026 12:48:14 +0100 Subject: [PATCH 14/20] The readers of the newest trunk revision read the newest again when it is removed under them ModuleInNewestView, SeedEntryHeld, entryOf, SeedModulesFrom, the first fill of the versions, the outdated tool's trunk read and the verb revision without a kept view use readNewestOf; none of them holds keepMu. Builds: design 47 G A-487d (hq 61bf1215) Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- .../internal/index/kept_trunk_test.go | 40 +++++++++++++++++++ .../internal/index/open_asks.go | 13 +----- .../internal/index/revision_verb.go | 7 +--- .../internal/index/seed_steps.go | 6 +-- .../internal/index/versions.go | 6 +-- .../internal/index/versions_edges.go | 8 ++-- .../internal/index/versions_fill.go | 6 +-- 7 files changed, 49 insertions(+), 37 deletions(-) diff --git a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go index badac4e4..d0202f05 100644 --- a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go +++ b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go @@ -229,3 +229,43 @@ func TestTrunkViewReadsTheNewerRevisionWhenTheOneNamedIsRemovedUnderIt(t *testin t.Errorf("TrunkView read revision %d (%v), want 2", at, err) } } + +// A-487d: the readers of the newest trunk revision in the indexer program (ModuleInNewestView, SeedEntryHeld, entryOf, the verb +// revision without a kept view) read the newest again when the revision they named is removed under them. +func TestTheNewestTrunkReadersReadAgainWhenTheRevisionIsRemovedUnderThem(t *testing.T) { + cases := map[string]func(ix *Indexer) (bool, error){ + "ModuleInNewestView": func(ix *Indexer) (bool, error) { return ix.ModuleInNewestView(context.Background(), "avahi") }, + "SeedEntryHeld": func(ix *Indexer) (bool, error) { return ix.SeedEntryHeld(context.Background(), "avahi", "fp2") }, + "entryOf": func(ix *Indexer) (bool, error) { + e, err := ix.entryOf(context.Background(), "avahi") + return e.Fp == "fp2", err + }, + "Revision": func(ix *Indexer) (bool, error) { + a, err := Revision(context.Background(), ix.Revisions, nil, "avahi", "avahi", meshcatalog.RevisionInput{}) + return a.Revision.Revision == 2, err + }, + } + for name, read := range cases { + t.Run(name, func(t *testing.T) { + mem := newMemObjects(0) + keep := func(revision int64) { + data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built", Fp: fmt.Sprintf("fp%d", revision)}) + digest := fmt.Sprintf("d%d", revision) + rev := meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: digest}}} + if err := KeepRevision(context.Background(), mem, rev, map[string][]byte{digest: data}); err != nil { + t.Fatal(err) + } + } + keep(1) + objects := &goneUnderRead{memObjects: mem} + objects.prune = func() { + keep(2) + _ = mem.Delete(context.Background(), RevisionObject(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk"})) + _ = mem.Delete(context.Background(), EntryObject("d1")) + } + if ok, err := read(&Indexer{Revisions: objects}); err != nil || !ok { + t.Errorf("answered %v (%v), want revision 2's entry", ok, err) + } + }) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/open_asks.go b/modules/mesh-delivery-indexer/internal/index/open_asks.go index 576dc802..684e9e33 100644 --- a/modules/mesh-delivery-indexer/internal/index/open_asks.go +++ b/modules/mesh-delivery-indexer/internal/index/open_asks.go @@ -3,7 +3,6 @@ package index import ( "context" "errors" - "fmt" "slices" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" @@ -51,14 +50,10 @@ func (ix *Indexer) SeedEntryHeld(ctx context.Context, module, fp string) (bool, if ctx.Err() != nil || ix.Revisions == nil { return false, ctx.Err() } - rev, found, err := newestTrunk(ctx, ix.Revisions) + _, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch) if err != nil || !found { return false, err } - _, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision)) - if err != nil { - return false, err - } return slices.ContainsFunc(entries, func(e meshcatalog.CatalogEntry) bool { return e.Module == module && e.Fp == fp }), nil } @@ -67,14 +62,10 @@ func (ix *Indexer) ModuleInNewestView(ctx context.Context, module string) (bool, if ctx.Err() != nil || ix.Revisions == nil { return false, ctx.Err() } - rev, found, err := newestTrunk(ctx, ix.Revisions) + _, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch) if err != nil || !found { return false, err } - _, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision)) - if err != nil { - return false, err - } return slices.ContainsFunc(entries, func(e meshcatalog.CatalogEntry) bool { return e.Module == module }), nil } diff --git a/modules/mesh-delivery-indexer/internal/index/revision_verb.go b/modules/mesh-delivery-indexer/internal/index/revision_verb.go index c3e4a589..20d0d8a1 100644 --- a/modules/mesh-delivery-indexer/internal/index/revision_verb.go +++ b/modules/mesh-delivery-indexer/internal/index/revision_verb.go @@ -77,7 +77,7 @@ func Revision(ctx context.Context, o Objects, kept *KeptTrunk, holder, caller st if kept != nil { newest, trunkEntries, ok, err = kept.Newest(ctx, o) } else { - newest, ok, err = newestTrunk(ctx, o) + newest, trunkEntries, ok, err = readNewestOf(ctx, o, TrunkBranch) } if err != nil { return out, fmt.Errorf("the catalog graph revisions cannot be read: %w", err) @@ -86,11 +86,6 @@ func Revision(ctx context.Context, o Objects, kept *KeptTrunk, holder, caller st // An answer is told by its fields (design 47 G §530): no trunk revision kept yet is `none-yet`, no error. return meshcatalog.RevisionAnswer{NoneYet: true}, nil } - if kept == nil { - if _, trunkEntries, err = ReadRevision(ctx, o, RevisionObject(newest)); err != nil { - return out, err - } - } caller = CallerModule(trunkEntries, caller) if err := views.Refusal(meshcatalog.Seat+"."+meshcatalog.VerbRevision, RevisionCallers(trunkEntries, holder), caller); err != nil { return out, fmt.Errorf("%w (novox/hq ADR 0663 §2: a part reads it by declaring %s in invokes)", err, RevisionInvoke) diff --git a/modules/mesh-delivery-indexer/internal/index/seed_steps.go b/modules/mesh-delivery-indexer/internal/index/seed_steps.go index 4fece72d..1aed0f99 100644 --- a/modules/mesh-delivery-indexer/internal/index/seed_steps.go +++ b/modules/mesh-delivery-indexer/internal/index/seed_steps.go @@ -63,17 +63,13 @@ func (ix *Indexer) SeedModulesFrom(ctx context.Context, defs map[string]seatdef. if ix.Revisions == nil { return nil, fmt.Errorf("the catalog graph is not kept here: no trunk revision to seed from") } - rev, found, err := newestTrunk(ctx, ix.Revisions) + _, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch) if err != nil { return nil, err } if !found { return nil, fmt.Errorf("the indexer keeps no trunk revision yet: the seed waits for its first") } - _, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision)) - if err != nil { - return nil, err - } seatOf := map[string]string{} // artifact reference -> its build seat, all found before any call is made unbuilt := map[string][]string{} // module -> each artifact of it no build seat builds, with its language or kind for _, e := range entries { diff --git a/modules/mesh-delivery-indexer/internal/index/versions.go b/modules/mesh-delivery-indexer/internal/index/versions.go index c2a608ee..5c5db15a 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions.go +++ b/modules/mesh-delivery-indexer/internal/index/versions.go @@ -177,17 +177,13 @@ func (ix *Indexer) entryOf(ctx context.Context, module string) (meshcatalog.Cata if ix.Revisions == nil { return meshcatalog.CatalogEntry{}, errors.New("the catalog graph is not kept here: no entry names the module's artifacts") } - rev, found, err := newestTrunk(ctx, ix.Revisions) + _, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch) if err != nil { return meshcatalog.CatalogEntry{}, err } if !found { return meshcatalog.CatalogEntry{}, errors.New("the indexer keeps no trunk revision yet") } - _, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision)) - if err != nil { - return meshcatalog.CatalogEntry{}, err - } for _, e := range entries { if e.Module == module { return e, nil diff --git a/modules/mesh-delivery-indexer/internal/index/versions_edges.go b/modules/mesh-delivery-indexer/internal/index/versions_edges.go index a4380010..430597b9 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions_edges.go +++ b/modules/mesh-delivery-indexer/internal/index/versions_edges.go @@ -194,11 +194,9 @@ func (ix *Indexer) Outdated(ctx context.Context, in meshcatalog.OutdatedInput) ( } trunk := map[string]meshcatalog.CatalogEntry{} // the newest trunk revision's entries, read where a kind needs them if ix.Revisions != nil { - if rev, found, err := newestTrunk(ctx, ix.Revisions); err == nil && found { - if _, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision)); err == nil { - for _, e := range entries { - trunk[e.Module] = e - } + if _, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch); err == nil && found { + for _, e := range entries { + trunk[e.Module] = e } } } diff --git a/modules/mesh-delivery-indexer/internal/index/versions_fill.go b/modules/mesh-delivery-indexer/internal/index/versions_fill.go index ac12fda6..b29aa4e3 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions_fill.go +++ b/modules/mesh-delivery-indexer/internal/index/versions_fill.go @@ -79,17 +79,13 @@ func (ix *Indexer) FillVersions(ctx context.Context, ask VersionsAsker) error { if ix.Revisions == nil { return errors.New("the catalog graph is not kept here: no entry names the modules' artifacts") } - rev, found, err := newestTrunk(ctx, ix.Revisions) + _, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch) if err != nil { return err } if !found { return errors.New("the indexer keeps no trunk revision yet: the first fill of the versions waits for its first") } - _, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision)) - if err != nil { - return err - } type found1 struct { v BuilderVersion name string From 6d6caf20e164042a7a7893df6735292ea471f632 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 19 Nov 2026 12:50:53 +0100 Subject: [PATCH 15/20] takein: the test's pending revision names a lost merge, since A-476 (b) no longer counts a build it waits for TestTakeInEvents failed on main since A-476 (b) dropped WaitsFor as a reason a pending revision holds; its sample and one refusal case still named only that. Seen in main.1025.mesh-delivery-indexer, unit and integration. Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- modules/mesh-delivery-indexer/takein/takein_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/mesh-delivery-indexer/takein/takein_test.go b/modules/mesh-delivery-indexer/takein/takein_test.go index a2376fe6..67029fd2 100644 --- a/modules/mesh-delivery-indexer/takein/takein_test.go +++ b/modules/mesh-delivery-indexer/takein/takein_test.go @@ -16,7 +16,7 @@ import ( func sampleTakeIns(since time.Time) []TakeIn { m := []Merge{{Repository: "novox/mesh-delivery", Number: 9, Merge: "eeee"}} return []TakeIn{ - {Kind: TakeInPending, Key: PendingKey(12), What: TakeInRevision, Merges: m, WaitsFor: "src2:abc", Since: since}, + {Kind: TakeInPending, Key: PendingKey(12), What: TakeInRevision, Merges: m, Lost: []string{"m.00000000000000000011"}, Since: since}, {Kind: TakeInPending, Key: GroupTakeInKey("feat/take-in"), What: TakeInGroup, Merges: m, Group: "feat/take-in", Members: []meshdeliveryclient.Head{{Repository: "novox/mesh-delivery", Number: 9, Head: "ffff"}, {Repository: "novox/mesh-sdk", Number: 37, Head: "dddd"}}, Since: since}, {Kind: TakeInTaken, Key: PendingKey(12), What: TakeInRevision, Merges: m, WaitsFor: "src2:abc", Since: since, @@ -70,7 +70,7 @@ func TestTakeInEvents(t *testing.T) { }{ {"group take-in without its group", "names its group", 1, func(t *TakeIn) { t.Group = "" }}, {"group take-in under another group's key", "g.other", 1, func(t *TakeIn) { t.Key = GroupTakeInKey("other") }}, - {"pending revision waiting for no build", "lost", 0, func(t *TakeIn) { t.WaitsFor = "" }}, + {"pending revision naming neither a lost merge nor a refusal", "lost", 0, func(t *TakeIn) { t.Lost = nil }}, {"pending with a hand act", "only a taken event", 0, func(t *TakeIn) { t.By = "someone@node" }}, {"hand act without its by", "by", 2, func(t *TakeIn) { t.By = "" }}, {"take-in without merges", "no merge", 0, func(t *TakeIn) { t.Merges = nil }}, From 900316e4bd13fedc317557c2e886bf08f011d32f Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 19 Nov 2026 12:51:40 +0100 Subject: [PATCH 16/20] The tool indexer_catalog_revision and the verb revision of a watched branch read the newest again The default read of the tool and the newest revision of a branch other than the trunk use readNewestOf; a read by number keeps failing loudly. One table covers every reader of the newest revision in the indexer program. Builds: design 47 G A-487d (hq 61bf1215) Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- .../internal/index/kept_trunk_test.go | 90 +++++++++++++------ .../internal/index/revision_tools.go | 14 +-- .../internal/index/revision_verb.go | 10 ++- 3 files changed, 79 insertions(+), 35 deletions(-) diff --git a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go index d0202f05..ab545752 100644 --- a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go +++ b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go @@ -8,6 +8,7 @@ import ( "strings" "sync/atomic" "testing" + "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-sdk/go/views" @@ -151,10 +152,11 @@ type goneUnderRead struct { *memObjects once atomic.Bool prune func() + on string // when set, only the read of this entry runs the pruning } func (g *goneUnderRead) Get(ctx context.Context, name string) ([]byte, error) { - if strings.HasPrefix(name, "entry.") && g.once.CompareAndSwap(false, true) { + if strings.HasPrefix(name, "entry.") && (g.on == "" || name == g.on) && g.once.CompareAndSwap(false, true) { g.prune() } return g.memObjects.Get(ctx, name) @@ -230,40 +232,76 @@ func TestTrunkViewReadsTheNewerRevisionWhenTheOneNamedIsRemovedUnderIt(t *testin } } -// A-487d: the readers of the newest trunk revision in the indexer program (ModuleInNewestView, SeedEntryHeld, entryOf, the verb -// revision without a kept view) read the newest again when the revision they named is removed under them. -func TestTheNewestTrunkReadersReadAgainWhenTheRevisionIsRemovedUnderThem(t *testing.T) { - cases := map[string]func(ix *Indexer) (bool, error){ - "ModuleInNewestView": func(ix *Indexer) (bool, error) { return ix.ModuleInNewestView(context.Background(), "avahi") }, - "SeedEntryHeld": func(ix *Indexer) (bool, error) { return ix.SeedEntryHeld(context.Background(), "avahi", "fp2") }, - "entryOf": func(ix *Indexer) (bool, error) { - e, err := ix.entryOf(context.Background(), "avahi") +// A-487d: every reader of the newest revision in the indexer program reads the newest again when the revision it named is +// removed under it (ModuleInNewestView, SeedEntryHeld, entryOf, SeedModulesFrom, FillVersions, the tool indexer_catalog_revision +// and the verb revision, trunk's and a watched branch's). +func TestTheNewestRevisionReadersReadAgainWhenTheRevisionIsRemovedUnderThem(t *testing.T) { + ctx := context.Background() + cases := map[string]struct { + branch string + read func(ix *Indexer) (bool, error) + }{ + "ModuleInNewestView": {"", func(ix *Indexer) (bool, error) { return ix.ModuleInNewestView(ctx, "avahi") }}, + "SeedEntryHeld": {"", func(ix *Indexer) (bool, error) { return ix.SeedEntryHeld(ctx, "avahi", "fp2") }}, + "entryOf": {"", func(ix *Indexer) (bool, error) { + e, err := ix.entryOf(ctx, "avahi") return e.Fp == "fp2", err - }, - "Revision": func(ix *Indexer) (bool, error) { - a, err := Revision(context.Background(), ix.Revisions, nil, "avahi", "avahi", meshcatalog.RevisionInput{}) - return a.Revision.Revision == 2, err - }, + }}, + "SeedModulesFrom": {"", func(ix *Indexer) (bool, error) { _, err := ix.SeedModulesFrom(ctx, nil, nil); return err == nil, err }}, + "FillVersions": {"", func(ix *Indexer) (bool, error) { + ix.Versions, ix.Heard = MemoryStore(), MemoryStore() + err := ix.FillVersions(ctx, func(context.Context, string, string) (BuilderVersionsAnswer, error) { + return BuilderVersionsAnswer{}, nil + }) + return err == nil, err + }}, + "indexer_catalog_revision": {"", func(ix *Indexer) (bool, error) { + var tool map[string]any + for _, tl := range CatalogRevisionTools(ix.Revisions, nil, time.Now, nil) { + if tl.Name == CatalogRevisionTool { + got, err := tl.Run(map[string]any{}) + if err != nil { + return false, err + } + tool = got.(map[string]any) + } + } + return tool != nil && tool["name"] == "trunk.2", nil + }}, + "Revision": {"", func(ix *Indexer) (bool, error) { + a, err := Revision(ctx, ix.Revisions, nil, "avahi", "avahi", meshcatalog.RevisionInput{}) + return a.Revision != nil && a.Revision.Revision == 2, err + }}, + "Revision of a branch": {"dev", func(ix *Indexer) (bool, error) { + a, err := Revision(ctx, ix.Revisions, nil, "avahi", "avahi", meshcatalog.RevisionInput{Branch: "dev"}) + return a.Revision != nil && a.Revision.Revision == 2 && len(a.Entries) == 1 && a.Entries[0].Fp == "fp2", err + }}, } - for name, read := range cases { + for name, c := range cases { t.Run(name, func(t *testing.T) { mem := newMemObjects(0) - keep := func(revision int64) { + keep := func(branch string, revision int64) { data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built", Fp: fmt.Sprintf("fp%d", revision)}) - digest := fmt.Sprintf("d%d", revision) - rev := meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: digest}}} - if err := KeepRevision(context.Background(), mem, rev, map[string][]byte{digest: data}); err != nil { + digest := fmt.Sprintf("%s-d%d", branch, revision) + rev := meshcatalog.CatalogRevision{Branch: branch, Revision: revision, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: digest}}} + if err := KeepRevision(ctx, mem, rev, map[string][]byte{digest: data}); err != nil { t.Fatal(err) } } - keep(1) - objects := &goneUnderRead{memObjects: mem} - objects.prune = func() { - keep(2) - _ = mem.Delete(context.Background(), RevisionObject(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk"})) - _ = mem.Delete(context.Background(), EntryObject("d1")) + keep("", 1) + if c.branch != "" { // the trunk stays at revision 1; the branch's revision 1 is removed under the read + keep("", 2) + keep(c.branch, 1) } - if ok, err := read(&Indexer{Revisions: objects}); err != nil || !ok { + objects := &goneUnderRead{memObjects: mem} + pruned := c.branch + objects.on = EntryObject(fmt.Sprintf("%s-d1", pruned)) + objects.prune = func() { + keep(pruned, 2) + _ = mem.Delete(ctx, RevisionObject(meshcatalog.CatalogRevision{Branch: pruned, Revision: 1, Kind: "trunk"})) + _ = mem.Delete(ctx, objects.on) + } + if ok, err := c.read(&Indexer{Revisions: objects}); err != nil || !ok { t.Errorf("answered %v (%v), want revision 2's entry", ok, err) } }) diff --git a/modules/mesh-delivery-indexer/internal/index/revision_tools.go b/modules/mesh-delivery-indexer/internal/index/revision_tools.go index 476e25ca..a21bbd3c 100644 --- a/modules/mesh-delivery-indexer/internal/index/revision_tools.go +++ b/modules/mesh-delivery-indexer/internal/index/revision_tools.go @@ -324,24 +324,28 @@ func CatalogRevisionTools(o Objects, ask func(key string, value []byte) (TakeInA ctx := context.Background() n, _ := args["revision"].(float64) name := "" + var rev meshcatalog.CatalogRevision + var entries []meshcatalog.CatalogEntry switch { case str(args, "group") != "" || str(args, "heads") != "": name = RevisionObject(meshcatalog.CatalogRevision{Kind: "candidate", Group: str(args, "group"), Heads: str(args, "heads")}) case n > 0: name = "trunk." + strconv.FormatInt(int64(n), 10) default: - newest, ok, err := newestTrunk(ctx, o) + newest, newestEntries, ok, err := readNewestOf(ctx, o, TrunkBranch) if err != nil { return nil, fmt.Errorf("the catalog graph revisions cannot be read: %w", err) } if !ok { return nil, ErrNoRevision } - name = RevisionObject(newest) + name, rev, entries = RevisionObject(newest), newest, newestEntries } - rev, entries, err := ReadRevision(ctx, o, name) - if err != nil { - return nil, err + if entries == nil { + var err error + if rev, entries, err = ReadRevision(ctx, o, name); err != nil { + return nil, err + } } return map[string]any{"name": name, "revision": rev, "entries": entries}, nil }}, diff --git a/modules/mesh-delivery-indexer/internal/index/revision_verb.go b/modules/mesh-delivery-indexer/internal/index/revision_verb.go index 20d0d8a1..f2feabb7 100644 --- a/modules/mesh-delivery-indexer/internal/index/revision_verb.go +++ b/modules/mesh-delivery-indexer/internal/index/revision_verb.go @@ -72,7 +72,7 @@ func Revision(ctx context.Context, o Objects, kept *KeptTrunk, holder, caller st var out meshcatalog.RevisionAnswer var newest meshcatalog.CatalogRevision var ok bool - var trunkEntries []meshcatalog.CatalogEntry + var trunkEntries, branchEntries []meshcatalog.CatalogEntry var err error if kept != nil { newest, trunkEntries, ok, err = kept.Newest(ctx, o) @@ -97,14 +97,14 @@ func Revision(ctx context.Context, o Objects, kept *KeptTrunk, holder, caller st return out, fmt.Errorf("the branch %q is no escaped name: %w", in.Branch, err) } if !isTrunkBranch(branch) { - other, found, err := newestOf(ctx, o, branch) + other, entries, found, err := readNewestOf(ctx, o, branch) if err != nil { return out, fmt.Errorf("the catalog graph revisions of %s cannot be read: %w", branch, err) } if !found { return meshcatalog.RevisionAnswer{NoneYet: true}, nil } - newest = other + newest, branchEntries = other, entries } name := RevisionObject(newest) switch { @@ -128,7 +128,9 @@ func Revision(ctx context.Context, o Objects, kept *KeptTrunk, holder, caller st } rev, entries := newest, trunkEntries if !isTrunkBranch(branch) && in.Group == "" && in.Heads == "" { - if rev, entries, err = ReadRevision(ctx, o, name); err != nil { + if name == RevisionObject(newest) { + entries = branchEntries // the newest of the branch, read by readNewestOf (A-487d) + } else if rev, entries, err = ReadRevision(ctx, o, name); err != nil { return out, err } } else if name != RevisionObject(newest) { From 04500fa29df15683f7ad681c08fbeb8d0e0194f2 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 19 Nov 2026 12:53:33 +0100 Subject: [PATCH 17/20] Outdated says a trunk read that fails twice instead of leaving out its kinds 3 and 5 Builds: design 47 G A-487d (hq 61bf1215) Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- .../internal/index/kept_trunk_test.go | 16 ++++++++++++++++ .../internal/index/versions_edges.go | 6 +++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go index ab545752..0dd1eb1f 100644 --- a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go +++ b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go @@ -202,6 +202,10 @@ func TestTheKeptTrunkFailsWhenTheSameRevisionIsGoneTwice(t *testing.T) { for name, read := range map[string]func() error{ "kept trunk": func() error { return (&KeptTrunk{}).Reread(context.Background(), mem) }, "TrunkView": func() error { _, _, _, err := TrunkView(context.Background(), mem); return err }, + "Outdated": func() error { + _, err := outdatedOfAvahi(mem).Outdated(context.Background(), meshcatalog.OutdatedInput{}) + return err + }, } { if err := read(); !errors.Is(err, ErrNoObject) { t.Errorf("%s: %v, want the lost entry said (no such object)", name, err) @@ -268,6 +272,10 @@ func TestTheNewestRevisionReadersReadAgainWhenTheRevisionIsRemovedUnderThem(t *t } return tool != nil && tool["name"] == "trunk.2", nil }}, + "Outdated": {"", func(ix *Indexer) (bool, error) { + a, err := outdatedOfAvahi(ix.Revisions).Outdated(ctx, meshcatalog.OutdatedInput{}) + return len(a.Behind) == 1 && a.Behind[0].Kind == 5 && strings.Contains(a.Behind[0].Newest, "fp2"), err + }}, "Revision": {"", func(ix *Indexer) (bool, error) { a, err := Revision(ctx, ix.Revisions, nil, "avahi", "avahi", meshcatalog.RevisionInput{}) return a.Revision != nil && a.Revision.Revision == 2, err @@ -307,3 +315,11 @@ func TestTheNewestRevisionReadersReadAgainWhenTheRevisionIsRemovedUnderThem(t *t }) } } + +// outdatedOfAvahi is an indexer whose state versions holds avahi at a kind-5 edge, so Outdated reads the trunk's entry of avahi. +func outdatedOfAvahi(o Objects) *Indexer { + ix := &Indexer{Versions: MemoryStore(), Heard: MemoryStore(), Revisions: o} + data, _ := json.Marshal(meshcatalog.VersionEntry{V: 1, Module: "avahi", Version: "0.1.1", Edges: []meshcatalog.VersionEdge{{Kind: 5, Part: "actor:program"}}}) + _ = ix.Versions.Create(context.Background(), meshcatalog.VersionsByModuleVersionKey("avahi", "0.1.1"), data) + return ix +} diff --git a/modules/mesh-delivery-indexer/internal/index/versions_edges.go b/modules/mesh-delivery-indexer/internal/index/versions_edges.go index 430597b9..4865bb2e 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions_edges.go +++ b/modules/mesh-delivery-indexer/internal/index/versions_edges.go @@ -194,7 +194,11 @@ func (ix *Indexer) Outdated(ctx context.Context, in meshcatalog.OutdatedInput) ( } trunk := map[string]meshcatalog.CatalogEntry{} // the newest trunk revision's entries, read where a kind needs them if ix.Revisions != nil { - if _, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch); err == nil && found { + _, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch) + if err != nil { + return out, fmt.Errorf("the trunk's entries, which the kinds 3 and 5 are told by, cannot be read: %w", err) + } + if found { for _, e := range entries { trunk[e.Module] = e } From b3e52ecdef497f560088d922926bca005eb0e47f Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 19 Nov 2026 13:41:48 +0100 Subject: [PATCH 18/20] A pipeline step's ask names the language of the build seat it is asked of stepAsk took the artifact's language word (typescript), which node-build-node's module refuses as unreadable (go, node, image). The language is now derived from the chosen seat (buildstep.AskLanguageOfSeat), and the placement test holds every ask to its seat. Builds: design 47 G A-488 (hq 25df48ad) Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- .../internal/pipeline/place.go | 13 +++++++++--- .../internal/pipeline/place_test.go | 21 ++++++++++++------- modules/mesh-delivery-sdk/buildstep/ask.go | 13 ++++++++++++ 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place.go b/modules/mesh-delivery-pipeline/internal/pipeline/place.go index a9bec3c3..14a82a20 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place.go @@ -127,9 +127,16 @@ func place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog steps = append(steps, s) continue } + // The ask names the language of the seat it is asked of, never the artifact's word (design 47 G A-488). + language, err := sdkstep.AskLanguageOfSeat(bs) + if err != nil { + s.Outcome, s.Started, s.Ended, s.Why = seat.StepResultOutcomeCouldNotRun, &now, &now, err.Error() + steps = append(steps, s) + continue + } s.Seat, s.Outcome, s.Started = bs, seat.StepResultOutcomeRunning, &now s.Ask = AskID(startOf(p, now), p.Pipeline, a.Ref, step) - body := stepAsk(p, entry, a, step, s.Ask, asker, now) + body := stepAsk(p, entry, a, language, step, s.Ask, asker, now) ask := Asked{Seat: bs, ID: s.Ask, Ask: body} if share && BoundSeat(bs) { digest := ShareDigest(p.Fingerprint, step, status, body.Step.Fetch) @@ -192,14 +199,14 @@ type Asked struct { Readers []seat.TestRunReader } -func stepAsk(p seat.Pipeline, e catalog.CatalogEntry, a catalog.Artifact, step, id, asker string, now time.Time) nodebuildgo.StepAsk { +func stepAsk(p seat.Pipeline, e catalog.CatalogEntry, a catalog.Artifact, language, step, id, asker string, now time.Time) nodebuildgo.StepAsk { // A step is for its module alone (principle 10, A-447): the fetch always names the module's directory, and the // artifact's named paths in it when the entry has them. fetch := nodebuildgo.Fetch{Repository: p.Repository, Commit: p.Commit, Dir: e.Path} if paths := artifactPaths(e, a.Ref); len(paths) > 0 { fetch.Paths = paths } - return nodebuildgo.StepAsk{V: 1, ID: id, Asker: asker, Kind: "step", Language: a.Language, At: &now, + return nodebuildgo.StepAsk{V: 1, ID: id, Asker: asker, Kind: "step", Language: language, At: &now, Step: &nodebuildgo.StepRun{Artifact: a.Ref, Branch: p.Branch, Commit: p.Commit, Fetch: fetch, Fingerprint: p.Fingerprint, Module: p.Module, Pipeline: p.Pipeline, Revision: p.Source, Step: step}} } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go index eec07992..13fcdbb4 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -295,8 +295,9 @@ func TestAnArtifactWithoutALanguageIsPlacedByItsKind(t *testing.T) { reg := catalog.Registration{Module: "app", Policy: catalog.Policy{Steps: catalog.StepStatuses{Lint: "required", Unit: "required", Integration: "required"}}} entries := map[string]catalog.CatalogEntry{ "app": {Module: "app", Artifacts: []catalog.Artifact{{Ref: "app:server", Kind: "image"}, {Ref: "app:files", Kind: "archive"}, - {Ref: "app:mirror", Kind: "upstream"}, {Ref: "app:web", Language: "rust"}}}, + {Ref: "app:mirror", Kind: "upstream"}, {Ref: "app:web", Language: "rust"}, {Ref: "app:lib", Language: "typescript"}}}, "mesh-build-image": {Module: "mesh-build-image", Acts: []catalog.ActingRule{{Seat: "node-build-image", Step: "build", On: seatdef.OnKindImage}}}, + "mesh-build-node": {Module: "mesh-build-node", Acts: []catalog.ActingRule{{Seat: "node-build-node", Step: "build", On: "typescript"}}}, } steps, asks := Place(seat.Pipeline{Pipeline: "main.4.app", Module: "app"}, reg, entries, time.Unix(0, 0), "the-runner") want := map[string]struct{ outcome, seat, status string }{ @@ -304,6 +305,7 @@ func TestAnArtifactWithoutALanguageIsPlacedByItsKind(t *testing.T) { "app:files": {seat.StepResultOutcomeBypassed, "", seat.StepResultStatusBypass}, "app:mirror": {seat.StepResultOutcomeBypassed, "", seat.StepResultStatusBypass}, "app:web": {seat.StepResultOutcomeCouldNotRun, "", "required"}, + "app:lib": {seat.StepResultOutcomeRunning, "node-build-node", "required"}, } n := 0 for _, s := range steps { @@ -316,16 +318,21 @@ func TestAnArtifactWithoutALanguageIsPlacedByItsKind(t *testing.T) { t.Errorf("%s %s is %s/%s at %q (%s), want %s/%s at %q", s.Artifact, s.Step, s.Status, s.Outcome, s.Seat, s.Why, w.status, w.outcome, w.seat) } } - if n != 12 { - t.Errorf("placed %d build steps of the four artifacts, want 12", n) + if n != 15 { + t.Errorf("placed %d build steps of the five artifacts, want 15", n) } for _, a := range asks { - if a.Seat != builds.Seat && a.Seat != wants.Seat && a.Seat != "node-build-image" { - t.Errorf("asked %s, want only the build, the composition and the image's build seat", a.Seat) + if a.Seat != builds.Seat && a.Seat != wants.Seat && a.Seat != "node-build-image" && a.Seat != "node-build-node" { + t.Errorf("asked %s, want only the build, the composition and the image's and the node's build seats", a.Seat) + } + // Design 47 G A-488: a step's ask names the language of the seat it is asked of (node-build-node node, node-build-image + // image), never the artifact's word (typescript). + if ask, ok := a.Ask.(nodebuildgo.StepAsk); ok && ask.Language != strings.TrimPrefix(a.Seat, "node-build-") { + t.Errorf("the ask %s on %s names the language %q, want %q", a.ID, a.Seat, ask.Language, strings.TrimPrefix(a.Seat, "node-build-")) } } - if len(asks) != 5 { - t.Errorf("asked %d, want the build, the image's three steps and the composition", len(asks)) + if len(asks) != 8 { + t.Errorf("asked %d, want the build, the image's and the node's three steps each and the composition", len(asks)) } } diff --git a/modules/mesh-delivery-sdk/buildstep/ask.go b/modules/mesh-delivery-sdk/buildstep/ask.go index 2e664ce9..e6d3e96f 100644 --- a/modules/mesh-delivery-sdk/buildstep/ask.go +++ b/modules/mesh-delivery-sdk/buildstep/ask.go @@ -141,6 +141,19 @@ func BuildSeat(language string) (string, error) { return "node-build-" + language, nil } +// AskLanguageOfSeat is the language of the asks a build seat takes: node-build-go go, node-build-node node, node-build-image +// image (BuildSeat's inverse). A pipeline step's ask names it, never the artifact's language word (design 47 G A-488). +func AskLanguageOfSeat(seat string) (string, error) { + language, ok := strings.CutPrefix(seat, "node-build-") + if !ok { + return "", fmt.Errorf("buildstep: %q is no build seat", seat) + } + if err := checkAskLanguage(language); err != nil { + return "", err + } + return language, nil +} + // AskSubject is the subject a build seat accepts a queue's asks on: the seat's accept subject of main's form, which names no // asker (design 47 G §600, §483). func AskSubject(language, queue string) (string, error) { From 2e03fcb8751ab849b982209bc5f4e4d5dadaaa38 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 19 Nov 2026 13:48:29 +0100 Subject: [PATCH 19/20] A step ask's toolchain is read by the artifact's toolchain language, not the ask's The ask's language names the seat (A-488), so the runner read the toolchain of node for a TypeScript step and none matched. Asked carries the artifact's language, and the placement test holds a go and a TypeScript artifact to their seats and their toolchains. Builds: design 47 G A-488, A-488b (hq 57c14047) Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- .../internal/pipeline/place.go | 7 ++++-- .../internal/pipeline/place_test.go | 22 ++++++++++++++----- .../internal/pipeline/start.go | 10 ++++----- 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place.go b/modules/mesh-delivery-pipeline/internal/pipeline/place.go index 14a82a20..72cb8195 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place.go @@ -137,7 +137,7 @@ func place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog s.Seat, s.Outcome, s.Started = bs, seat.StepResultOutcomeRunning, &now s.Ask = AskID(startOf(p, now), p.Pipeline, a.Ref, step) body := stepAsk(p, entry, a, language, step, s.Ask, asker, now) - ask := Asked{Seat: bs, ID: s.Ask, Ask: body} + ask := Asked{Seat: bs, ID: s.Ask, Ask: body, Toolchain: a.Language} if share && BoundSeat(bs) { digest := ShareDigest(p.Fingerprint, step, status, body.Step.Fetch) run := TestRunKey(candidate, step, digest) @@ -196,7 +196,10 @@ type Asked struct { // Run is the test run the ask asks, its key in test-runs, with its level, its share digest and the steps of the pipeline // reading it, the asking step first (§724, §725); empty for an ask of no test run. Run, Level, Digest string - Readers []seat.TestRunReader + // Toolchain is the language of the artifact's toolchain as the catalog keys it (go, typescript; design 47 G A-104, A-488b), + // never the ask's language, which names the seat; empty for an artifact with none (an image). + Toolchain string + Readers []seat.TestRunReader } func stepAsk(p seat.Pipeline, e catalog.CatalogEntry, a catalog.Artifact, language, step, id, asker string, now time.Time) nodebuildgo.StepAsk { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go index 13fcdbb4..3251eb56 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -242,7 +242,9 @@ func TestAStepAskCarriesTheKeptResultsAndTheToolchain(t *testing.T) { reg, entries := placeFixture() e := entries["app"] e.Artifacts[0].Package = &catalog.ArtifactPackage{Ecosystem: "go", Name: "git.example/o/app"} + e.Artifacts = append(e.Artifacts, catalog.Artifact{Ref: "app:lib", Language: "typescript"}) entries["app"] = e + entries["mesh-build-node"] = catalog.CatalogEntry{Module: "mesh-build-node", Acts: []catalog.ActingRule{{Seat: "node-build-node", Step: "build", On: "typescript"}}} store, asks := created{}, &stepAsks{} reads := 0 s := &Starter{Pipelines: store, Say: func(string, any) error { return nil }, Steps: asks, Name: "the-runner", @@ -260,7 +262,7 @@ func TestAStepAskCarriesTheKeptResultsAndTheToolchain(t *testing.T) { if _, err := s.Start(Begin{Branch: "main", Source: 4, Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app", Entries: entries}); err != nil { t.Fatal(err) } - var program, tools *nodebuildgo.StepRun + var program, tools, lib *nodebuildgo.StepRun for _, a := range asks.asks { if ask, ok := a.(nodebuildgo.StepAsk); ok { switch { @@ -268,6 +270,8 @@ func TestAStepAskCarriesTheKeptResultsAndTheToolchain(t *testing.T) { program = ask.Step case ask.Step.Artifact == "app:tools" && ask.Step.Step == "unit": tools = ask.Step + case ask.Step.Artifact == "app:lib" && ask.Step.Step == "unit": + lib = ask.Step } } } @@ -277,6 +281,10 @@ func TestAStepAskCarriesTheKeptResultsAndTheToolchain(t *testing.T) { if tools == nil || len(tools.Kept) != 0 || tools.Toolchain == nil { t.Errorf("an artifact with no package carries no kept result and its language's toolchain: %+v", tools) } + // Design 47 G A-488b: the toolchain is read by the artifact's toolchain language (typescript), never by the ask's (node). + if lib == nil || lib.Toolchain == nil || lib.Toolchain.Language != "typescript" { + t.Errorf("the TypeScript artifact's ask carries the toolchain %+v, want the one of typescript", lib) + } for pkg, want := range map[string]bool{"a/b": true, "a/b/c": true, "a/bc": false, "a": false} { if InPackage(pkg, "a/b") != want { t.Errorf("InPackage(%s, a/b) = %v, want %v", pkg, !want, want) @@ -295,7 +303,8 @@ func TestAnArtifactWithoutALanguageIsPlacedByItsKind(t *testing.T) { reg := catalog.Registration{Module: "app", Policy: catalog.Policy{Steps: catalog.StepStatuses{Lint: "required", Unit: "required", Integration: "required"}}} entries := map[string]catalog.CatalogEntry{ "app": {Module: "app", Artifacts: []catalog.Artifact{{Ref: "app:server", Kind: "image"}, {Ref: "app:files", Kind: "archive"}, - {Ref: "app:mirror", Kind: "upstream"}, {Ref: "app:web", Language: "rust"}, {Ref: "app:lib", Language: "typescript"}}}, + {Ref: "app:mirror", Kind: "upstream"}, {Ref: "app:web", Language: "rust"}, {Ref: "app:lib", Language: "typescript"}, {Ref: "app:cli", Language: "go"}}}, + "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}, "mesh-build-image": {Module: "mesh-build-image", Acts: []catalog.ActingRule{{Seat: "node-build-image", Step: "build", On: seatdef.OnKindImage}}}, "mesh-build-node": {Module: "mesh-build-node", Acts: []catalog.ActingRule{{Seat: "node-build-node", Step: "build", On: "typescript"}}}, } @@ -306,6 +315,7 @@ func TestAnArtifactWithoutALanguageIsPlacedByItsKind(t *testing.T) { "app:mirror": {seat.StepResultOutcomeBypassed, "", seat.StepResultStatusBypass}, "app:web": {seat.StepResultOutcomeCouldNotRun, "", "required"}, "app:lib": {seat.StepResultOutcomeRunning, "node-build-node", "required"}, + "app:cli": {seat.StepResultOutcomeRunning, "node-build-go", "required"}, } n := 0 for _, s := range steps { @@ -318,11 +328,11 @@ func TestAnArtifactWithoutALanguageIsPlacedByItsKind(t *testing.T) { t.Errorf("%s %s is %s/%s at %q (%s), want %s/%s at %q", s.Artifact, s.Step, s.Status, s.Outcome, s.Seat, s.Why, w.status, w.outcome, w.seat) } } - if n != 15 { - t.Errorf("placed %d build steps of the five artifacts, want 15", n) + if n != 18 { + t.Errorf("placed %d build steps of the six artifacts, want 18", n) } for _, a := range asks { - if a.Seat != builds.Seat && a.Seat != wants.Seat && a.Seat != "node-build-image" && a.Seat != "node-build-node" { + if a.Seat != builds.Seat && a.Seat != wants.Seat && a.Seat != "node-build-image" && a.Seat != "node-build-node" && a.Seat != "node-build-go" { t.Errorf("asked %s, want only the build, the composition and the image's and the node's build seats", a.Seat) } // Design 47 G A-488: a step's ask names the language of the seat it is asked of (node-build-node node, node-build-image @@ -331,7 +341,7 @@ func TestAnArtifactWithoutALanguageIsPlacedByItsKind(t *testing.T) { t.Errorf("the ask %s on %s names the language %q, want %q", a.ID, a.Seat, ask.Language, strings.TrimPrefix(a.Seat, "node-build-")) } } - if len(asks) != 8 { + if len(asks) != 11 { t.Errorf("asked %d, want the build, the image's and the node's three steps each and the composition", len(asks)) } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index dcc1ab14..625d8dcb 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -483,16 +483,16 @@ func (s *Starter) carry(key string, asks []Asked) { if a.Package != "" { ask.Step.Kept = kept[a.Package] } - if s.Toolchain == nil || ask.Language == "" { + if s.Toolchain == nil || a.Toolchain == "" { continue } - t, done := chains[ask.Language] + t, done := chains[a.Toolchain] if !done { var err error - if t, err = s.Toolchain(ask.Language); err != nil { - s.logf("the toolchain of %s for the steps of %s is not read: %v", ask.Language, key, err) + if t, err = s.Toolchain(a.Toolchain); err != nil { + s.logf("the toolchain of %s for the steps of %s is not read: %v", a.Toolchain, key, err) } - chains[ask.Language] = t + chains[a.Toolchain] = t } ask.Step.Toolchain = t } From 5bca54bb59b31f1df2f92ecc43106cf630461a04 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 19 Nov 2026 14:31:52 +0100 Subject: [PATCH 20/20] assigner: the verb assignments with seats answers from the watched copy of seat-holders, falling back to a direct listing only while it cannot be read (issue 1982) --- .../cmd/mesh-delivery-assigner/seat_tools.go | 39 ++++++++--- .../mesh-delivery-assigner/seat_tools_test.go | 64 +++++++++++++++++++ .../cmd/mesh-delivery-assigner/tools.go | 2 + 3 files changed, 97 insertions(+), 8 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go index 6be29e84..8d4b894f 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go @@ -93,6 +93,11 @@ type seatTools struct { // seats is the seat list the bundle reads (seatlist.go): until it is read, propose judges no mesh seat the bootstrapped // definitions do not name as held once (ADR 0636 §1). Nil reads nothing. seatList *seatlist.Part + // watched is the tools process's watched copy of the state seat-holders (tools.go wants.watched, which gate.Holders + // also reads: A-298), read by the verb assignments with seats instead of a bus get per key (issue 1982: under load + // that took about 14 s, where the watched copy answers at once). Nil, or a call it cannot answer, falls back to a + // direct listing. + watched *stdio.WatchedStates } func newSeatTools(st seatStore, ask func(string, any) (json.RawMessage, error)) *seatTools { @@ -554,25 +559,43 @@ func (s *seatTools) tools() []stdio.Tool { return out } -// seats is the verb assignments with seats (ADR 0571 §4): who holds each seat and on which node, as the state seat-holders -// holds it; with node, the keys of seats held on that node. -func (s *seatTools) seats(node string) (any, error) { +// seatHolders is every key of the state seat-holders and its raw value: from the watched copy (s.watched) when +// there is one and it answers, else a direct listing, key by key, as the verb answered before (issue 1982). +func (s *seatTools) seatHolders() (map[string][]byte, error) { + if s.watched != nil { + if vals, err := s.watched.Values(mma.StateSeatHolders); err == nil { + return vals, nil + } + } keys, err := s.st.keys(mma.StateSeatHolders) if err != nil { return nil, fmt.Errorf("seat-holders cannot be listed: %w", err) } - sort.Strings(keys) - out := []mma.SeatHolders{} + out := make(map[string][]byte, len(keys)) for _, key := range keys { raw, _, found, err := s.st.get(mma.StateSeatHolders, key) if err != nil { return nil, fmt.Errorf("seat-holders %s cannot be read: %w", key, err) } - if !found { - continue + if found { + out[key] = raw } + } + return out, nil +} + +// seats is the verb assignments with seats (ADR 0571 §4): who holds each seat and on which node, as the state seat-holders +// holds it; with node, the keys of seats held on that node. +func (s *seatTools) seats(node string) (any, error) { + raws, err := s.seatHolders() + if err != nil { + return nil, err + } + keys := slices.Sorted(maps.Keys(raws)) + out := []mma.SeatHolders{} + for _, key := range keys { var sh mma.SeatHolders - if json.Unmarshal(raw, &sh) != nil { + if json.Unmarshal(raws[key], &sh) != nil { continue } if node != "" && sh.Node != node && !slices.ContainsFunc(sh.Holders, func(h mma.SeatHoldersHoldersItem) bool { return h.Node == node }) { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go index 1916b044..fbe233ca 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go @@ -352,6 +352,53 @@ func TestTheSeatsVerbs(t *testing.T) { t.Errorf("seats on ace = %+v, %v", out, err) } }) + t.Run("issue 1982: assignments with seats reads the tools process's watched copy of seat-holders, the store only while it cannot", func(t *testing.T) { + seeded := map[string][]byte{} + put := func(seat, node string, holders ...mma.SeatHoldersHoldersItem) { + raw, err := json.Marshal(mma.SeatHolders{Seat: seat, Node: node, Holders: holders}) + if err != nil { + t.Fatal(err) + } + seeded[seat] = raw + } + put("mesh-broker", "", mma.SeatHoldersHoldersItem{Module: "nats", Node: "ace"}) + put("node-shell.g14", "g14") + st := &countingSeatStore{memStore: memStore{mma.StateSeatHolders: &memKV{values: seeded}}} + sv := newSeatTools(st, nil) + watches := 0 + sv.watched = stdio.WatchedStatesOver(func(state string, handle func(stdio.StateChange) error) error { + watches++ + for k, v := range seeded { + if err := handle(stdio.StateChange{Key: k, Op: stdio.OpPut, Value: v, Current: true}); err != nil { + return err + } + } + return nil + }, func(state, key string) ([]byte, bool, error) { + v, ok := seeded[key] + return v, ok, nil + }) + out, err := sv.seats("") + if err != nil { + t.Fatal(err) + } + if got := out.(map[string]any)["seats"].([]mma.SeatHolders); len(got) != 2 || watches != 1 || st.keysCalls != 0 || st.getCalls != 0 { + t.Errorf("seats from the watched copy = %+v, watched %d time(s), the store's own keys %d get %d (want none)", + got, watches, st.keysCalls, st.getCalls) + } + // The lead's answer on issue 1982: the verb falls back to the store's own listing only while the watched copy + // cannot be read (here, its watch refused); the answer's form does not change. + sv.watched = stdio.WatchedStatesOver(func(string, func(stdio.StateChange) error) error { + return errors.New("refused") + }, func(state, key string) ([]byte, bool, error) { return nil, false, nil }) + out, err = sv.seats("") + if err != nil { + t.Fatal(err) + } + if got := out.(map[string]any)["seats"].([]mma.SeatHolders); len(got) != 2 || st.keysCalls != 1 || st.getCalls != 2 { + t.Errorf("seats falling back to the store = %+v, the store's own keys %d get %d (want 1, 2)", got, st.keysCalls, st.getCalls) + } + }) t.Run("research 072 row 37: a node is written as its keys, and history reads the node key", func(t *testing.T) { ctx := context.Background() kv := &revKV{values: map[string][]byte{}, revs: map[string]uint64{}} @@ -750,6 +797,23 @@ func readManifest() (catalogue.Manifest, error) { return catalogue.Decode(raw) } +// countingSeatStore is a memStore that counts its calls to keys and get, to prove a reader takes the state from the +// watched copy and the store only while that copy cannot be read (issue 1982). +type countingSeatStore struct { + memStore + keysCalls, getCalls int +} + +func (c *countingSeatStore) keys(state string) ([]string, error) { + c.keysCalls++ + return c.memStore.keys(state) +} + +func (c *countingSeatStore) get(state, key string) ([]byte, uint64, bool, error) { + c.getCalls++ + return c.memStore.get(state, key) +} + // countingKV is a memKV that counts its puts. type countingKV struct { memKV diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go index 0382caf9..e1d31602 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go @@ -419,6 +419,8 @@ func serveTools() error { wants.watched = stdio.NewWatchedStates() // the large states read whole from one watched copy (ADR 0580 clause 3) // The callers' seat entries are resolved from the assigner's own seat-holders, its watched copy (A-298). gate.Holders(seatlist.HoldersIn(func() (map[string][]byte, error) { return wants.watched.Values(mma.StateSeatHolders) })) + // The verb assignments with seats answers from the same watched copy, so every caller of it is fast too (issue 1982). + seatVerbs.watched = wants.watched wants.revisions = runtimeObjects{busstate.ObjectsOf(assigngraph.RevisionsObjects)} tools = append(tools, wants.tools()...) keeper := &inputKeeper{vals: stateobjects.ForState(buckets.InputsState), now: time.Now, logf: func(string, ...any) {}}