diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go index d1b03a7e..e6688fb0 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go @@ -1,8 +1,6 @@ package main import ( - "fmt" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" stdio "git.novox.be/novox/mesh-sdk/go" @@ -19,20 +17,13 @@ func catalogRead(modules views.Source, manifest func(digest string) (string, err return index.CatalogRead{ SeatDefinitions: seatDefinitions, Marks: marks, Modules: func() (map[string]meshcatalog.ModulesByModuleValue, error) { - keys, err := modules.Keys() + values, err := index.SourceValues(modules) if err != nil { return nil, err } out := map[string]meshcatalog.ModulesByModuleValue{} - for _, k := range keys { - e, ok, err := modules.Get(k) - if err != nil { - return nil, fmt.Errorf("modules.%s: %w", k, err) - } - if !ok { - continue - } - if m, err := meshcatalogclient.DecodeModule(e.Value); err == nil { + for k, v := range values { + if m, err := meshcatalogclient.DecodeModule(v); err == nil { out[k] = m } } @@ -52,20 +43,13 @@ func catalogRead(modules views.Source, manifest func(digest string) (string, err }, Manifest: manifest, Graph: func() (graph.Graph, error) { - keys, err := nodes.Keys() + values, err := index.SourceValues(nodes) if err != nil { return nil, err } g := graph.Graph{} - for _, k := range keys { - e, ok, err := nodes.Get(k) - if err != nil { - return nil, fmt.Errorf("graph.%s: %w", k, err) - } - if !ok { - continue - } - if n, err := graph.Decode(e.Value); err == nil { + for k, v := range values { + if n, err := graph.Decode(v); err == nil { g[k] = n } } @@ -75,7 +59,27 @@ func catalogRead(modules views.Source, manifest func(digest string) (string, err } // catalogTools are the seat's verbs over the indexer's own states, served beside the indexer's own tools, since the indexer's manifest claims the seat (ADR 0604 §2). -func catalogTools(state func(string) views.Source, kept *index.KeptTrunk) []stdio.Tool { +// Every state a verb reads whole is answered from memory (the view of the trunk held, or a watched copy), so a call costs no +// round trip per key: four node-engines ask every module at once. +func catalogTools(state func(string) views.Source, kept *index.KeptTrunk, watched *stdio.WatchedStates) []stdio.Tool { return index.CatalogTools(catalogRead(state("modules"), kept.Manifest, state(graph.State), - keptSource(index.SeatDefinitionsState), keptSource(views.MarksState))) + watchedSource{watched, index.SeatDefinitionsState}, keptSource(views.MarksState))) } + +// watchedSource is a state as its watched copy holds it (stdio.WatchedStates), as a Source; it offers every value in one read. +type watchedSource struct { + w *stdio.WatchedStates + state string +} + +func (s watchedSource) Keys() ([]string, error) { return s.w.Keys(s.state) } + +func (s watchedSource) Get(key string) (*views.Entry, bool, error) { + v, ok, err := s.w.Get(s.state, key) + if err != nil || !ok { + return nil, false, err + } + return &views.Entry{Value: v}, true, nil +} + +func (s watchedSource) Values() (map[string][]byte, error) { return s.w.Values(s.state) } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index 9f7f936e..69921525 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -307,7 +307,8 @@ func main() { os.Exit(1) } follow.Start(context.Background()) - tools = append(tools, catalogTools(trunk.State, kept)...) + watched := stdio.NewWatchedStates() + tools = append(tools, catalogTools(trunk.State, kept, watched)...) // The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly // the parts whose entry declares it (novox/hq ADR 0663). tools = append(tools, index.RevisionTool(revisions, kept, os.Getenv(stdio.ServedAs)), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}), @@ -336,15 +337,11 @@ func main() { })...) tools = append(tools, viewTools(os.Getenv, trunk.State, time.Now)...) tools = append(tools, index.PendingTools(keptRead{stdio.State(index.PendingState)}, keptRead{stdio.State(index.OwnBuildStateName)}, asks, time.Now)...) - tools = append(tools, index.ValidationTools(asks, time.Now, func(what, why string) error { - _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.ValidateAgainTool, Verb: "validate-again"}) - return err - })...) tools = append(tools, index.FingerprintsTools(asks, time.Now, func(what, why string) error { _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.FingerprintsAgainTool, Verb: "fingerprints-again"}) return err }, 10*time.Second)...) - tools = append(tools, configReadersTool(stdio.NewWatchedStates(), trunk.State("modules")), busStateTool(trunk.State), requirementsTool(keptSource), + tools = append(tools, configReadersTool(watched, trunk.State("modules")), busStateTool(trunk.State), requirementsTool(keptSource), packageUsersTool(trunk.State), seatDefinitionTool(keptSource), outOfStepTool(os.Getenv(stdio.ServedAs), keptSource)) if err := stdio.Serve("", tools); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index b11129a2..2e9dde7f 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -22,7 +22,6 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" @@ -97,11 +96,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 // The forge bridge's tool bridge_merged_message (issue 1585): answers the merged message published for a merge, by // repository@merge, as the bridge does; a restarted indexer holds none in memory and asks it. var bridgeMessages sync.Map - // The build seat node-build-go's stand-in (design 47 G §712, §713): it hears the indexer's validation asks and ends each with - // the manifest validator's verdict; refusing names, by the commit the job runs at, the manifest a revision's change refuses. - var validations *standIn - var refusing sync.Map - js, ctx, candidates, _, restart := startIndexerRun(t, func(ctx context.Context, builds buildLog) { + js, ctx, candidates, _, _ := startIndexerRun(t, func(ctx context.Context, builds buildLog) { // The builder's log entries of two builds' ends (ADR 0571 clause 1), as the builder appends them. builds.end(t, "novox/mesh-delivery", "a", "modules/a") builds.end(t, "novox/mesh-catalog", "c", "modules/c") @@ -128,9 +123,8 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 if _, err := tc.Put(ctx, key, value); err != nil { t.Fatal(err) } - // The trunk holds the indexer's entry, taken in at an earlier commit with its own manifest; the validation names no Go module (A-268). + // The trunk holds the indexer's entry, taken in at an earlier commit with its own manifest; the take-in asks no validation (A-476 a). seedBuildModules(t, ctx, js, map[string]meshcatalog.CatalogEntry{testIndexer: trunkIndexerEntry(t, ctx, js)}) - validations = standInBuildSeat(t, ctx, js, &refusing) // The node-engine's composition on n1 (process.Conn.Call's subjects): the build of the indexer that runs there. answer := func(m *nats.Msg) { raw, _ := json.Marshal(map[string]any{"result": map[string]any{"runs": map[string]string{"fp": *running.Load()}}, "node": "n1"}) @@ -281,17 +275,13 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 } return takein.TakeIn{}, false } - held, ok := said(takein.TakeInPending, m1Key) - if !ok || held.What != takein.TakeInRevision || held.WaitsFor == "" || held.WaitsFor == first || len(held.Merges) != 1 { - t.Fatalf("take-in.pending.%s = %+v (said %v) within 20 s, want the revision waiting for another build than %s", m1Key, held, ok, first) + // A-476 a, b: the revision changing the indexer's own tree is taken in at once, with the build this program runs and no + // validation ask: it is never said pending, and the build the revision names is not waited for. + if taken, ok := said(takein.TakeInTaken, m1Key); !ok || len(taken.Merges) != 1 || taken.Merges[0].Merge != m1.Merge { + t.Fatalf("take-in.taken.%s = %+v (said %v) within 20 s, want the revision taken in at once", m1Key, taken, ok) } - named := held.WaitsFor - // IX5 (§712): the revision was validated first, by one run ask whose job names the indexer entry's repository, path and - // the revision's head, and no heads; accepted, it is never asked again while it waits for the named indexer. - if asks := validations.asked(); len(asks) != 1 || asks[0].Run == nil || asks[0].Kind != sdkstep.AskCheck || - asks[0].Run.Program.Repository != m1.Repository || asks[0].Run.Program.Dir != "modules/"+ix || asks[0].Run.Program.Commit != m1.Merge || - len(asks[0].Run.Heads) != 0 || asks[0].Run.Program.Package != "./cmd/mesh-judge" { - t.Fatalf("validation asks = %+v, want one run of the indexer's manifest validator at %s", asks, m1.Merge) + if held := saidAll(takein.TakeInPending, m1Key); len(held) != 0 { + t.Errorf("the revision was said pending: %+v", held) } get := func(bucket, key string) ([]byte, bool) { t.Helper() @@ -312,11 +302,10 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 } return e.Value(), true } - // Only what building and moving the named indexer needs is written. raw, ok := get(index.ModulesState, ix) own, err := meshcatalogclient.DecodeModule(raw) - if !ok || err != nil || own.Fp2 != named || own.Commit != m1.Merge { - t.Fatalf("modules.%s = %+v (%v), want the named build %s at the merge", ix, own, err, named) + if !ok || err != nil || own.Commit != m1.Merge { + t.Fatalf("modules.%s = %+v (%v), want the indexer at the merge", ix, own, err) } if _, ok := get(index.Bucket(testIndexer, index.ManifestsState), own.Manifest); !ok { t.Errorf("manifests.%s is not written", own.Manifest) @@ -325,16 +314,11 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 t.Errorf("the verdict on the indexer's manifest %s is not written", own.Manifest) } repoKey, _ := trunk.RepositoryKey(m1.Repository) - if _, ok := get(index.ModulesState, "a"); ok { - t.Errorf("modules.a is written while the revision is pending") + if raw, ok := get(index.ModulesState, "a"); !ok || !strings.Contains(string(raw), m1.Merge) { + t.Errorf("modules.a = %s, want it at the merge %s", raw, m1.Merge) } - if _, ok := get(index.Bucket(testIndexer, index.TakenState), repoKey); ok { - t.Errorf("taken.%s is written while the revision is pending", repoKey) - } - if e, err := candidates.Get(ctx, "feat/x"); err != nil { - t.Errorf("the candidate feat/x is gone while the merge of #12 is pending: %v", err) - } else if c, err := index.DecodeCandidate(e.Value()); err != nil || len(c.Pulls) != 2 { - t.Errorf("the candidate feat/x = %+v while the merge of #12 is pending, want its two pull requests", c) + if _, ok := get(index.Bucket(testIndexer, index.TakenState), repoKey); !ok { + t.Errorf("taken.%s is not written", repoKey) } // A group's first member arrives meanwhile; its end is still to come (the knowledge base's remark: it stays across the restart). @@ -365,34 +349,13 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 } var w []takein.PendingWait for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { - if w = waits(); len(w) == 2 && w[0].WaitsFor == named && w[1].Said != "" { + if w = waits(); len(w) == 1 && w[0].Said != "" { break } } - if len(w) != 2 || w[0].Key != m1Key || w[0].WaitsFor != named || !slices.Contains(w[0].Holds, "a") || !slices.Contains(w[0].Holds, ix) || - w[1].Key != groupKey || w[1].What != takein.TakeInGroup || len(w[1].Merges) != 1 || w[1].Said != takein.TakeInPendingEvent { - t.Fatalf("indexer_pending = %+v, want the revision %s waiting for %s holding a and the indexer, then the group gathered", w, m1Key, named) - } - - // The named indexer starts (the node-engine now runs that build): it takes the revision in, and leaves the group. - running.Store(&named) - restart() - if taken, ok := said(takein.TakeInTaken, m1Key); !ok || len(taken.Merges) != 1 || taken.Merges[0].Merge != m1.Merge { - t.Fatalf("take-in.taken.%s = %+v (said %v) within 20 s of the start of %s", m1Key, taken, ok, named) - } - for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { - if w = waits(); len(w) == 1 { - break - } - } - if raw, ok := get(index.ModulesState, "a"); !ok || !strings.Contains(string(raw), m1.Merge) { - t.Errorf("modules.a = %s after the take-in at the start, want it at the merge %s", raw, m1.Merge) - } - if _, ok := get(index.Bucket(testIndexer, index.TakenState), repoKey); !ok { - t.Errorf("taken.%s is not written by the take-in at the start", repoKey) - } - if len(w) != 1 || w[0].Key != groupKey || len(w[0].Keys) != 1 || w[0].Keys[0] != g1Key { - t.Fatalf("indexer_pending after the start = %+v, want only the group gathered, its first member %s", w, g1Key) + if len(w) != 1 || w[0].Key != groupKey || w[0].What != takein.TakeInGroup || len(w[0].Merges) != 1 || len(w[0].Keys) != 1 || w[0].Keys[0] != g1Key || + w[0].Said != takein.TakeInPendingEvent { + t.Fatalf("indexer_pending = %+v, want only the group gathered, its first member %s", w, g1Key) } if _, ok := get(index.ModulesState, "c"); ok { t.Errorf("modules.c is written while its group is gathered") @@ -425,23 +388,6 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 t.Errorf("the group's first member %s was said pending as a revision of its own", g1Key) } - // IX5 (§712, §713): a merge whose changed manifest gains a problem the trunk case lacks stays pending, refused naming it, - // said in take-in.pending; nothing of it is taken in. - m3 := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/y", Number: 13, Head: h1, - Merge: "3333333333333333333333333333333333333333", Root: "r0", Trees: map[string]string{"modules/a": "tree-a-3"}, - Manifests: map[string]string{"modules/a": `{"module":"a","version":"2.0"}`}} - refusing.Store(m3.Merge, "a") - m3Key := takein.PendingKey(merged(m3)) - if refused, ok := said(takein.TakeInPending, m3Key); !ok || !strings.Contains(refused.Refused, "a: the stand-in's problem") || refused.Failed != "" { - t.Fatalf("take-in.pending.%s = %+v (said %v) within 20 s, want the revision refused naming a's problem", m3Key, refused, ok) - } - if w = waits(); len(w) != 1 || w[0].Key != m3Key || w[0].Validation == nil || w[0].Validation.Accepted != 0 || len(validations.asked()) != 3 { - t.Fatalf("indexer_pending = %+v, asks %d: want the refused revision pending with its validation, asked once after m1 and the group", w, len(validations.asked())) - } - if raw, ok := get(index.ModulesState, "a"); !ok || strings.Contains(string(raw), m3.Merge) { - t.Errorf("modules.a = %s: the refused revision was taken in", raw) - } - } // trunkIndexerEntry keeps the indexer's own manifest by its digest, accepted, and answers its trunk entry at an earlier commit. @@ -475,63 +421,6 @@ func trunkIndexerEntry(t *testing.T, ctx context.Context, js jetstream.JetStream return e } -// standIn is the build seat node-build-go as the indexer's validation meets it: the asks it heard, oldest first. -type standIn struct { - mu sync.Mutex - asks []sdkstep.StepAsk -} - -func (s *standIn) asked() []sdkstep.StepAsk { - s.mu.Lock() - defer s.mu.Unlock() - return slices.Clone(s.asks) -} - -// standInBuildSeat keeps the seat's check queue as a stream and hears the indexer's asks on it (accept.check.indexer): each is -// ended at once with step-ended on the seat's family, its answer inline (A-254), the program's last line: the manifest validator's verdict over both cases, -// the change case refusing the manifest refusing names for the job's commit. -func standInBuildSeat(t *testing.T, ctx context.Context, js jetstream.JetStream, refusing *sync.Map) *standIn { - t.Helper() - subject, err := sdkstep.AskSubjectBy(sdkstep.AskGo, sdkstep.AskCheck, index.ValidationAsker) - if err != nil || subject != "mesh.seat.node-build-go.accept.check.mesh-catalog" { - t.Fatalf("the validation's subject %q %v, want the indexer's seat name its asker (A-163, A-232)", subject, err) - } - if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "SEAT_NODE_BUILD_GO_CHECK", Subjects: []string{"mesh.seat.node-build-go.accept.check.>"}}); err != nil { - t.Fatal(err) - } - // The indexer's state inputs and its objects, where the validation's request is kept for the build seat's holder (§354). - if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, index.InputsState), History: 1}); err != nil { - t.Fatal(err) - } - if _, err := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: catalogue.BusName(testIndexer, catalogue.StateObjectsName(index.InputsState))}); err != nil { - t.Fatal(err) - } - s := &standIn{} - if _, err := js.Conn().Subscribe(subject, func(m *nats.Msg) { - a, err := sdkstep.DecodeStepAsk(m.Data) - if err != nil || a.Run == nil { - t.Errorf("the stand-in build seat heard what is no run ask: %v", err) - return - } - s.mu.Lock() - s.asks = append(s.asks, a) - s.mu.Unlock() - change := composer.CaseVerdict{Name: "change", Manifests: map[string]composer.ManifestRead{}} - if module, ok := refusing.Load(a.Run.Program.Commit); ok { - change.Manifests[module.(string)] = composer.ManifestRead{Refused: "the stand-in's problem"} - } - verdict, _ := json.Marshal(composer.Verdict{V: 1, Cases: []composer.CaseVerdict{change, {Name: judge.TrunkCase, Manifests: map[string]composer.ManifestRead{}}}}) - end, _ := json.Marshal(sdkstep.StepEnded{V: 1, ID: a.ID, Ask: a.ID, Asker: index.ValidationAsker, Node: "n1", Outcome: sdkstep.OutcomePassed, - Output: string(verdict), At: time.Now().UTC()}) - if _, err := js.Publish(ctx, nodebuildgo.SubjectStepEnded("n1"), end); err != nil { - t.Errorf("the stand-in build seat could not say step-ended: %v", err) - } - }); err != nil { - t.Fatal(err) - } - return s -} - // jsSource is a bucket as a views.Source. type jsSource struct { ctx context.Context @@ -1529,7 +1418,8 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) { t.Fatal(err) } toolchains = kv - for node, version := range map[string]string{"n1": "go1.27.3", "n2": "go1.27.2"} { + // A-471: a toolchain is agreed by its language version (go1.27), not its patch; n2 reports another language version. + for node, version := range map[string]string{"n1": "go1.27", "n2": "go1.28.1"} { key, _ := index.ToolchainKey(node, "go") value, err := index.EncodeToolchain(nodebuildgo.Toolchain{Language: "go", Version: version, Recipe: "1", At: ptr(time.Now())}) if err != nil { @@ -1573,15 +1463,15 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) { return got } agreed := func(m meshcatalog.ModulesByModuleValue) bool { - return m.Fp2 != "" && m.Toolchains["go"].Version == "go1.27.3" + return m.Fp2 != "" && m.Toolchains["go"].Version == "go1.27" } held := read(agreed) if !agreed(held) { - t.Fatalf("within 20 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27.3 alone", held.Fp2, held.None2, held.Toolchains) + t.Fatalf("within 20 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27 alone", held.Fp2, held.None2, held.Toolchains) } place(ctx, t, "n2", "mesh-build-go") key, _ := index.ToolchainKey("n2", "go") - value, err := index.EncodeToolchain(nodebuildgo.Toolchain{Language: "go", Version: "go1.27.2", Recipe: "1", At: ptr(time.Now().Add(time.Second))}) + value, err := index.EncodeToolchain(nodebuildgo.Toolchain{Language: "go", Version: "go1.28.1", Recipe: "1", At: ptr(time.Now().Add(time.Second))}) if err != nil { t.Fatal(err) } @@ -1604,7 +1494,7 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) { t.Fatal(err) } disagree := func(m meshcatalog.ModulesByModuleValue) bool { - return m.Fp2 == "" && strings.HasPrefix(m.None2, "builders disagree: go1.27.2 recipe 1 on n2") + return m.Fp2 == "" && strings.HasPrefix(m.None2, "builders disagree: go1.27 recipe 1 on n1; go1.28 recipe 1 on n2") } if m := read(disagree); !disagree(m) { t.Fatalf("within 20 s of a merge after n2's assignments avahi has fp2 %q, none2 %q: want the builders to disagree", m.Fp2, m.None2) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index b99d4b94..9bc84e90 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -479,11 +479,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear; // until then every merge is taken in at once, as before. Pending: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.PendingState), "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)}, - SayTakeIn: sayTakeIn(js, cred), - // The validation's run ask (design 47 G §713, A-247, A-248), put under the indexer's own asker. - ValidationAsks: buildrun.Asker{Name: index.ValidationAsker, Publisher: buildrun.JetStreamAsks{JS: js}}, - // The build seat's verb output, for a validation's answer kept as an object (A-254). - AskVerb: calls.CallFile} + SayTakeIn: sayTakeIn(js, cred)} // The registrations and the changes that wait for their warrant (design 47 G §2, §667): their states are bound when they // appear; a warrant is heard on the router's decided event for this module, below. registry := newRegistry(js, conn, cred, calls.Call, diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go index 9eaa74d8..7751e791 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go @@ -206,6 +206,7 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T) if pub.Commit != m2 || len(pub.Packages) != 3 { t.Fatalf("packages.%s = %+v within 20 s, want the merge %s with a digest of go, announce and buildstep", key, pub, m2) } + // A-476a: mesh-sdk's merge moves importer, of another repository, which imports a package it moved. moved := module("importer", func(m meshcatalog.ModulesByModuleValue) bool { return m.Fp2 != importer.Fp2 }) if moved.Packages[sdkGo] != m2 { t.Errorf("importer moved to %+v, want it built against %s", moved.Packages, m2) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go index c2b5210b..75d37b44 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go @@ -28,7 +28,6 @@ type takerIn interface { NamedDrop(ctx context.Context, fingerprint, member, by, why string) index.TakenIn Unpin(ctx context.Context, revision, record, by, why string) index.TakenIn DropRunByHand(ctx context.Context, run, by, why string) index.TakenIn - ValidateAgainByHand(ctx context.Context, key, by, why string) index.TakenIn RegistryAct(ctx context.Context, a index.TakeInAsk) index.TakenIn } @@ -75,9 +74,6 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream } else if a.Merge != "" { asked, r = "hand take-in of a merge", ix.TakeInMerge(ctx, a.MergeRepository, a.Merge, a.Sequence, a.By, a.Why) a.Module = r.Module - } else if a.Pending != "" && a.ValidateAgain { - asked, r = "validation asked again", ix.ValidateAgainByHand(ctx, a.Pending, a.By, a.Why) - a.Module = a.Pending } else if a.Pending != "" && a.Drop { asked, r = "drop of a pending merge", ix.DropPending(ctx, a.Pending, a.By, a.Why, a.Abandon) a.Module = a.Pending diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go index e50e8423..cb8076b2 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go @@ -7,8 +7,6 @@ import ( "sync/atomic" "time" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" - "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -155,15 +153,6 @@ func heardStepEnded(ctx context.Context, ix *index.Indexer, m jetstream.Msg, log _ = m.Ack() return } - // The end of a validation ask resumes its pending revision (design 47 G §713); its own fault is given again by the bus. - var end buildstep.StepEnded - if err := json.Unmarshal(m.Data(), &end); err == nil { - if err := ix.HearValidationEnded(ctx, end); err != nil { - logf("take-in: the end of the validation ask %s is not taken in, so the bus gives it again: %v", end.Ask, err) - leftForBus(m, "the end of the validation ask "+end.Ask, err, logf) - return - } - } if err := ix.HearInputEnded(ctx, e); err != nil { logf("inputs: the input of the ask %s could not be deleted, so the bus gives it again: %v", e.Ask, err) leftForBus(m, "the step-ended of the ask "+e.Ask, err, logf) diff --git a/modules/mesh-delivery-indexer/internal/index/behind.go b/modules/mesh-delivery-indexer/internal/index/behind.go index 81c02951..d7906ae9 100644 --- a/modules/mesh-delivery-indexer/internal/index/behind.go +++ b/modules/mesh-delivery-indexer/internal/index/behind.go @@ -91,7 +91,7 @@ func (ix *Indexer) TakeInBehind(ctx context.Context) []string { } switch { case held[head]: - // A head the pending holds (a member waiting for its group's end or its build) is taken in by that path, whole with + // A head the pending holds (a member waiting for its group's end) is taken in by that path, whole with // its group: it is not behind (A-463a). ix.clearUntaken(ctx, key, "the head of "+repository+" waits in the indexer's pending") case merge == head: diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index eaa2ed47..1390342c 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -6,7 +6,6 @@ import ( "context" "errors" "fmt" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/stateobjects" "reflect" "slices" @@ -234,12 +233,6 @@ type Indexer struct { Pending Store // SayTakeIn publishes a take-in event (take-in.pending, take-in.taken; ADR 0633 §5); nil says none. SayTakeIn func(ctx context.Context, t takein.TakeIn) error - // ValidationAsks puts the validation's run ask (design 47 G §713, A-247: buildrun.Asker, named as the indexer's asker, - // A-248); nil validates nothing, and a revision is taken in as before. - ValidationAsks ValidationAsks - // AskVerb asks a seat verb as the runtime calls it: the build seat's verb output, for a run's answer the seat keeps as an - // object (design 47 G A-254, buildstep.StepEnded.ReadOutput); nil reads none, and such an answer could not validate. - AskVerb buildstep.AskVerb // Runs is the state runs: the runs on request the indexer holds a candidate for (runcandidates.go, A-174); nil keeps none. Runs Store runMu sync.Mutex // one run event at a time (runcandidates.go) diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_calls_test.go b/modules/mesh-delivery-indexer/internal/index/catalog_calls_test.go new file mode 100644 index 00000000..445ebad6 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/catalog_calls_test.go @@ -0,0 +1,72 @@ +package index + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "sync/atomic" + "testing" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/views" +) + +// listingObjects counts the listings of the bucket: each costs the whole bucket (56 ms at 11 000 objects, more through the runtime). +type listingObjects struct { + *memObjects + listings atomic.Int64 +} + +func (l *listingObjects) Names(ctx context.Context) ([]string, error) { + l.listings.Add(1) + return l.memObjects.Names(ctx) +} + +// A seat verb that answers from the whole view of the trunk (modules, provides, dependents, module) reads the catalog-revisions +// bucket's listing once per call, never once per module: the node-engines ask `module` for every module at once, and 33 asks +// in 5 minutes passed the tool runner's 30 s (2026-11-17). +func TestTrunkViewIsReadOnceForAWholeAnswer(t *testing.T) { + objects := &listingObjects{memObjects: newMemObjects(0)} + const modules = 40 + var refs []meshcatalog.EntryRef + for m := 0; m < modules; m++ { + name := fmt.Sprintf("mod%02d", m) + digest := fmt.Sprintf("%064x", m) + e := meshcatalog.CatalogEntry{Module: name, Repository: "o/r", Path: "m/" + name, Commit: "c", Fp: "src2:" + strings.Repeat("a", 64), + Manifest: map[string]any{"name": name}, ManifestDigest: "sha256:" + digest} + b, _ := json.Marshal(e) + if err := objects.Put(context.Background(), EntryObject(digest), b); err != nil { + t.Fatal(err) + } + refs = append(refs, meshcatalog.EntryRef{Module: name, Digest: digest}) + } + b, _ := json.Marshal(meshcatalog.CatalogRevision{Kind: "trunk", Revision: 1, Entries: refs}) + if err := objects.Put(context.Background(), "trunk.1", b); err != nil { + t.Fatal(err) + } + ts := TrunkStates{Objects: objects, Under: func(string) views.Source { return nil }, Kept: &KeptTrunk{}} + for _, state := range []string{"modules", "graph"} { + objects.listings.Store(0) + values, err := SourceValues(ts.State(state)) + if err != nil || len(values) != modules { + t.Fatalf("the %s of the view = %d values (%v), want %d", state, len(values), err, modules) + } + if n := objects.listings.Load(); n > 2 { + t.Errorf("a whole read of the %s listed the bucket %d times, want at most 2 (once per call, not once per module)", state, n) + } + } + if _, err := meshcatalogclient.DecodeModule(mustValue(t, ts)); err != nil { + t.Errorf("a value of the view does not decode: %v", err) + } +} + +func mustValue(t *testing.T, ts TrunkStates) []byte { + t.Helper() + v, err := SourceValues(ts.State("modules")) + if err != nil { + t.Fatal(err) + } + return v["mod00"] +} diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index 12f15438..7a104b44 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -691,6 +691,57 @@ func (r revisionSource) Get(key string) (*views.Entry, bool, error) { return &views.Entry{Value: value, Revision: uint64(at)}, true, nil } +// Values is every value of the modules (or the module graph) of the newest trunk revision from one read of it: a seat verb +// that answers from the whole view asks which revision is newest once, never once per key (each ask lists the whole +// catalog-revisions bucket: 6 s for the 110 modules of a bucket of 11 000 objects, and 35 s with 12 callers at once). +func (r revisionSource) Values() (map[string][]byte, error) { + s, _, err := r.t.read(r.nodes) + if err != nil { + return nil, err + } + ctx := context.Background() + keys, err := s.Keys(ctx) + if err != nil { + return nil, err + } + out := make(map[string][]byte, len(keys)) + for _, k := range keys { + value, _, found, err := s.Get(ctx, k) + if err != nil { + return nil, err + } + if found { + out[k] = value + } + } + return out, nil +} + +// SourceValues is every key's value of a source: in one read where the source offers it (the view of the trunk, a +// watched copy of a state), else a get per key. +func SourceValues(src views.Source) (map[string][]byte, error) { + if all, ok := src.(interface { + Values() (map[string][]byte, error) + }); ok { + return all.Values() + } + keys, err := src.Keys() + if err != nil { + return nil, err + } + out := make(map[string][]byte, len(keys)) + for _, k := range keys { + e, ok, err := src.Get(k) + if err != nil { + return nil, fmt.Errorf("%s: %w", k, err) + } + if ok && e != nil { + out[k] = e.Value + } + } + return out, nil +} + // Page is the values in key order after a key from one read of the revision, as many as fit maxBytes and at least one // (views.Pager): a first view's page reads the revision once, never once per key. func (r revisionSource) Page(after string, maxBytes int, _ time.Duration) ([]views.KeyedEntry, string, error) { diff --git a/modules/mesh-delivery-indexer/internal/index/fingerprints.go b/modules/mesh-delivery-indexer/internal/index/fingerprints.go index a275cffa..14a2be8f 100644 --- a/modules/mesh-delivery-indexer/internal/index/fingerprints.go +++ b/modules/mesh-delivery-indexer/internal/index/fingerprints.go @@ -60,7 +60,7 @@ func (ix *Indexer) FingerprintsAgain(ctx context.Context, module string) TakenIn ix.freshMu.Unlock() // The path of a merge's recompute: the modules, then the candidates their fingerprints reach (recomputeWith), then the // pending merges a changed fingerprint settles (PendingAgain, as after a toolchain report). - err = ix.recomputeWith(ctx, nil, nil) + err = ix.recomputeWith(ctx, nil, nil, "") ix.freshMu.Lock() ix.fresh = nil ix.freshMu.Unlock() diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index 6aacb8ab..8f2ed205 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -144,7 +144,7 @@ func (ix *Indexer) applyMoving(ctx context.Context, f forgemsg.Forge, drop bool, ix.logf("%s at %s: its packages are not published: %v", f.Repository, f.Merge, err) } // What this merge moved reaches the modules standing on it (a base's tree, build-agent's fingerprint), in the graph's order. - return w, ix.recomputeWith(ctx, overlay, w) + return w, ix.recomputeWith(ctx, overlay, w, f.Repository) } // baseFP answers the new-form fingerprint of a base module as modules. holds it, or why it has none. diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index 6a373b1f..fe930595 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -13,7 +13,6 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" @@ -23,13 +22,10 @@ import ( "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-indexer/judge" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph" ) -// The take-in step (novox/hq ADR 0633): a source revision is taken in only by the indexer it names. A merged message is -// kept in the indexer's own state pending, one key per merge, until the take-in that ends it; a revision naming another -// build of the indexer writes only what building and moving that indexer needs, and the rest stays pending for it. +// The take-in step (novox/hq ADR 0633, A-476): a merged message is kept in the indexer's own state pending, one key per +// merge, until the take-in that ends it; a revision is taken in by the running indexer whatever build of the indexer it names. const ( // PendingState is the indexer's state of held merges: the bucket mesh-delivery-indexer_pending, keyed // takein.PendingKey, so the key order is the merge order. An old `state` entry (the indexer is foundation, ADR @@ -48,10 +44,6 @@ var errClaimLost = errors.New("another take-in claimed the merges first, so this // runs such a build, so the revision waits until a later merge gives it one or a hand take-in (indexer_take_in_pending). const NoBuild = "no build: " -// ErrOwnBuildUnknown is what a merged message answers while the indexer cannot learn the build it runs (ADR 0633 §1): -// the message is left unacknowledged, so the bus delivers it again, and nothing is written. -var ErrOwnBuildUnknown = errors.New("the build of the indexer this program runs is not known") - // PendingRef is what a pending value keeps of its merged message instead of a copy (a merge of a repository with many // modules is about 0.5 MB, over the state's 256 KiB value cap; issue 1585): the small fields the waits are worked out // from. The message itself is read from the forge bridge's stream by Pending.Seq when a take-in needs it. @@ -98,8 +90,6 @@ type Pending struct { // Held is why a ready merge is not taken in although its revision names this build: objects full (HeldObjectsFull), // until room is said (RoomAgain). Held string `json:"held,omitempty"` - // Validation is the revision's validation (design 47 G §713, A-247), on the revision's newest value alone. - Validation *takein.Validation `json:"validation,omitempty"` } // PendingEntry is one key of the pending state as read: its key, revision, value and the merged message decoded. For a @@ -389,26 +379,13 @@ func (ix *Indexer) pendingOnBus(ctx context.Context) bool { } // ownBuildNow is the build this program runs, as FixOwnBuild fixed it at the start: empty while unknown, and never asked again. +// It names only the writer of what is written (LastWriter); nothing waits for it (A-476 b). func (ix *Indexer) ownBuildNow(context.Context) string { ix.ownMu.Lock() defer ix.ownMu.Unlock() return ix.ownFP } -// sayOwnBuildUnknown says, for a merge this process cannot take in because it does not know its own build (ADR 0633 §1), a -// take-in.pending that names why and since when, so the watchman's take-in-waits bound covers it. The merge is not written to -// pending; the stream delivers it again to the process that knows its build. -func (ix *Indexer) sayOwnBuildUnknown(ctx context.Context, seq uint64, f forgemsg.Forge) { - ix.ownMu.Lock() - since, why := ix.ownSince, ix.ownWhy - ix.ownMu.Unlock() - if since.IsZero() { // never fixed: the start did not run (no engine wired) - since, why = ix.now(), "the build this program runs was never asked" - } - ix.sayTakeIn(ctx, takein.TakeIn{Kind: takein.TakeInPending, Key: takein.PendingKey(seq), What: takein.TakeInRevision, - Merges: []takein.Merge{{Repository: f.Repository, Number: f.Number, Merge: f.Merge}}, Since: since, Unknown: why}) -} - // sayTakeIn publishes a take-in event and keeps what it said on the entries, so indexer_pending reads the last event of // each wait from the state. A publish that fails is said in the log: the next evaluation says it again. func (ix *Indexer) sayTakeIn(ctx context.Context, t takein.TakeIn) bool { @@ -456,10 +433,6 @@ func (ix *Indexer) Arrive(ctx context.Context, seq uint64, f forgemsg.Forge) (Ap return a, "", err } own := ix.ownBuildNow(ctx) - if own == "" { - ix.sayOwnBuildUnknown(ctx, seq, f) - return Applied{}, "", ErrOwnBuildUnknown - } ix.pendingMu.Lock() defer ix.pendingMu.Unlock() key := takein.PendingKey(seq) @@ -502,9 +475,6 @@ func (ix *Indexer) EndGroup(ctx context.Context, end forgemsg.ForgeGroupEnd) (Ap return Applied{}, "no pending state: the group's merges were taken in one by one", nil } own := ix.ownBuildNow(ctx) - if own == "" { - return Applied{}, "", ErrOwnBuildUnknown - } ix.pendingMu.Lock() defer ix.pendingMu.Unlock() entries, err := ReadPending(ctx, ix.Pending, ix.logf) @@ -677,21 +647,15 @@ func (ix *Indexer) takenAlready(ctx context.Context, m forgemsg.GroupMerge) (boo } // TakeInAtStart is the take-in at the start (ADR 0633 §2), before the consumer on the forge bridge's stream starts: -// it evaluates pending once with the build this program runs, taking in the whole revisions that name this build; a -// group still being gathered stays, and a revision naming yet another build is held for that one. A claim left by an -// indexer that ran before is void here. When the build is not known (the node-engine did not answer), it waits for -// nothing: it says so and the consumer starts all the same; a merged message is then left unacknowledged until the -// build is known (clause 1), and its redelivery, or a toolchain report (ToolchainReported), asks again. No timer. +// it evaluates pending once, taking in the whole revisions whatever build of the indexer they name (A-476 b); a group +// still being gathered stays. A claim left by an indexer that ran before is void here. The build this program runs, known +// or not, only names the writer. No timer. func (ix *Indexer) TakeInAtStart(ctx context.Context) (Applied, string, error) { if ix.Pending == nil || !ix.pendingOnBus(ctx) { // Nothing can wait in a state not on the bus: the start asks no build and does not wait for one. return Applied{}, "the pending state is not on the bus yet: nothing waits", nil } own := ix.ownBuildNow(ctx) - if own == "" { - ix.logf("take-in: the build this program runs is not known at its start, so nothing pending is taken in now; the forge consumer starts, and a merged message left unacknowledged asks it again") - return Applied{}, "the build this program runs is not known yet: nothing pending is taken in at the start", nil - } ix.pendingMu.Lock() defer ix.pendingMu.Unlock() a, words, err := ix.evaluate(ctx, own, true) @@ -699,8 +663,8 @@ func (ix *Indexer) TakeInAtStart(ctx context.Context) (Applied, string, error) { return a, "the pending state is not on the bus yet: nothing waits", nil } if err == nil && len(a.Written) == 0 { - // A start that takes nothing in says so, with the build it learnt (the wait's own build is in the line holdFor says). - ix.logf("take-in: at the start, this program runs build %s and took nothing in: %s", own, words) + // A start that takes nothing in says so. + ix.logf("take-in: at the start, this program (build %q) took nothing in: %s", own, words) } return a, words, err } @@ -731,16 +695,12 @@ func (ix *Indexer) UnreadAgain(ctx context.Context) error { } // PendingAgain evaluates pending again after something a fingerprint is computed from changed, a toolchain report -// (ADR 0633 §1, §3), or the verdicts state bound (issue 1966): a revision that now names the running build is taken in, and one that names another build is held -// for that one. It does nothing while the pending state is not on the bus or the build this program runs is not known. +// (ADR 0633 §3), or the verdicts state bound (issue 1966): what waits is taken in. It does nothing while the pending state is not on the bus. func (ix *Indexer) PendingAgain(ctx context.Context) error { if ix.Pending == nil || !ix.pendingOnBus(ctx) { return nil } own := ix.ownBuildNow(ctx) - if own == "" { - return nil - } ix.pendingMu.Lock() defer ix.pendingMu.Unlock() _, words, err := ix.evaluate(ctx, own, false) @@ -786,31 +746,37 @@ func (ix *Indexer) evaluate(ctx context.Context, own string, start bool) (Applie if len(lost) > 0 { return Applied{}, ix.waitingLine(w), ix.holdLost(ctx, w.Ready, lost, reasons, unread) } - if wait, words, err := ix.validationGate(ctx, ready, own); wait || err != nil { - return Applied{}, words, err - } - names, dry, err := ix.computeOver(ctx, ready) + dry, err := ix.computeOver(ctx, ready) if err != nil { return Applied{}, "", err } if words, refused, err := ix.refuseCycle(ctx, ready, dry); refused || err != nil { return Applied{}, words, err } - if names == own { - a, err := ix.takeInWhole(ctx, ready, own, "") - if errors.Is(err, errClaimLost) { - return a, "claimed by another take-in", nil - } - if errors.Is(err, errVerdictsNotYet) { - return a, "the verdicts state is not on the bus yet: the revision waits for it, taken in when it binds", nil - } - if errors.Is(err, ErrObjectsFull) { - ix.holdFull(ctx, ready) - return a, ix.waitingLine(w) + "; held: " + HeldObjectsFull, nil - } - return a, ix.waitingLine(w), err + // The indexer takes in with the build it runs, whatever build of the indexer the revision names (A-476 b). + a, err := ix.takeInWhole(ctx, ready, own, "") + if errors.Is(err, errClaimLost) { + return a, "claimed by another take-in", nil } - return Applied{}, ix.waitingLine(w), ix.holdFor(ctx, ready, names, own, dry) + if errors.Is(err, errVerdictsNotYet) { + return a, "the verdicts state is not on the bus yet: the revision waits for it, taken in when it binds", nil + } + if errors.Is(err, ErrObjectsFull) { + ix.holdFull(ctx, ready) + return a, ix.waitingLine(w) + "; held: " + HeldObjectsFull, nil + } + return a, ix.waitingLine(w), err +} + +// newestOfRevision is the index in ready of the revision's newest merge. +func newestOfRevision(ready []PendingEntry) int { + n := 0 + for i, e := range ready { + if e.Value.Seq > ready[n].Value.Seq { + n = i + } + } + return n } // refuseCycle refuses the revision whose catalog graph would hold a cycle with a build edge the take-in refuses (design 47 G @@ -951,7 +917,6 @@ func (ix *Indexer) dry() *dryRun { x := &Indexer{Module: ix.Module, Manifests: ix.Manifests, Packages: ix.Packages, JudgeID: ix.JudgeID, Files: ix.Files, GoSum: ix.GoSum, GoMod: ix.GoMod, Now: ix.Now, Builds: ix.Builds, Named: ix.Named, Trunk: ix.Trunk, BranchHead: ix.BranchHead, Read: ix.Read, Contains: ix.Contains, ReadOpen: nil, OwnBuild: nil, Message: ix.Message, SayRefused: nil, SayAccepted: nil, SayTakeIn: nil, Pending: nil, - ValidationAsks: nil, AskVerb: nil, // a dry run asks no validation (§713) // The registry milestone's fields: what a take-in would write on the bus stays out of a dry run, so the states of the // requirements, findings, seat definitions, holds and their bounds are not overlaid, nothing is said, and no hold is // weighed (Holds and Counted nil hold nothing); ReadFile only reads. @@ -991,16 +956,15 @@ func (ix *Indexer) dry() *dryRun { } // computeOver applies the ready merges, in merge order, to a dry indexer, as one overlay: each later merge's trees -// govern (ADR 0633 §3). It answers the fingerprint the indexer's own module has once they are applied (none is "", -// with its reason), and the dry run, whose overlays hold what the take-in would write. A merge that can never be read +// govern (ADR 0633 §3). It answers the dry run, whose overlays hold what the take-in would write. A merge that can never be read // is left out, as its take-in will drop it. -func (ix *Indexer) computeOver(ctx context.Context, ready []PendingEntry) (string, *dryRun, error) { +func (ix *Indexer) computeOver(ctx context.Context, ready []PendingEntry) (*dryRun, error) { d := ix.dry() // The manifests are judged against the seat definitions the revision carries (seatsof.go, ADR 0633 §1), so the indexer // it names is not refused for a verb or an event the built-in list lacks. seats, err := ix.revisionSeats(ctx, ready) if err != nil { - return "", d, fmt.Errorf("reading the seat definitions of %s: %w", ready[0].Key, err) + return d, fmt.Errorf("reading the seat definitions of %s: %w", ready[0].Key, err) } var applyErr error judge.WithSeats(seats, func() { @@ -1018,24 +982,9 @@ func (ix *Indexer) computeOver(ctx context.Context, ready []PendingEntry) (strin } }) if applyErr != nil { - return "", d, applyErr + return d, applyErr } - key, _ := meshcatalogclient.ModulesKey(ix.Module) - value, _, found, err := d.ix.Modules.Get(ctx, key) - if err != nil { - return "", d, err - } - if !found { - return NoBuild + "the indexer holds no record of itself", d, nil - } - m, err := meshcatalogclient.DecodeModule(value) - if err != nil { - return "", d, nil - } - if m.Fp2 == "" { - return NoBuild + m.None2, d, nil - } - return m.Fp2, d, nil + return d, nil } // claim marks the entries as taken in by this take-in, by compare-and-set on the revisions read. A lost one (another @@ -1238,8 +1187,7 @@ func (ix *Indexer) nextSource(branch string) int64 { // takenWaits are the taken events of a take-in: every wait the entries were said under (saidWaits), every wait // indexer_pending names the claimed entries by now (WaitsOf: a revision by its oldest merge's key, a group by its g. key) and // the key of every claimed entry, since indexer_pending named it by that key earlier when it waited behind a group. So a wait -// the watchman learned of without the indexer marking it (seeded from indexer_pending, or said by the own-build-unknown -// path) ends too (design 47, Taken in by the indexer it names). No key is said twice. +// the watchman learned of without the indexer marking it (seeded from indexer_pending) ends too. No key is said twice. func takenWaits(entries []PendingEntry) []takein.TakeIn { out := saidWaits(entries) said := map[string]bool{} @@ -1298,83 +1246,6 @@ func saidWaits(entries []PendingEntry) []takein.TakeIn { return out } -// holdFor keeps the ready merges pending for the indexer build they name and writes only what building and moving -// that indexer needs (ADR 0633 §1): the indexer's own module entry, its key of the module graph and the verdict on its -// manifest, from the dry run; the manifests and the packages are written by the dry run itself. The revision is said -// pending under its oldest merge's key, and a wait it was said under before by another key is said taken. -func (ix *Indexer) holdFor(ctx context.Context, ready []PendingEntry, names, own string, d *dryRun) error { - key, _ := meshcatalogclient.ModulesKey(ix.Module) - if err := copyKey(ctx, d.modules, ix.Modules, key); err != nil { - return fmt.Errorf("the indexer's own module entry: %w", err) - } - if gk, err := graph.Key(ix.Module); err == nil { - if err := copyKey(ctx, d.graph, ix.Graph, gk); err != nil && !errors.Is(err, process.ErrNotYet) { - return fmt.Errorf("the indexer's key of the module graph: %w", err) - } - } - // Why the manifest of the indexer the revision names was refused, when it was (judged with the seat definitions the - // revision carries): said with the wait, as a merge the forge bridge could not give is (holdLost). - refused := "" - if value, _, found, err := d.ix.Modules.Get(ctx, key); err == nil && found { - if m, err := meshcatalogclient.DecodeModule(value); err == nil && m.Manifest != "" { - if vk, err := meshcatalogclient.VerdictsKey(m.Manifest); err == nil { - if err := copyKey(ctx, d.verdicts, ix.Verdicts, vk); err != nil && !errors.Is(err, process.ErrNotYet) { - return fmt.Errorf("the verdict on the indexer's manifest: %w", err) - } - if vv, _, ok, err := d.ix.Verdicts.Get(ctx, vk); err == nil && ok { - if v, err := meshcatalogclient.DecodeVerdict(vv); err == nil && meshcatalogclient.VerdictRefuses(v) { - refused = strings.Join(v.Problems, "; ") - } - } - } - } - } - // The indexer's own entry the held revision names is kept as the trunk revision after the newest, every other entry as - // it was (milestone 47/G W4.7: a merged revision naming a newer indexer replaces only the indexer's entry). - if err := ix.KeepView(ctx); err != nil { - return fmt.Errorf("the revision with the indexer's own entry: %w", err) - } - t := revisionWait(takein.TakeInPending, ready, names) - t.Refused = refused - var ended, unmarked []PendingEntry - said, changed := false, false - for _, e := range ready { - switch { - case e.Value.Said == t.Key && e.Value.SaidKind == takein.TakeInPending: - said = true - changed = changed || e.Value.WaitsFor != names || e.Value.Refused != refused - if e.Value.WaitsFor != names || e.Value.Refused != refused { - unmarked = append(unmarked, e) - } - case e.Value.Said != "" && e.Value.SaidKind == takein.TakeInPending: - ended = append(ended, e) - unmarked = append(unmarked, e) - default: - unmarked = append(unmarked, e) - } - } - for _, old := range saidWaits(ended) { - ix.sayTakeIn(ctx, old) - } - if said && !changed { - // Nothing to say on the bus, but the start decided nothing and a reader of the journal sees why: the build learnt, the build the wait names. - ix.logf("take-in: %d merges, oldest %s, still name the indexer build %s, this program runs %s: they wait, as already said", len(ready), t.Key, names, own) - } - if !said || changed { - ix.logf("take-in: %d merges, oldest %s, name the indexer build %s, not %s, which runs: they wait for it", len(ready), t.Key, names, own) - if refused != "" { - ix.logf("take-in: the manifest of the indexer they name is refused, judged with the seat definitions they carry, so no such indexer runs: %s", refused) - } - if !ix.sayTakeIn(ctx, t) { - ix.markSaid(ctx, unmarked, nil, func(p *Pending) { p.WaitsFor, p.LastWriter, p.Refused = names, own, refused }) - return nil - } - unmarked = ready - } - ix.markSaid(ctx, unmarked, &t, func(p *Pending) { p.WaitsFor, p.LastWriter, p.Refused = names, own, refused }) - return nil -} - // copyKey writes what the overlay holds of key to the store, when it wrote or deleted it. func copyKey(ctx context.Context, from *overlayStore, to Store, key string) error { if from == nil || to == nil { @@ -1439,9 +1310,6 @@ func waitOf(entries []PendingEntry, what, behind string) takein.PendingWait { if e.Value.Refused != "" { pw.Refused = e.Value.Refused } - if e.Value.Validation != nil { - pw.Validation = e.Value.Validation - } if e.Value.Said != "" && e.Value.SaidAt.After(pw.SaidAt) { pw.Said, pw.SaidAt = "take-in."+e.Value.SaidKind, e.Value.SaidAt } @@ -1450,9 +1318,9 @@ func waitOf(entries []PendingEntry, what, behind string) takein.PendingWait { return pw } -// TakeInPendingByHand is the hand take-in (indexer_take_in_pending, ADR 0633 §5): the wait named by its key, the -// ready revision (its m. key) or a group gathered so far (its g. key), is taken in with this indexer and its judge, -// whatever build it names, for when the named indexer can never run or the group's end never comes. by and why are +// TakeInPendingByHand is the hand take-in (indexer_take_in_pending, ADR 0633 §5, A-476, principle 4): the pending revision (its +// m. key) or the group gathered so far (its g. key) is taken in now with this indexer and its judge, +// for a revision whose message is unread or lost or held for room, or a group whose end never comes. by and why are // said in the journal and on the take-in.taken event. What waited behind it is evaluated again after. func (ix *Indexer) TakeInPendingByHand(ctx context.Context, key, by, why string) TakenIn { r := TakenIn{Module: key} diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index 3d001ffe..07792014 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -106,6 +106,16 @@ func (b *takeInBed) merge(number int, c, avahi, indexer string) forgemsg.Forge { return f } +// arriveHeld arrives f while the verdicts state is not on the bus, so the take-in holds it (ADR 0668 §4): the way a merge is +// left pending now that no merge waits for the indexer build it names (A-476 b). +func (b *takeInBed) arriveHeld(t *testing.T, seq uint64, f forgemsg.Forge) { + t.Helper() + bound := b.ix.Verdicts + b.ix.Verdicts = &memStore{unbound: process.ErrNotYet} + defer func() { b.ix.Verdicts = bound }() + b.arrive(t, seq, f) +} + func (b *takeInBed) arrive(t *testing.T, seq uint64, f forgemsg.Forge) { t.Helper() b.publish(t, f) @@ -176,63 +186,21 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) }) } }) - t.Run("T6 a merge while the own build is unknown writes nothing", func(t *testing.T) { + t.Run("T6 a merge while the own build is unknown is taken in all the same (A-476 b)", func(t *testing.T) { b := newTakeInBed(t) b.ix.ownFP, b.ix.ownFixed = "", false // this process is started again and asks the engine, which does not answer b.ix.OwnBuild = func(context.Context) (string, error) { return "", errors.New("the engine does not answer") } b.ix.FixOwnBuild(ctx) b.said = nil - _, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", "")) - if !errors.Is(err, ErrOwnBuildUnknown) || len(b.keys()) != 0 || b.commit(t, "avahi") == sha('b') { - t.Errorf("err %v, pending %v, avahi at %s: want ErrOwnBuildUnknown and nothing written", err, b.keys(), b.commit(t, "avahi")) + if _, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", "tree-indexer-2")); err != nil || len(b.keys()) != 0 || b.commit(t, "avahi") != sha('b') { + t.Errorf("err %v, pending %v, avahi at %s: want the merge taken in at once", err, b.keys(), b.commit(t, "avahi")) } - // The merge is said pending with why and since when (so the watchman's bound covers it), and indexer_pending says the - // build is unknown, from the own-build state. - if len(b.said) != 1 || b.said[0].Kind != takein.TakeInPending || b.said[0].Key != takein.PendingKey(1) || !strings.Contains(b.said[0].Unknown, "does not answer") || - b.said[0].Since.IsZero() || b.said[0].WaitsFor != "" { - t.Errorf("said %+v, want one take-in.pending of m.1 saying why the own build is unknown", b.said) - } else if _, err := takein.EncodeTakeIn(b.said[0]); err != nil { - t.Errorf("the unknown-build event is refused by its own contract: %v", err) + if len(b.said) != 1 || b.said[0].Kind != takein.TakeInTaken { + t.Errorf("said %+v, want one take-in.taken and no wait said", b.said) } answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{}) - if a, _ := answer.(takein.PendingAnswer); err != nil || !strings.Contains(a.OwnBuildUnknown, "does not answer") || a.OwnBuildSince.IsZero() { - t.Errorf("indexer_pending = %+v %v, want the build unknown with why and since", answer, err) - } - // Before the pending state is on the bus nothing waits: the start asks no build, and the merge is taken in at once, - // as before ADR 0633, though the build is unknown (TestIndexerWhole found the start waiting on it for ever). - b.ix.Pending = &memStore{unbound: process.ErrNotYet} - if _, _, err := b.ix.TakeInAtStart(ctx); err != nil { - t.Fatal(err) - } - if _, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", "")); err != nil || b.commit(t, "avahi") != sha('b') { - t.Errorf("no pending state: err %v, avahi at %s: want the merge taken in at once", err, b.commit(t, "avahi")) - } - }) - t.Run("1966/AC195-2 AC195-3 a merge said pending with its build unknown and taken in on redelivery ends that wait under its key; a revision wait is said once", func(t *testing.T) { - b := newTakeInBed(t) - ownBuild := b.ix.OwnBuild - b.ix.ownFP, b.ix.ownFixed = "", false - b.ix.OwnBuild = func(context.Context) (string, error) { return "", errors.New("the engine does not answer") } - b.ix.FixOwnBuild(ctx) - b.said = nil - if _, _, err := b.ix.Arrive(ctx, 7, b.merge(30, sha('b'), "tree-avahi-2", "")); !errors.Is(err, ErrOwnBuildUnknown) { - t.Fatal(err) - } - b.ix.OwnBuild, b.ix.ownFP, b.ix.ownFixed = ownBuild, "", false - b.ix.FixOwnBuild(ctx) - b.arrive(t, 7, b.merge(30, sha('b'), "tree-avahi-2", "")) // the stream delivers it again - if want := []string{"pending " + takein.PendingKey(7), "taken " + takein.PendingKey(7)}; !slices.Equal(b.events(), want) || len(b.keys()) != 0 { - t.Fatalf("said %v, pending %v: want %v and nothing pending", b.events(), b.keys(), want) - } - // Two merges said pending under the oldest's key (waiting for a named indexer), then taken in together: one taken, under - // that key, and no entry's key said twice. - b.said = nil - b.arrive(t, 8, b.merge(31, sha('c'), "tree-avahi-3", "tree-indexer-2")) - b.arrive(t, 9, b.merge(32, sha('d'), "tree-avahi-4", "tree-indexer-2")) - b.said = nil - b.arrive(t, 10, b.merge(33, sha('e'), "tree-avahi-5", b.base.Trees["modules/"+testSelf])) - if want := []string{"taken " + takein.PendingKey(8), "taken " + takein.PendingKey(9), "taken " + takein.PendingKey(10)}; !slices.Equal(b.events(), want) || len(b.keys()) != 0 { - t.Errorf("said %v, pending %v: want %v, the wait's key (m.8, said pending) once and each other entry's own", b.events(), b.keys(), want) + if a, _ := answer.(takein.PendingAnswer); err != nil || a.OwnBuild != "" || a.OwnBuildSince.IsZero() || len(a.Waits) != 0 { + t.Errorf("indexer_pending = %+v %v, want no wait and no known build", answer, err) } }) t.Run("T7 and T3 a merge alone, an indexer change, a merge behind it and a revert", func(t *testing.T) { @@ -259,42 +227,28 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) t.Fatalf("alone: avahi %s (was %s), pending %v, said %v: want taken in, said taken under its key (1966/AC195-1)", avahi1, avahi0, b.keys(), b.events()) } b.said = nil - avahi1, verdicts := b.fp(t, "avahi"), len(b.ix.Verdicts.(*memStore).vals) - // T4: a change to the indexer writes only its entry (with its graph key and verdict); the rest is pending. + avahi1 := b.fp(t, "avahi") + // A-476 b: a merge naming another indexer build is taken in at once, with the build this program runs: the indexer's entry + // and avahi are written, nothing is held and said pending. b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) named := b.fp(t, testSelf) - if named == own || b.fp(t, "avahi") != avahi1 || b.commit(t, "avahi") != sha('b') { - t.Fatalf("indexer change: indexer %s (own %s), avahi %s at %s: want only the indexer written", named, own, b.fp(t, "avahi"), b.commit(t, "avahi")) + if named == own || b.fp(t, "avahi") == avahi1 || b.commit(t, "avahi") != sha('c') || len(b.keys()) != 0 { + t.Fatalf("indexer change: indexer %s (own %s), avahi %s at %s, pending %v: want the whole revision taken in at once", named, own, b.fp(t, "avahi"), b.commit(t, "avahi"), b.keys()) } if g := graphOf(t, b.ix, testSelf); g.FP != named { t.Errorf("graph of the indexer names %s, want %s", g.FP, named) } - if n := len(b.ix.Verdicts.(*memStore).vals); n > verdicts+1 { - t.Errorf("verdicts %d, want at most %d: only the new indexer manifest's", n, verdicts+1) + if want := []string{"taken " + takein.PendingKey(2)}; !slices.Equal(b.events(), want) { + t.Errorf("said %v, want %v: nothing said pending", b.events(), want) } - if want := []string{takein.PendingKey(2)}; !reflect.DeepEqual(b.keys(), want) || !slices.Equal(b.events(), []string{"pending " + takein.PendingKey(2)}) || b.said[0].WaitsFor != named { - t.Fatalf("pending %v, said %+v: want m.2 pending for %s", b.keys(), b.said, named) - } - // T5: the merge delivered again writes no second key and says nothing again. - b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) - if len(b.keys()) != 1 || len(b.said) != 1 { - t.Errorf("redelivered: pending %v, said %v", b.keys(), b.events()) - } - // A later merge joins: the newest governs and still names the new indexer. + // A later merge is taken in at once too, and a revert to the first indexer. b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")) - if len(b.keys()) != 2 || b.commit(t, "avahi") != sha('b') || len(b.said) != 1 { - t.Fatalf("behind: pending %v, avahi at %s, said %v: want both pending and nothing said again", b.keys(), b.commit(t, "avahi"), b.events()) - } - // A revert brings the indexer back to the running build: everything is taken in, oldest first. b.arrive(t, 4, b.merge(23, sha('e'), "tree-avahi-5", b.base.Trees["modules/"+testSelf])) if len(b.keys()) != 0 || b.fp(t, testSelf) != own || b.commit(t, "avahi") != sha('e') { t.Fatalf("revert: pending %v, indexer %s, avahi at %s: want all taken in", b.keys(), b.fp(t, testSelf), b.commit(t, "avahi")) } - if want := []string{"pending " + takein.PendingKey(2), "taken " + takein.PendingKey(2), "taken " + takein.PendingKey(3), "taken " + takein.PendingKey(4)}; !slices.Equal(b.events(), want) { - t.Errorf("said %v, want %v", b.events(), want) - } }) - t.Run("IX22 a merged revision naming a newer indexer is the previous catalog graph revision with only the indexer's entry replaced", func(t *testing.T) { // design 47 G §273, §237 + t.Run("IX22 a merged revision naming a newer indexer is a catalog graph revision with its entry and the merge's others replaced", func(t *testing.T) { // design 47 G §273, §237 b := newTakeInBed(t) store := newMemObjects(0) b.ix.Revisions = store @@ -305,14 +259,14 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) if err != nil || before.Revision != 1 { t.Fatalf("the view's revision %+v (%v), want revision 1", before, err) } - // The merge changes avahi and the indexer: only the indexer is written, the revision is made at once. + // The merge changes avahi and the indexer: both are taken in at once (A-476 b), in a revision after the first. b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) after, _, err := newestTrunk(ctx, store) - if err != nil || after.Revision != 2 { - t.Fatalf("after the merge naming a newer indexer the newest revision is %+v (%v), want revision 2", after, err) + if err != nil || after.Revision <= before.Revision { + t.Fatalf("after the merge naming a newer indexer the newest revision is %+v (%v), want one after revision %d", after, err, before.Revision) } - if changed := changedEntries(before.Entries, after.Entries); !slices.Equal(changed, []string{testSelf}) { - t.Errorf("the revision differs from the previous in %v, want the indexer's entry alone", changed) + if changed := changedEntries(before.Entries, after.Entries); !slices.Equal(changed, []string{"avahi", testSelf}) && !slices.Equal(changed, []string{testSelf, "avahi"}) { + t.Errorf("the revision differs from the previous in %v, want avahi and the indexer", changed) } }) t.Run("P1 a revision that adds a Go module its own indexer uses is computed against that module's packages at its merge", func(t *testing.T) { @@ -353,71 +307,13 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) t.Fatal(err) } b.arrive(t, 2, f) - if len(b.said) != 1 || !strings.HasPrefix(b.said[0].WaitsFor, "src2:") { - t.Errorf("said %+v, want m.2 waiting for the indexer build its merge's packages give, not for a version no take-in has published", b.said) + if len(b.said) != 1 || b.said[0].Kind != takein.TakeInTaken || b.said[0].WaitsFor != "" || len(b.keys()) != 0 { + t.Errorf("said %+v, pending %v, want m.2 taken in at once", b.said, b.keys()) } if _, ok := b.ix.publishedAll(ctx)[sibling]; !ok { t.Errorf("the packages the revision publishes at its merge lack %s", sibling) } }) - t.Run("K1 the indexer a revision names is judged with the seat definitions the revision carries", func(t *testing.T) { - // A successor's manifest invokes a verb of the node-engine's seat that the seat list lacks and the revision's - // definitions add (ADR 0633 §1, §6): refused without them, with its reason said on the wait and in indexer_pending; - // accepted with them, and the revision waits for the build it names. - const verb, contracts = "revision-verb", "modules/seat-contracts" - // The third row's definer is the Delivery domain's contracts module defining a Core seat: not taken (ADR 0636 §5). - for _, row := range []struct { - name string - defined bool - refused bool - module string - }{{"definitions the revision does not carry", false, true, ""}, {"definitions the revision carries", true, false, "seat-contracts"}, - {"definitions of a seat outside the carrier's domain", true, true, "mesh-delivery-contracts"}} { - b := newTakeInBed(t) - f := b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2") - f.Manifests = map[string]string{} - for dir, raw := range b.base.Manifests { - f.Manifests[dir] = raw - } - dir := "modules/" + testSelf - f.Manifests[dir] = strings.Replace(f.Manifests[dir], `"module": "`+testSelf+`"`, - `"module": "`+testSelf+`", "invokes": ["seat:node-engine.`+verb+`"]`, 1) - if row.defined { - f.Trees[contracts] = "tree-seat-contracts" - f.Locks = map[string]map[string]string{} - for d, l := range b.base.Locks { - f.Locks[d] = l - } - f.Locks[contracts] = map[string]string{} - f.Manifests[contracts] = `{"module":"` + row.module + `","version":"1.0","build":{"artifacts":[{"name":"go-seats","kind":"package","language":"go","paths":["go"],"series":"0.1","generated-from":"."}]}}` - b.ix.Files = func(_ context.Context, repository, commit string, dirs []string) (map[string]map[string]string, error) { - if repository != f.Repository || commit != f.Merge || !slices.Equal(dirs, []string{contracts + "/seats"}) { - t.Errorf("listed %v of %s at %s", dirs, repository, commit) - } - return map[string]map[string]string{dirs[0]: {"node-engine/seat.json": "1", "node-engine/fixtures/a.json": "2"}}, nil - } - b.ix.ReadFile = func(_ context.Context, _, _, p string) ([]byte, error) { - switch p { - case contracts + "/seats/node-engine/seat.json": - return []byte(`{"format":1,"seat":"node-engine","scope":"node","grain":"seat-list","decision":"d","domain":"Core","verbs":[{"name":"` + verb + `"}]}`), nil - } - return nil, nil - } - } - b.arrive(t, 2, f) - waits := PendingWaits(readPending(t, b)) - refused := "" - if len(b.said) > 0 { - refused = b.said[0].Refused - } - if got := strings.Contains(refused, `serves no verb "`+verb+`"`); got != row.refused || (len(waits) == 1 && waits[0].Refused != refused) { - t.Errorf("%s: said %q, indexer_pending %+v: want a refusal naming the verb %v", row.name, refused, waits, row.refused) - } - if _, ok := catalogue.SeatNamed("node-engine"); !ok || judgeSeatsHave("node-engine", verb) { - t.Errorf("%s: the seats are not put back after the dry run", row.name) - } - } - }) t.Run("A-326 a revision whose catalog graph holds a cycle with a build edge no build seat breaks is refused, naming it", func(t *testing.T) { // avahi's artifact needs the indexer's (kind 2), and the indexer is built on avahi (kind 3): avahi holds no build seat, so // no running version breaks the cycle, and the revision is refused at take-in naming every artifact on it (design 47 G §49). @@ -443,15 +339,15 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) t.Errorf("indexer_pending %+v, want the refusal naming the cycle", waits) } }) - t.Run("T10 the named indexer takes the revision in at its start; a gathered group stays", func(t *testing.T) { + t.Run("T10 a start takes what waits in whatever its build; a gathered group stays", func(t *testing.T) { b := newTakeInBed(t) - b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) + b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) g := b.merge(30, sha('f'), "", "") g.Repository, g.Trees, g.Manifests, g.Locks, g.Group = "novox/mesh-sdk", map[string]string{}, nil, nil, "feat/x" g.Members = []meshdeliveryclient.Head{{Repository: "novox/mesh-sdk", Number: 30, Head: g.Head}, {Repository: "novox/mesh-host", Number: 4, Head: sha('9')}} - b.arrive(t, 5, g) - named := b.fp(t, testSelf) - // The new indexer starts; its build is not known (the engine does not answer at its start): a hand ask is answered + b.arriveHeld(t, 5, g) + own, _ := b.ix.OwnBuild(ctx) + // The indexer starts; its build is not known (the engine does not answer at its start): a hand ask is answered // meanwhile (R5), and the start waits for nothing. It is never asked again, so it stays unknown until it is started again. mk := func(ask func(context.Context) (string, error)) *Indexer { n := &Indexer{Module: testSelf, Store: b.ix.Store, Modules: b.ix.Modules, Manifests: b.ix.Manifests, Graph: b.ix.Graph, Verdicts: b.ix.Verdicts, @@ -462,39 +358,21 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) n.FixOwnBuild(ctx) return n } - // A start that decides nothing says so (issue 1921): the build it learnt, the build the held wait names, also when the - // wait was said before; and indexer_pending names the known build and since when. - oldBuild, _ := b.ix.OwnBuild(ctx) - oldStart := mk(func(context.Context) (string, error) { return oldBuild, nil }) - b.logs.Reset() - if _, words, err := oldStart.TakeInAtStart(ctx); err != nil || len(b.keys()) != 2 || !strings.Contains(b.logs.String(), "still name the indexer build") || - !strings.Contains(b.logs.String(), "at the start, this program runs build") { - t.Fatalf("a start that holds: %q %v, journal %q: want the build learnt and the build the wait names said", words, err, b.logs.String()) - } - if answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{}); err != nil || answer.(takein.PendingAnswer).OwnBuild != oldBuild || answer.(takein.PendingAnswer).OwnBuildSince.IsZero() { - t.Errorf("indexer_pending = %+v %v, want the known build and since", answer, err) - } - asked := 0 - next := mk(func(context.Context) (string, error) { asked++; return "", errors.New("not yet") }) + // The build this program runs is not known (the engine does not answer at its start): nothing waits for it, so the start + // takes the revision in all the same (A-476 b), and a hand ask names what waits. + next := mk(func(context.Context) (string, error) { return "", errors.New("not yet") }) if r := next.TakeInPendingByHand(ctx, "m.nothing", "lead@shanks", "a test"); r.Stored || !strings.Contains(r.Answer, "nothing waits") { - t.Errorf("a hand ask while the own build is unknown: %+v", r) + t.Errorf("a hand ask for what does not wait: %+v", r) } - if _, words, err := next.TakeInAtStart(ctx); err != nil || len(b.keys()) != 2 || !strings.Contains(words, "not known") { - t.Fatalf("a start with its build unknown: %q %v, pending %v: want it to answer at once and take nothing in", words, err, b.keys()) - } - // The engine answers now, and the same process does not ask it again: it learnt nothing it did not know at its start. - next.OwnBuild = func(context.Context) (string, error) { asked++; return named, nil } - if _, words, err := next.TakeInAtStart(ctx); err != nil || asked != 1 || len(b.keys()) != 2 || !strings.Contains(words, "not known") { - t.Fatalf("the window of a replacement: %q %v, asked %d, pending %v: want the build fixed at the start, never learnt later", words, err, asked, b.keys()) - } - // The module started again fixes the build at its start and takes the revision in. - next = mk(func(context.Context) (string, error) { return named, nil }) if _, _, err := next.TakeInAtStart(ctx); err != nil { t.Fatal(err) } if !reflect.DeepEqual(b.keys(), []string{takein.PendingKey(5)}) || b.commit(t, "avahi") != sha('c') { t.Errorf("start: pending %v, avahi at %s: want the revision taken in, the group left", b.keys(), b.commit(t, "avahi")) } + if answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{}); err != nil || answer.(takein.PendingAnswer).OwnBuild != own { + t.Errorf("indexer_pending = %+v %v, want the build the original program knows", answer, err) + } }) t.Run("T8 T9 T11 a group gathered until its end, a merge behind it, one alone, an end again", func(t *testing.T) { b := newTakeInBed(t) @@ -544,7 +422,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) if _, words, err := b.ix.EndGroup(ctx, end); err != nil || !strings.Contains(words, "taken in already") { t.Errorf("an end again: %q %v", words, err) } - // T11 (the lead's row): a group whose revision changes the indexer: the group's wait is taken, then the revision pending. + // T11: a group whose revision changes the indexer: the group's wait is taken and the revision taken in at once (A-476 b). b.said = nil g := member(catalogueRepo, 40, sha('c')) g.Group, g.Trees = "feat/two", b.merge(0, "", "", "tree-indexer-7").Trees @@ -553,7 +431,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) if _, _, err := b.ix.EndGroup(ctx, forgemsg.ForgeGroupEnd{Group: "feat/two", Members: []forgemsg.GroupMerge{{Repository: catalogueRepo, Number: 40, Merge: sha('c')}}}); err != nil { t.Fatal(err) } - if want := []string{"pending g.feat/two", "taken g.feat/two", "pending " + takein.PendingKey(9)}; !slices.Equal(b.events(), want) { + if want := []string{"pending g.feat/two", "taken g.feat/two", "taken " + takein.PendingKey(9)}; !slices.Equal(b.events(), want) { t.Errorf("a group naming another indexer: said %v, want %v", b.events(), want) } }) @@ -593,11 +471,8 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) } held := func() *takeInBed { b := newTakeInBed(t) - b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) - b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")) - b.ix.ownMu.Lock() - b.ix.ownFP = b.fp(t, testSelf) - b.ix.ownMu.Unlock() + b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) + b.arriveHeld(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")) return b } // A probe that fails keeps the group's wait and stops nothing else: an unrelated merge is still taken in. @@ -719,7 +594,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) }) t.Run("T5 two take-ins of one pending revision: the one that loses the claim writes nothing", func(t *testing.T) { b := newTakeInBed(t) - b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) + b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) named := b.fp(t, testSelf) b.ix.ownMu.Lock() b.ix.ownFP = named @@ -774,8 +649,8 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) {"a take-in that fails", func(b *takeInBed) { b.mods.failWrite = once(func(string) bool { return true }) }}, } { b := newTakeInBed(t) - b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) - b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")) + b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) + b.arriveHeld(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")) named := b.fp(t, testSelf) b.ix.ownMu.Lock() b.ix.ownFP = named @@ -794,7 +669,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) }) t.Run("W4.6 a full store holds the merge with the reason objects full, says full once, and room takes it in", func(t *testing.T) { b := newTakeInBed(t) - b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) + b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) named := b.fp(t, testSelf) b.ix.ownMu.Lock() b.ix.ownFP = named @@ -826,7 +701,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) // The same take-in into a store it fills to 85 % of its cap says filling once, and is taken in. kept := store.used() b2 := newTakeInBed(t) - b2.arrive(t, 2, b2.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) + b2.arriveHeld(t, 2, b2.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) b2.ix.ownMu.Lock() b2.ix.ownFP = b2.fp(t, testSelf) b2.ix.ownMu.Unlock() @@ -845,7 +720,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) t.Run("W4.12 a candidate's half dropped by hand frees its own entries, says room and takes the held merge in; the read tools list it; a recompute that moves nothing is equal", func(t *testing.T) { bed := func(store *memObjects) *takeInBed { b := newTakeInBed(t) - b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) + b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) b.ix.ownMu.Lock() b.ix.ownFP = b.fp(t, testSelf) b.ix.ownMu.Unlock() @@ -930,40 +805,10 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) t.Errorf("indexer_recompute = %+v (%v): want equal to the kept revision 2", got, err) } }) - t.Run("T7 a toolchain report evaluates pending again: the revision then names the running build and is taken in", func(t *testing.T) { - b := newTakeInBed(t) - b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) - entries, err := ReadPending(ctx, b.pending, nil) - if err != nil { - t.Fatal(err) - } - // The build the revision names under the next toolchain, worked out on a dry indexer; this one runs it. - x := b.ix.dry().ix - for _, node := range []string{"novox", "ace"} { - key, _ := ToolchainKey(node, "go") - x.PutToolchain("mesh-build-go", key, toolchainValue(t, "go", "go1.28.0", "1"), false) - } - ready, _, _, _ := b.ix.loadNewest(ctx, WaitsOf(entries).Ready) - next, _, err := x.computeOver(ctx, ready) - if err != nil || next == "" || next == b.said[0].WaitsFor { - t.Fatalf("the build under the next toolchain: %q %v (the revision names %s)", next, err, b.said[0].WaitsFor) - } - b.ix.ownMu.Lock() - b.ix.ownFP = next - b.ix.ownMu.Unlock() - reportToolchain(t, b.ix, "mesh-build-go", "novox", "go", "go1.28.0") - if len(b.keys()) != 1 { - t.Fatalf("one node's report: pending %v, want the revision still held (the nodes disagree)", b.keys()) - } - reportToolchain(t, b.ix, "mesh-build-go", "ace", "go", "go1.28.0") - if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('c') { - t.Errorf("after the toolchain report: pending %v, avahi at %s: want the revision taken in", b.keys(), b.commit(t, "avahi")) - } - }) t.Run("T12 T13 indexer_pending names the wait; a hand take-in and a drop say what they ended", func(t *testing.T) { b := newTakeInBed(t) - b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) - b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")) + b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) + b.arriveHeld(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")) var answer takein.PendingAnswer for _, tool := range PendingTools(b.pending, nil, nil, nil) { if tool.Name == PendingTool { @@ -975,7 +820,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) } } if len(answer.Waits) != 1 || answer.Waits[0].Key != takein.PendingKey(2) || !slices.Equal(answer.Waits[0].Holds, []string{testSelf, "avahi"}) || - answer.Waits[0].Said != takein.TakeInPendingEvent || len(answer.Waits[0].Keys) != 2 { + len(answer.Waits[0].Keys) != 2 { t.Fatalf("indexer_pending = %+v", answer) } // ADR 0668 §6: a later merge covers a pending one only when its commit has the pending merge's as an ancestor. @@ -990,32 +835,47 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) } history = "cd" b.said = nil + bound := b.ix.Verdicts + b.ix.Verdicts = &memStore{unbound: process.ErrNotYet} // m.3 stays held while m.2 is dropped if r := b.ix.DropPending(ctx, takein.PendingKey(2), "lead@shanks", "replaced by m.3", false); !r.Stored { t.Fatalf("drop: %+v", r) } - if len(b.said) < 2 || !b.said[0].Dropped || b.said[0].Key != takein.PendingKey(2) || b.said[0].By != "lead@shanks" || b.said[1].Key != takein.PendingKey(3) { - t.Fatalf("drop said %+v: want m.2 taken (dropped, by) and m.3 pending", b.said) + b.ix.Verdicts = bound + if len(b.keys()) != 1 || b.keys()[0] != takein.PendingKey(3) { + t.Fatalf("pending %v: want m.2 dropped and m.3 pending", b.keys()) } // Issue 1966 (#650), ADR 0633 §5: the newest pending merge of its repository, which no taken merge covers, is not - // dropped; the answer names the read again. abandon drops it all the same (below). + // dropped; the answer names abandon, which drops it all the same (below). if r := b.ix.DropPending(ctx, takein.PendingKey(3), "lead@shanks", "lost message", false); r.Stored || - !strings.Contains(r.Answer, TakeInPendingTool) || len(b.keys()) != 1 { + !strings.Contains(r.Answer, "abandon") || len(b.keys()) != 1 { t.Fatalf("a drop nothing covers: %+v, pending %v", r, b.keys()) } + // A pending revision is taken in by the tool, by its key (A-476, principle 4). + if _, _, err := NewPendingAsk(takein.PendingKey(3), "x", "lead@shanks", false, false, time.Unix(0, 0)); err != nil { + t.Fatalf("the ask to take a pending revision in: %v", err) + } + if r := b.ix.TakeInPendingByHand(ctx, takein.PendingKey(3), "lead@shanks", "its message was unread"); !r.Stored || len(b.keys()) != 0 || b.commit(t, "avahi") != sha('d') { + t.Fatalf("hand take-in of a revision: %+v, pending %v, avahi at %s", r, b.keys(), b.commit(t, "avahi")) + } + // A delivery group gathered is taken in by hand (A-476 b: only a group is). b.ix.Pins = newMem() b.ix.Revisions = newMemObjects(0) // A-448: the taken event names the catalog graph revision the take-in made + gm := b.merge(24, sha('f'), "tree-avahi-6", "") + gm.Group, gm.Head = "feat/h", sha('1') + gm.Members = []meshdeliveryclient.Head{{Repository: catalogueRepo, Number: 24, Head: sha('1')}, {Repository: "novox/mesh-sdk", Number: 25, Head: sha('2')}} + b.arrive(t, 5, gm) b.said = nil - if r := b.ix.TakeInPendingByHand(ctx, takein.PendingKey(3), "lead@shanks", "the named indexer can never run"); !r.Stored { + if r := b.ix.TakeInPendingByHand(ctx, takein.GroupTakeInKey("feat/h"), "lead@shanks", "the group's end never comes"); !r.Stored { t.Fatalf("hand take-in: %+v", r) } - if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('d') || len(b.said) != 1 || b.said[0].Kind != takein.TakeInTaken || b.said[0].By != "lead@shanks" { + if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('f') || len(b.said) < 1 || b.said[0].Kind != takein.TakeInTaken || b.said[0].By != "lead@shanks" { t.Errorf("hand take-in: pending %v, avahi at %s, said %+v", b.keys(), b.commit(t, "avahi"), b.said) } // §476, §493: a hand take-in's taken event carries the branch, the source revision it makes and its entries. - if len(b.said) == 1 { + if len(b.said) >= 1 { got := b.said[0] - if got.Branch != TrunkBranch || got.Source <= 0 || len(got.Entries) != 1 || got.Entries[0].Module != "avahi" || - got.Entries[0].Commit != sha('d') || got.Entries[0].Fingerprint == "" { + if got.Branch != TrunkBranch || got.Source <= 0 || len(got.Entries) != 2 || got.Entries[1].Module != "avahi" || + got.Entries[1].Commit != sha('f') || got.Entries[1].Fingerprint == "" { t.Errorf("hand take-in said branch %q source %d entries %+v: want main, a source revision and avahi at its commit", got.Branch, got.Source, got.Entries) } else if newest, any, _ := newestTrunk(ctx, b.ix.Revisions); !any || got.Revision != newest.Revision { t.Errorf("hand take-in said revision %d, want the revision it made, %d (A-448)", got.Revision, newest.Revision) @@ -1025,7 +885,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) t.Errorf("the taken event is refused by its own contract: %v", err) } } - b.arrive(t, 4, b.merge(23, sha('e'), "tree-avahi-5", "tree-indexer-2")) + b.arriveHeld(t, 4, b.merge(23, sha('e'), "tree-avahi-5", "tree-indexer-2")) if r := b.ix.DropPending(ctx, takein.PendingKey(4), "lead@shanks", "abandoned on the operator's word", true); !r.Stored || len(b.keys()) != 0 { t.Errorf("an abandoned drop: %+v, pending %v", r, b.keys()) } @@ -1062,8 +922,8 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) if _, _, found, _ := b.ix.Store.Get(ctx, key); !found { t.Fatal("the group has no candidate") } - b.arrive(t, 2, big(b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))) - b.arrive(t, 3, big(b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))) + b.arriveHeld(t, 2, big(b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))) + b.arriveHeld(t, 3, big(b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))) raw, _, _, _ := b.pending.Get(ctx, takein.PendingKey(2)) if len(b.keys()) != 2 || len(raw) > 4096 || b.commit(t, "avahi") != sha('b') { t.Fatalf("a big merge held: pending %v, value %d bytes, avahi at %s: want two small values", b.keys(), len(raw), b.commit(t, "avahi")) @@ -1139,7 +999,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) b := newTakeInBed(t) f := b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2") f.Words, f.Branch = strings.Repeat("x", 300*1024), "feat/g" - b.arrive(t, 2, f) // held by reference, for an indexer build that is not this one + b.arriveHeld(t, 2, f) // held by reference, with the verdicts state not on the bus b.ix.Taken = newMem() b.ix.Contains = func(context.Context, string, string, string) (bool, error) { return c.behind, nil } data, _ := takein.EncodeTaken(takein.Taken{Repository: trunk.RepositoryName(catalogueRepo), Merge: sha('d')}) diff --git a/modules/mesh-delivery-indexer/internal/index/pending_tools.go b/modules/mesh-delivery-indexer/internal/index/pending_tools.go index c27abaa7..ba52e019 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_tools.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_tools.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "strings" "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" @@ -44,15 +43,13 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T return []stdio.Tool{ {Name: PendingTool, Description: "What waits for a take-in (ADR 0633): merges the indexer holds in its pending state instead of taking them in. " + - "A revision (what revision, its key its oldest merge's m.) waits for the build of the indexer it names " + - "(waits-for): the running indexer wrote only what building and moving that indexer needs, and the new indexer takes " + - "it in at its start. A group (what group, key g.) is a delivery group whose members are gathered until the " + + "A revision (what revision, its key its oldest merge's m.) waits only while its message is unread or lost or the " + + "objects are full. A group (what group, key g.) is a delivery group whose members are gathered until the " + "forge bridge says its end; a merge in one of its members' repositories waits behind it (behind). Each names its " + "merges, since when it waits, the modules it holds back (holds), the merges whose message the forge bridge's stream no " + "longer holds (lost: they are not taken in; indexer_drop_pending drops them) and the last take-in event said of it. Empty waits: " + "every merge is taken in. The watchman raises take-in-waits past 60 minutes for a revision, 30 for a group; " + - "indexer_take_in_pending and indexer_drop_pending steer a wait. own-build says which build the indexer knows it runs and own-build-since since when; own-build-unknown, when set, says the indexer cannot learn the build it " + - "runs (since when): it fixes it once at its start, so every merge waits (a take-in.pending that says so) until the module is started again.", + "indexer_take_in_pending and indexer_drop_pending steer a wait. own-build says which build the indexer knows it runs and own-build-since since when; it only names the writer, and no merge waits for it.", Input: map[string]any{"type": "object", "properties": map[string]any{}}, Run: func(map[string]any) (any, error) { var unread []string @@ -63,7 +60,7 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T return nil, err } answer := takein.PendingAnswer{Waits: PendingWaits(entries), Unread: unread} - // Whether this indexer knows the build it runs: when it does not, every merge waits, and this says since when and why. + // The build this indexer knows it runs, and since when. if ownBuild != nil { if v, _, found, err := ownBuild.Get(context.Background(), OwnBuildKey); err == nil && found { var o OwnBuild @@ -71,8 +68,6 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T answer.OwnBuildSince = o.Since if o.Known { answer.OwnBuild = o.FP - } else { - answer.OwnBuildUnknown = o.Why } } } @@ -80,10 +75,10 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T return answer, nil }}, {Name: TakeInPendingTool, - Description: "Take a wait in by hand with the running indexer and its judge (ADR 0633 §5): the revision under its m. key, " + - "whatever build of the indexer it names, or a delivery group under its g. key with the members gathered so far. For " + - "when the named indexer can never run, or a group's end never comes. A hand act: who and why are said in the " + - "indexer's journal and on the take-in.taken event, which clears take-in-waits. indexer_pending names the keys.", + Description: "Take in now a pending revision (its m. key), for a merge whose message is unread or lost or that is held " + + "for room, or a delivery group (its g. key) with the members gathered so far, for when its end never comes (ADR 0633 §5). " + + "A hand act: who and why are said in the indexer's journal and on the take-in.taken event, which clears take-in-waits. " + + "indexer_pending names the keys.", Input: map[string]any{"type": "object", "required": []string{"key", "why"}, "properties": map[string]any{ "key": map[string]any{"type": "string", "description": "the wait's key, m. or g., as indexer_pending names it"}, "why": map[string]any{"type": "string", "description": "why it is taken in by hand, in plain words"}}}, @@ -112,33 +107,3 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T Run: merge}, } } - -// ValidationTools is indexer_validate_again (class act, design 47 G §713): the tools bundle writes an ask in takeins and the -// indexer asks the revision's validation again; record says the hand act in the hand-act log before it is asked. -func ValidationTools(ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time, record func(what, why string) error) []stdio.Tool { - return []stdio.Tool{{Name: ValidateAgainTool, - Description: "Ask again the validation of a pending revision that could not validate (design 47 G §712, §713): the manifest " + - "validator's run on the build seat node-build-go ended could not run, exited other than 0, or answered not both cases. " + - "The revision is asked under its next try; the answer says the validation's state (the ask, its try, accepted, refused, " + - "or why it could not validate). A revision accepted, refused or whose ask is out is asked nothing: a refusal is asked " + - "again only by a later merge of its branch. A hand act, recorded through mesh-hand-acts before it is asked. " + - "indexer_pending names the keys and shows the validation whole.", - Input: map[string]any{"type": "object", "required": []string{"key", "why"}, "properties": map[string]any{ - "key": map[string]any{"type": "string", "description": "the revision's wait key, m., as indexer_pending names it"}, - "why": map[string]any{"type": "string", "description": "why it is asked again, in plain words"}}}, - Run: func(args map[string]any) (any, error) { - str := func(k string) string { s, _ := args[k].(string); return s } - key, value, err := NewValidateAgainAsk(str("key"), str("why"), stdio.CallerOf(args), now()) - if err != nil { - return nil, err - } - if err := record("asked the validation of the pending revision "+strings.TrimSpace(str("key"))+" again by hand", strings.TrimSpace(str("why"))); err != nil { - return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was asked: %w", err) - } - a, err := ask(key, value) - if err != nil { - return nil, err - } - return a.Result, nil - }}} -} diff --git a/modules/mesh-delivery-indexer/internal/index/takein.go b/modules/mesh-delivery-indexer/internal/index/takein.go index ff3f4d88..ae21ccc0 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein.go +++ b/modules/mesh-delivery-indexer/internal/index/takein.go @@ -51,9 +51,6 @@ type TakeInAsk struct { // Abandon, with Drop, drops a pending merge no later merge covers (ADR 0633 §5 asks a later merge; issue 1966, ADR // 0668 §5): the hand act says why it is abandoned all the same. Abandon bool `json:"abandon,omitempty"` - // ValidateAgain, with Pending, asks the validation of the revision under that key again (indexer_validate_again, design 47 - // G §713): one that could not validate. - ValidateAgain bool `json:"validate-again,omitempty"` // Merge asks the merge of MergeRepository at this commit to be taken in by hand (indexer_take_in_merge): its merged message // is asked of the forge bridge and run through the take-in as a delivered one, under its stream sequence Sequence. Merge string `json:"merge,omitempty"` @@ -167,20 +164,6 @@ func NewMergeAsk(repository, merge string, sequence uint64, why, by string, at t return "merge." + merge, data, err } -// NewValidateAgainAsk is the value indexer_validate_again writes: its key in takeins, `pending.validate-again.`, and the -// value. key is a revision's wait key, m., as indexer_pending names it. -func NewValidateAgainAsk(key, why, by string, at time.Time) (string, []byte, error) { - key = strings.TrimSpace(key) - if !strings.HasPrefix(key, "m.") { - return "", nil, errors.New("name the revision whose validation is asked again by its key, m., as indexer_pending names it") - } - if strings.TrimSpace(why) == "" { - return "", nil, errors.New("say why (why): a hand act is recorded through mesh-hand-acts") - } - data, err := json.Marshal(TakeInAsk{V: 1, Pending: key, ValidateAgain: true, Why: strings.TrimSpace(why), By: by, At: at.UTC()}) - return "pending.validate-again." + key, data, err -} - // NewPendingAsk is the value indexer_take_in_pending (drop false) or indexer_drop_pending (drop true) writes: its key // in takeins, `pending..`, and the value. key is a wait's key as indexer_pending names it, for a // drop one pending merge's; abandon drops one no later merge covers. diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index 87c225c7..c3c1927d 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -13,6 +13,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" "git.novox.be/novox/mesh-sdk/go/catalogue" "mesh-delivery-indexer/judge" @@ -127,24 +128,26 @@ type known struct { // the order the bucket lists them in; modules that stand on each other have a cycle as their reason. A module is written // only when its fingerprint or its reason changes, and only if its manifest is still the one computed from. A module with // no kept inputs (a message that carried no manifest) is left as it is. -func (ix *Indexer) Recompute(ctx context.Context) error { return ix.recomputeWith(ctx, nil, nil) } +func (ix *Indexer) Recompute(ctx context.Context) error { return ix.recomputeWith(ctx, nil, nil, "") } // recomputeWith is Recompute for a merge as well: the records the merge gives (overlay, by module key) stand in place // of what the bucket holds, are computed with the others, and are written once, with their final fingerprint or reason // (issue 1444: a merge writes each module once). A record of the overlay that has no kept inputs is written as it is. w is -// the merge's write (order.go), nil for none; it is told each module written. -func (ix *Indexer) recomputeWith(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite) error { +// the merge's write (order.go), nil for none; it is told each module written. A merge's recompute names its repository +// (A-476 c, A-476a): the modules registered for it are moved, and those of another repository only when they import a package it moved; "" moves every module (a toolchain +// report, a hand recompute). +func (ix *Indexer) recomputeWith(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite, repository string) error { if err := ix.fillPaths(ctx); err != nil { return err } - if err := ix.recomputeModules(ctx, overlay, w); err != nil { + if err := ix.recomputeModules(ctx, overlay, w, repository); err != nil { return err } // What the modules moved to reaches the candidates' fingerprints too (a base, the builder, a toolchain). return ix.RecomputeCandidates(ctx) } -func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite) error { +func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite, repository string) error { if ix.Modules == nil { return nil } @@ -153,11 +156,11 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]mesh } ix.recompute.Lock() defer ix.recompute.Unlock() - return ix.recomputeLocked(ctx, overlay, w) + return ix.recomputeLocked(ctx, overlay, w, repository) } // recomputeLocked is recomputeModules with ix.recompute held by the caller (a hold that moves, holds.go). -func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite) error { +func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite, repository string) error { if ix.Manifests == nil { return ix.writeOverlay(ctx, overlay, nil, w) } @@ -298,6 +301,14 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc r.tcs, r.pkgs = toolchainsUsed(toolchains), f.versions } } + if _, merged := overlay[module]; !merged && repository != "" && !trunk.SameRepository(k.cur.Repository, repository) && + maps.Equal(k.held.Packages, map[string]string(r.pkgs)) { + // A module registered for another repository is moved by this repository's commit only when it imports a package the + // commit moved (A-476a, ADR 0593 §3); any other stands as held (A-476 c), and so it is for a base of this repository's. + r = result{fp: k.held.Fp2, none: k.held.None2, tcs: k.held.Toolchains, pkgs: sdkstep.PackageVersions(k.held.Packages)} + done[module] = r + return r + } done[module] = r k.cur.Inputs2 = &kept return r diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index 21804c57..e5bb68e1 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -832,6 +832,8 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) { if err := ix.Apply(context.Background(), f); err != nil { t.Fatal(err) } + // A-476a: a commit of mesh-sdk moves avahi, of another repository, which imports its package, and leaves mesh-tools, + // which imports nothing it moved. a, _ := moduleOf(t, mods, "avahi") if a.Fp2 == "" || !reflect.DeepEqual(a.Packages, map[string]string{sdk: commit}) { t.Fatalf("after mesh-sdk's merge %.8s: fp2 %q none2 %q packages %v", commit, a.Fp2, a.None2, a.Packages) @@ -841,12 +843,24 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) { } return a } + // A-476 c: mesh-tools, of another repository than mesh-sdk, holds a fingerprint a recompute would move (here a stale one); a + // commit of mesh-sdk, which moved nothing it imports, leaves it as held, and only a full recompute moves it. + stale := tools + stale.Fp2 = "src2:" + strings.Repeat("5", 64) + mods.put("mesh-tools", mustEncode(t, stale)) + tools = stale first := sdkMerge(strings.Repeat("1", 40), "tree-sdk-1") second := sdkMerge(strings.Repeat("2", 40), "tree-sdk-2") if second.Fp2 == first.Fp2 { t.Errorf("a mesh-sdk trunk merge kept avahi's fingerprint %q", first.Fp2) } - n = second + if err := ix.Recompute(context.Background()); err != nil { + t.Fatal(err) + } + if o, _ := moduleOf(t, mods, "mesh-tools"); o.Fp2 == stale.Fp2 { + t.Errorf("a full recompute left mesh-tools at %q: the check above proves nothing", o.Fp2) + } + n, _ = moduleOf(t, mods, "avahi") // moved by the recompute too: its base mesh-tools moved // Nodes that disagree keep the fingerprint already computed, with its toolchains (issue 1940, 2026-10-25 20:56: a // stale report blanked every module and the controller's take-in refused modules already built); only a new // fingerprint needs the agreement: a merge that moves the module's tree gives it none, naming the disagreement. diff --git a/modules/mesh-delivery-indexer/internal/index/validate.go b/modules/mesh-delivery-indexer/internal/index/validate.go deleted file mode 100644 index 7bb23f69..00000000 --- a/modules/mesh-delivery-indexer/internal/index/validate.go +++ /dev/null @@ -1,264 +0,0 @@ -package index - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "maps" - "slices" - "strings" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" - "git.novox.be/novox/mesh-sdk/go/catalogue" - - "mesh-delivery-indexer/judge" -) - -// The validation before take-in (design 47 G §712, A-213, A-232): for a pending revision of a watched branch the running -// indexer asks the seat node-build-go a `run` of its own manifest validator over two cases, change and trunk, and refuses only -// what the revision causes (ADR 0579 §2). The ask rides the check queue under the indexer's name, its kind `run` in the body. -const ( - // validationProgram is the manifest validator's program, by its directory under the indexer entry's Go module: the indexer's - // own code, so no repository or module name is fixed here (ADR 0666 §5). - validationProgram = "cmd/mesh-judge" - // ValidationAsker is the asker the ask is put under (buildrun.Asker.Name, A-248): the indexer's seat name, mesh-catalog - // (design 47 G A-163, A-232). - ValidationAsker = meshcatalog.Seat - // validationCatalogSeat is the seat the indexer entry claims. - validationCatalogSeat = "mesh-catalog" - // The cases of the request, as the manifest validator names them. - validationChange = "change" - validationTrunk = judge.TrunkCase -) - -// validationInputVerb is where the build seat's holder asks the input: the seat mesh-catalog's verb input (§354). -var validationInputVerb = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbInput - -// validationRequest is the manifest validator's request for the ready merges of branch over the branch's current revision -// (§712): case change, every module of the revision by its entry's manifest, each one the merges carry replaced by the merged -// message's bytes (the newest merge governs) and one added for a new directory; case trunk, every module as it stands; the -// seat definitions the merges' trees carry; the packages and the predefined list as the take-in's own validator has them; -// no machines. It answers the indexer entry of the change case too, the one that claims the seat mesh-catalog. -func (ix *Indexer) validationRequest(ctx context.Context, branch string, ready []PendingEntry) (composer.Request, meshcatalog.CatalogEntry, error) { - var indexer meshcatalog.CatalogEntry - if ix.Revisions == nil { - return composer.Request{}, indexer, errors.New("the catalog graph is not kept here: no revision to validate over") - } - rev, found, err := newestOf(ctx, ix.Revisions, branch) - if err != nil { - return composer.Request{}, indexer, err - } - var entries []meshcatalog.CatalogEntry - if found { - if _, entries, err = ReadRevision(ctx, ix.Revisions, RevisionObject(rev)); err != nil { - return composer.Request{}, indexer, err - } - } - trunkCase := composer.Case{Name: validationTrunk, Manifests: map[string]composer.Manifest{}} - byPlace := map[string]string{} // -> module - of := map[string]meshcatalog.CatalogEntry{} - for _, e := range entries { - raw := ix.manifestBytes(ctx, e.ManifestDigest) - if raw == "" { - b, err := json.Marshal(e.Manifest) - if err != nil { - return composer.Request{}, indexer, fmt.Errorf("the manifest of %s: %w", e.Module, err) - } - raw = string(b) - } - trunkCase.Manifests[e.Module] = composer.Manifest{Raw: json.RawMessage(raw)} - byPlace[placeKey(e.Repository, e.Path)] = e.Module - of[e.Module] = e - } - change := composer.Case{Name: validationChange, Manifests: maps.Clone(trunkCase.Manifests)} - for _, e := range ready { - if e.Covered { - continue - } - for _, dir := range slices.Sorted(maps.Keys(e.Forge.Manifests)) { - raw := e.Forge.Manifests[dir] - module, known := byPlace[placeKey(e.Forge.Repository, dir)] - if !known { - m, err := judge.ForInputs([]byte(raw)) - if err != nil || m.Module == "" { - module = e.Forge.Repository + "/" + dir // a new directory whose manifest names no module: judged under its place - } else { - module = m.Module - } - byPlace[placeKey(e.Forge.Repository, dir)] = module - of[module] = meshcatalog.CatalogEntry{Module: module, Repository: e.Forge.Repository, Path: dir} - } - change.Manifests[module] = composer.Manifest{Raw: json.RawMessage(raw)} - } - } - for _, module := range slices.Sorted(maps.Keys(change.Manifests)) { - m, err := judge.ForInputs(change.Manifests[module].Raw) - if err != nil || !slices.ContainsFunc(m.Claims, func(c catalogue.Claim) bool { return c.Name == validationCatalogSeat }) { - continue - } - indexer = of[module] - if indexer.Module == "" { - indexer.Module = module - } - break - } - seats, err := ix.revisionDefinitions(ctx, ready) - if err != nil { - return composer.Request{}, indexer, err - } - req := composer.Request{V: composer.Version, Input: json.RawMessage("{}"), Cases: []composer.Case{change, trunkCase}, Seats: seats, - Predefined: ix.predefinedList(ctx)} - if known := ix.knownPackages(ctx); known != nil { - req.Packages = slices.Sorted(maps.Keys(known)) - } - return req, indexer, nil -} - -func placeKey(repository, path string) string { - return strings.ToLower(trunk.RepositoryName(repository)) + " " + path -} - -// revisionDefinitions are the validation's seat definitions, raw (§712 `seats`, A-241): the branch's revision's seat -// definitions with the definitions the merges' trees carry laid over them, so a revision that touches no definition is still -// judged against the whole seat list. A revision's own definitions are not kept yet (A-234), so the revision's are the -// trunk's, read from the state seat-definitions as the verb seat-definitions answers them. -func (ix *Indexer) revisionDefinitions(ctx context.Context, ready []PendingEntry) ([]json.RawMessage, error) { - defs, err := ix.trunkDefinitions(ctx) - if err != nil { - return nil, err - } - merged, err := ix.revisionSeatDefinitions(ctx, ready) - if err != nil { - return nil, err - } - for seat, d := range merged { - raw, err := json.Marshal(d) - if err != nil { - return nil, fmt.Errorf("the definition of %s: %w", seat, err) - } - defs[seat] = raw - } - var out []json.RawMessage - for _, seat := range slices.Sorted(maps.Keys(defs)) { - out = append(out, defs[seat]) - } - return out, nil -} - -// trunkDefinitions are the trunk's seat definitions of the state seat-definitions, raw, by seat: every `.trunk` entry -// that holds a definition; none where the indexer keeps no such state. -func (ix *Indexer) trunkDefinitions(ctx context.Context) (map[string]json.RawMessage, error) { - defs := map[string]json.RawMessage{} - if ix.SeatDefinitions == nil { - return defs, nil - } - keys, err := ix.SeatDefinitions.Keys(ctx) - if err != nil { - return nil, fmt.Errorf("the %s state cannot be listed: %w", SeatDefinitionsState, err) - } - for _, key := range keys { - seat, ok := strings.CutSuffix(key, "."+SourceTrunk) - if !ok || strings.HasPrefix(key, outOfStepPrefix) { - continue - } - v, _, found, err := ix.SeatDefinitions.Get(ctx, key) - if err != nil { - return nil, fmt.Errorf("%s.%s cannot be read: %w", SeatDefinitionsState, key, err) - } - var e SeatDefinitionEntry - if !found || json.Unmarshal(v, &e) != nil || e.Digest == "" || len(e.Definition) == 0 { - continue // an entry that holds no definition is no seat of the list - } - defs[seat] = e.Definition - } - return defs, nil -} - -// validationJob is the run of the manifest validator for the request (§712): the program cmd/mesh-judge, its directory under -// the Go module the go.mod at the entry's path names (read by the build, A-268), at the entry's path, built at head, the revision's head of the entry's repository; no -// heads and no arguments; the input the request's bytes, served by the seat mesh-catalog's verb input. -func validationJob(indexer meshcatalog.CatalogEntry, head, branch string, seq uint64, input []byte) (buildstep.RunJob, error) { - switch { - case indexer.Repository == "": - return buildstep.RunJob{}, fmt.Errorf("no entry of the revision claims the seat %s", validationCatalogSeat) - case head == "": - return buildstep.RunJob{}, fmt.Errorf("the revision names no head of %s, the repository of the indexer entry %s", indexer.Repository, indexer.Module) - } - in, err := buildstep.RunInputOf(validationInputVerb, input) - if err != nil { - return buildstep.RunJob{}, err - } - dir := indexer.Path - if dir == "" { - dir = "." - } - return buildstep.RunJob{ - Program: buildstep.RunProgram{Repository: indexer.Repository, Commit: head, Dir: dir, Package: "./" + validationProgram}, - Input: in, - Reader: fmt.Sprintf("manifest validator of %s at %d", branchName(branch), seq), - }, nil -} - -func branchName(branch string) string { - if isTrunkBranch(branch) { - return TrunkBranch - } - return branch -} - -// ValidationRefusal is what a validation's verdict refuses (ADR 0579 §2, §712): of each manifest the change case refuses that -// the revision adds or changes (its bytes differ from the trunk case's), each problem the trunk case lacks; and each problem -// between manifests the trunk case lacks. A problem only in unchanged manifests refuses nothing. It is empty when the revision -// is accepted. An answer that holds not both cases is no verdict: it answers an error, so the run could not validate. -func ValidationRefusal(req composer.Request, v composer.Verdict) (map[string][]string, error) { - cases := map[string]composer.CaseVerdict{} - for _, c := range v.Cases { - cases[c.Name] = c - } - change, okChange := cases[validationChange] - trunkV, okTrunk := cases[validationTrunk] - if !okChange || !okTrunk { - return nil, fmt.Errorf("the manifest validator's answer holds not both cases %s and %s", validationChange, validationTrunk) - } - raws := map[string]map[string]string{} - for _, c := range req.Cases { - raws[c.Name] = map[string]string{} - for name, m := range c.Manifests { - raws[c.Name][name] = InputDigest(m.Raw) - } - } - refused := map[string][]string{} - for _, module := range slices.Sorted(maps.Keys(change.Manifests)) { - r := change.Manifests[module] - if r.Accepted || r.Refused == "" { - continue - } - before, had := raws[validationTrunk][module] - if module != judge.Unnamed && had && before == raws[validationChange][module] { - continue // unchanged: its problems refuse nothing, and its entry keeps its verdict (ADR 0579 §1) - } - old := map[string]bool{} - for _, p := range strings.Split(trunkV.Manifests[module].Refused, "\n") { - old[p] = true - } - for _, p := range strings.Split(r.Refused, "\n") { - if p != "" && !old[p] { - refused[module] = append(refused[module], p) - } - } - } - return refused, nil -} - -// RefusedWords is a refusal as the pending value's `refused` says it: each manifest and its problems, one manifest a line. -func RefusedWords(refused map[string][]string) string { - var lines []string - for _, module := range slices.Sorted(maps.Keys(refused)) { - lines = append(lines, module+": "+strings.Join(refused[module], "; ")) - } - return strings.Join(lines, "\n") -} diff --git a/modules/mesh-delivery-indexer/internal/index/validate_test.go b/modules/mesh-delivery-indexer/internal/index/validate_test.go deleted file mode 100644 index 73bd29a3..00000000 --- a/modules/mesh-delivery-indexer/internal/index/validate_test.go +++ /dev/null @@ -1,130 +0,0 @@ -package index - -import ( - "context" - "encoding/json" - "reflect" - "strings" - "testing" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer" - - "mesh-delivery-indexer/judge" -) - -// The validation refuses only what the revision causes (design 47 G §712, ADR 0579 §2), and its run names the indexer entry's -// repository, path and the revision's head, and no heads. -func TestValidationRefusesWhatTheRevisionCauses(t *testing.T) { - manifests := func(m map[string]string) map[string]composer.Manifest { - out := map[string]composer.Manifest{} - for k, v := range m { - out[k] = composer.Manifest{Raw: json.RawMessage(v)} - } - return out - } - req := composer.Request{Cases: []composer.Case{ - {Name: "change", Manifests: manifests(map[string]string{"a": `{"module":"a","v":2}`, "b": `{"module":"b"}`})}, - {Name: "trunk", Manifests: manifests(map[string]string{"a": `{"module":"a"}`, "b": `{"module":"b"}`})}, - }} - verdict := func(change, trunk map[string]string) composer.Verdict { - read := func(m map[string]string) map[string]composer.ManifestRead { - out := map[string]composer.ManifestRead{"a": {Accepted: true}, "b": {Accepted: true}} - for k, v := range m { - out[k] = composer.ManifestRead{Refused: v} - } - return out - } - return composer.Verdict{Cases: []composer.CaseVerdict{{Name: "change", Manifests: read(change)}, {Name: "trunk", Manifests: read(trunk)}}} - } - for _, c := range []struct { - name string - v composer.Verdict - want, wantErr string - }{ - {"a changed manifest gains a problem", verdict(map[string]string{"a": "new\nold"}, map[string]string{"a": "old"}), "a: new", ""}, - {"an unchanged manifest's problem refuses nothing", verdict(map[string]string{"b": "bad"}, nil), "", ""}, - {"a problem between manifests the trunk lacks", verdict(map[string]string{judge.Unnamed: "x\ny"}, map[string]string{judge.Unnamed: "y"}), judge.Unnamed + ": x", ""}, - {"an answer with one case is no verdict", composer.Verdict{Cases: verdict(nil, nil).Cases[:1]}, "", "not both cases"}, - } { - got, err := ValidationRefusal(req, c.v) - if c.wantErr != "" { - if err == nil || !strings.Contains(err.Error(), c.wantErr) { - t.Errorf("%s: %v, want %q", c.name, err, c.wantErr) - } - continue - } - if err != nil || RefusedWords(got) != c.want { - t.Errorf("%s: %q, %v, want %q", c.name, RefusedWords(got), err, c.want) - } - } - head := strings.Repeat("a", 40) - entry := meshcatalog.CatalogEntry{Module: "mesh-delivery-indexer", Repository: "novox/mesh-delivery", Path: "modules/mesh-delivery-indexer", - Artifacts: []meshcatalog.Artifact{{Language: "go", Kind: "bundle"}}} - job, err := validationJob(entry, head, "main", 7, []byte(`{"v":1}`)) - if err != nil { - t.Fatal(err) - } - want := buildstep.RunProgram{Repository: "novox/mesh-delivery", Commit: head, Dir: "modules/mesh-delivery-indexer", Package: "./cmd/mesh-judge"} - if job.Program != want || len(job.Heads) != 0 || len(job.Args) != 0 || job.Input.Verb != "seat:mesh-catalog.input" || job.Reader != "manifest validator of main at 7" { - t.Errorf("the job %+v", job) - } - if _, err := validationJob(meshcatalog.CatalogEntry{}, head, "main", 7, nil); err == nil || !strings.Contains(err.Error(), "claims the seat mesh-catalog") { - t.Errorf("no indexer entry: %v", err) - } -} - -// The validation's seats are the branch's revision's seat definitions with the merges' changed ones laid over them (A-241): a -// revision that touches no definition is judged against the whole seat list, and a merge's definition replaces its seat's. -func TestValidationSeatsAreTheRevisionsWithTheMergesOver(t *testing.T) { - ctx := context.Background() - defs := newMem() - for seat, raw := range map[string]string{"node-engine": `{"seat":"node-engine","verbs":[{"name":"old"}]}`, "git": `{"seat":"git"}`} { - v, _ := json.Marshal(SeatDefinitionEntry{Seat: seat, Source: SourceTrunk, Digest: "d-" + seat, Definition: json.RawMessage(raw)}) - defs.put(SeatDefinitionKey(seat, SourceTrunk, "", 0), v) - } - v, _ := json.Marshal(SeatDefinitionEntry{Seat: "pulled", Source: SourcePull, Pull: 3, Digest: "d", Definition: json.RawMessage(`{"seat":"pulled"}`)}) - defs.put(SeatDefinitionKey("pulled", SourcePull, "r", 3), v) - ix := &Indexer{Module: testSelf, SeatDefinitions: defs} - seatsOf := func(raws []json.RawMessage) map[string]string { - out := map[string]string{} - for _, raw := range raws { - var d struct { - Seat string `json:"seat"` - Verbs []struct { - Name string `json:"name"` - } `json:"verbs"` - } - if err := json.Unmarshal(raw, &d); err != nil { - t.Fatal(err) - } - out[d.Seat] = "" - if len(d.Verbs) > 0 { - out[d.Seat] = d.Verbs[0].Name - } - } - return out - } - got, err := ix.revisionDefinitions(ctx, nil) - if err != nil || !reflect.DeepEqual(seatsOf(got), map[string]string{"node-engine": "old", "git": ""}) { - t.Errorf("a revision touching no definition: %v (%v), want the trunk's whole list and no pull request's", seatsOf(got), err) - } - const contracts = "modules/seat-contracts" - f := forgemsg.Forge{Repository: "r", Merge: strings.Repeat("c", 40), Manifests: map[string]string{ - contracts: `{"module":"seat-contracts","version":"1.0","build":{"artifacts":[{"name":"go-seats","kind":"package","language":"go","paths":["go"],"series":"0.1","generated-from":"."}]}}`}} - ix.Files = func(_ context.Context, _, _ string, dirs []string) (map[string]map[string]string, error) { - return map[string]map[string]string{dirs[0]: {"node-engine/seat.json": "1"}}, nil - } - ix.ReadFile = func(_ context.Context, _, _, p string) ([]byte, error) { - if p == contracts+"/seats/node-engine/seat.json" { - return []byte(`{"format":1,"seat":"node-engine","scope":"node","grain":"seat-list","decision":"d","domain":"Core","verbs":[{"name":"new"}]}`), nil - } - return nil, nil - } - got, err = ix.revisionDefinitions(ctx, []PendingEntry{{Forge: f}}) - if err != nil || !reflect.DeepEqual(seatsOf(got), map[string]string{"node-engine": "new", "git": ""}) { - t.Errorf("a merge changing a definition: %v (%v), want its definition over the trunk's and the rest kept", seatsOf(got), err) - } -} diff --git a/modules/mesh-delivery-indexer/internal/index/validation_wait.go b/modules/mesh-delivery-indexer/internal/index/validation_wait.go deleted file mode 100644 index ec052ce4..00000000 --- a/modules/mesh-delivery-indexer/internal/index/validation_wait.go +++ /dev/null @@ -1,329 +0,0 @@ -package index - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "strings" - "time" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" -) - -// The pending revision's wait for its validation (design 47 G §713, A-232, A-247): the take-in puts the run ask of §712, writes -// `validation` on the revision's newest pending value and ends there; the build seat's step-ended of that ask, heard through the -// indexer's own durable consumer, resumes it: accepted takes the revision in as before, refused keeps it pending with `refused`, -// and an end that could not validate keeps it pending with `failed`, asked again by a later merge of the branch or by hand. - -// ValidationAsks puts a run ask on the build seat (buildrun.Asker). -type ValidationAsks interface { - Put(ctx context.Context, a buildstep.StepAsk) error -} - -// newestOfRevision is the index in ready of the revision's newest merge, the value that carries `validation` (A-247). -func newestOfRevision(ready []PendingEntry) int { - n := 0 - for i, e := range ready { - if e.Value.Seq > ready[n].Value.Seq { - n = i - } - } - return n -} - -// branchOfRevision is the watched branch the revision's merges landed on, its newest merged message's base (design 47 G §714); -// a message that names none is the trunk's. A pending value's own branch is its pull request's, never the branch validated. -func branchOfRevision(ready []PendingEntry) string { - if base := ready[newestOfRevision(ready)].Forge.Base; base != "" { - return base - } - return TrunkBranch -} - -// validationGate says whether the ready revision waits for its validation (§713): it goes on to the take-in once its newest -// value's validation was accepted at that value's sequence; it waits while its ask is out, while it is refused, and while it -// could not validate; otherwise it puts a new ask, the next try, and waits. Without an asker nothing is validated. -func (ix *Indexer) validationGate(ctx context.Context, ready []PendingEntry, own string) (bool, string, error) { - if ix.ValidationAsks == nil { - return false, "", nil - } - newest := ready[newestOfRevision(ready)] - seq := newest.Value.Seq - if v := newest.Value.Validation; v != nil && v.Seq == seq { - switch { - case v.Accepted == seq: - return false, "", nil - case v.Failed != "": - return true, fmt.Sprintf("the validation of the revision at %d could not validate: %s", seq, v.Failed), nil - case newest.Value.Refused != "": - return true, fmt.Sprintf("the revision at %d is refused: %s", seq, newest.Value.Refused), nil - case v.Ask != "": - return true, fmt.Sprintf("the revision at %d waits for its validation, the ask %s", seq, v.Ask), nil - } - } - return true, "", ix.askValidation(ctx, ready, own) -} - -// askValidation puts the revision's validation ask (§712) and writes it on its newest value, the try after every earlier try -// of the revision; an older value's validation goes (A-247: on the newest alone). A request or job that cannot be made could not -// validate: it is written as `failed`, said, and no ask is put. -func (ix *Indexer) askValidation(ctx context.Context, ready []PendingEntry, own string) error { - n := newestOfRevision(ready) - newest := ready[n] - try := 0 - for _, e := range ready { - if v := e.Value.Validation; v != nil && v.Try > try { - try = v.Try - } - } - v := &takein.Validation{Seq: newest.Value.Seq, Try: try + 1, Asked: ix.now()} - branch := branchOfRevision(ready) - ask := buildstep.StepAsk{ID: buildstep.NewID(), Kind: buildstep.AskCheck, Language: buildstep.AskGo, Asker: ValidationAsker, At: ix.now().UTC()} - req, entry, err := ix.validationRequest(ctx, branch, ready) - var raw []byte - if err == nil { - raw, err = json.Marshal(req) - } - var job buildstep.RunJob - if err == nil { - job, err = validationJob(entry, revisionHead(ready, entry.Repository, entry.Commit), branch, v.Seq, raw) - } - if err == nil { - v.Input, err = ix.KeepInput(ctx, ask.ID, raw) - } - if err != nil { - v.Failed = "the validation could not be asked: " + err.Error() - } else { - v.Ask, ask.Run = ask.ID, &job - } - if err := ix.writeValidation(ctx, ready, n, v, own); err != nil { - return err - } - if v.Failed != "" { - ix.sayValidation(ctx, ready, v) - return nil - } - if err := ix.ValidationAsks.Put(ctx, ask); err != nil { - return fmt.Errorf("the validation ask %s of the revision at %d could not be put: %w", ask.ID, v.Seq, err) - } - ix.logf("take-in: the revision at %d is validated by the ask %s (try %d); it waits for its end", v.Seq, ask.ID, v.Try) - return nil -} - -// revisionHead is the revision's head of repository: the newest merge of it the revision covers, else the commit its entry stands at. -func revisionHead(ready []PendingEntry, repository, commit string) string { - var seq uint64 - head := commit - for _, e := range ready { - if e.Forge.Merge != "" && strings.EqualFold(e.Forge.Repository, repository) && e.Value.Seq >= seq { - seq, head = e.Value.Seq, e.Forge.Merge - } - } - return head -} - -// writeValidation writes v on ready[n] and takes it from every other value, each by compare-and-set; a new validation clears the -// value's refusal. A value lost under it is an error: the evaluation is made again. -func (ix *Indexer) writeValidation(ctx context.Context, ready []PendingEntry, n int, v *takein.Validation, own string) error { - for i, e := range ready { - if i != n && e.Value.Validation == nil { - continue - } - p := e.Value - p.Validation, p.LastWriter = nil, own - if i == n { - p.Validation, p.Refused = v, "" - } - data, err := encodePending(p) - if err != nil { - return err - } - if err := ix.Pending.Update(ctx, e.Key, data, e.Revision); err != nil { - return fmt.Errorf("%s.%s: %w", PendingState, e.Key, err) - } - } - return nil -} - -// sayValidation says the revision's wait in take-in.pending with its refusal or why it could not validate. -func (ix *Indexer) sayValidation(ctx context.Context, ready []PendingEntry, v *takein.Validation) { - t := revisionWait(takein.TakeInPending, ready, "") - t.Failed = v.Failed - t.Refused = ready[newestOfRevision(ready)].Value.Refused - ix.sayTakeIn(ctx, t) -} - -// errNoValidationAnswer is an end whose answer is no verdict of both cases. -var errNoValidationAnswer = errors.New("no answer") - -// HearValidationEnded takes in the end of a validation ask (§713), heard on the build seat's step-ended through the indexer's own -// durable consumer: only an end of the indexer's asks outside a pipeline; an end whose ask no pending value names now is dropped, -// written in the journal. The answer is read from the event's own output or output object (A-247). -func (ix *Indexer) HearValidationEnded(ctx context.Context, e buildstep.StepEnded) error { - if e.Asker != ValidationAsker || e.Pipeline != "" || e.Ask == "" || ix.Pending == nil || !ix.pendingOnBus(ctx) { - return nil - } - own := ix.ownBuildNow(ctx) - ix.pendingMu.Lock() - defer ix.pendingMu.Unlock() - entries, err := ReadPending(ctx, ix.Pending, ix.logf) - if err != nil { - return err - } - ready := WaitsOf(entries).Ready - n := -1 - for i, p := range ready { - if p.Value.Validation != nil && p.Value.Validation.Ask == e.Ask { - n = i - } - } - if n < 0 { - ix.logf("take-in: the end of the validation ask %s (%s on %s) is dropped: no pending revision names that ask now", e.Ask, e.Outcome, e.Node) - return nil - } - ready, lost, _, _ := ix.loadNewest(ctx, ready) - if len(lost) > 0 { - return fmt.Errorf("the revision the validation ask %s ended for holds a merge that cannot be read", e.Ask) - } - n = newestOfRevision(ready) - v := *ready[n].Value.Validation - refused, failed := ix.validationAnswer(ctx, ready, e, v) - p := ready[n].Value - switch { - case failed != "": - v.Failed = failed - case refused != "": - p.Refused = refused - default: - v.Accepted = v.Seq - } - p.Validation, p.LastWriter = &v, own - data, err := encodePending(p) - if err != nil { - return err - } - if err := ix.Pending.Update(ctx, ready[n].Key, data, ready[n].Revision); err != nil { - return fmt.Errorf("%s.%s: %w", PendingState, ready[n].Key, err) - } - if v.Accepted == 0 { - ready[n].Value = p - ix.sayValidation(ctx, ready, &v) - ix.logf("take-in: the validation ask %s of the revision at %d ended: refused %q, could not validate %q", e.Ask, v.Seq, refused, failed) - return nil - } - ix.logf("take-in: the validation ask %s accepted the revision at %d", e.Ask, v.Seq) - if own == "" { - return nil // the take-in waits for the build this program runs, as every take-in does - } - _, _, err = ix.evaluate(ctx, own, false) - return err -} - -// validationAnswer reads the end's answer: the refusal of the revision, or why it could not validate (the step did not pass, the -// program exited other than 0, its answer is no verdict of both cases, or its request is no longer the one asked). -func (ix *Indexer) validationAnswer(ctx context.Context, ready []PendingEntry, e buildstep.StepEnded, v takein.Validation) (string, string) { - if e.Outcome != buildstep.OutcomePassed { - return "", strings.TrimSpace(e.Outcome + ": " + e.Said) - } - // The answer is the program's last line of standard output, inline or kept by the seat as an object (A-254); a program that - // exited other than 0 ended the step failed, read above. - out, err := e.ReadOutput(buildstep.AskGo, ix.AskVerb) - if err != nil { - return "", "the run's answer could not be read: " + err.Error() - } - var verdict composer.Verdict - if err := json.Unmarshal(out, &verdict); err != nil { - return "", fmt.Sprintf("%v: the manifest validator's output is no verdict: %v", errNoValidationAnswer, err) - } - req, _, err := ix.validationRequest(ctx, branchOfRevision(ready), ready) - var raw []byte - if err == nil { - raw, err = json.Marshal(req) - } - if err != nil { - return "", "the request cannot be made again to read the verdict: " + err.Error() - } - if runDigest(raw) != v.Input { - return "", "the revision's request is no longer the one the ask " + v.Ask + " was asked with" - } - refused, err := ValidationRefusal(req, verdict) - if err != nil { - return "", err.Error() - } - return RefusedWords(refused), "" -} - -// ValidateAgainTool asks the validation of a revision that could not validate again by hand (§713). -const ValidateAgainTool = "indexer_validate_again" - -// validationWords is the validation's state as the hand ask answers it. -func validationWords(v *takein.Validation, refused string) string { - switch { - case v == nil: - return "no validation is asked yet" - case v.Accepted != 0: - return fmt.Sprintf("accepted at %d (try %d, the ask %s)", v.Accepted, v.Try, v.Ask) - case v.Failed != "": - return fmt.Sprintf("could not validate at %d (try %d): %s", v.Seq, v.Try, v.Failed) - case refused != "": - return fmt.Sprintf("refused at %d (try %d): %s", v.Seq, v.Try, refused) - default: - return fmt.Sprintf("the ask %s is out since %s (try %d at %d)", v.Ask, v.Asked.UTC().Format(time.RFC3339), v.Try, v.Seq) - } -} - -// ValidateAgainByHand is indexer_validate_again answered (§713): the ready revision under key (its oldest merge's m. key, as -// indexer_pending names it) whose validation could not validate is asked again, the next try; its answer says the ask's state. -// A revision accepted, refused or whose ask is out is asked nothing: a refusal is asked again only by a later merge of its branch. -func (ix *Indexer) ValidateAgainByHand(ctx context.Context, key, by, why string) TakenIn { - r := TakenIn{Module: key} - if ix.Pending == nil || ix.ValidationAsks == nil { - r.Answer = "this indexer asks no validation: nothing asked" - return r - } - own := ix.ownBuildNow(ctx) - ix.pendingMu.Lock() - defer ix.pendingMu.Unlock() - entries, err := ReadPending(ctx, ix.Pending, ix.logf) - if err != nil { - r.Answer = err.Error() - return r - } - ready := WaitsOf(entries).Ready - if len(ready) == 0 || ready[0].Key != key { - r.Answer = fmt.Sprintf("no revision waits under %s: indexer_pending names the waits", key) - return r - } - ready, lost, reasons, _ := ix.loadNewest(ctx, ready) - if len(lost) > 0 { - r.Answer = fmt.Sprintf("nothing asked: the message of %s is not read (%s)", lost[0].Key, reasons[lost[0].Key]) - return r - } - newest := ready[newestOfRevision(ready)] - v := newest.Value.Validation - if v == nil || v.Seq != newest.Value.Seq || v.Failed == "" { - r.Answer = "nothing asked: the validation is " + validationWords(v, newest.Value.Refused) + "; only one that could not validate is asked again by hand" - return r - } - ix.logf("take-in: the validation of %s is asked again by hand by %s: %s", key, by, why) - if err := ix.askValidation(ctx, ready, own); err != nil { - r.Answer = "the validation was not asked again: " + err.Error() - return r - } - after, err := ReadPending(ctx, ix.Pending, ix.logf) - if err == nil { - ready = WaitsOf(after).Ready - } - if err != nil || len(ready) == 0 { - r.Answer = "asked again; its state is not read back" - return r - } - n := ready[newestOfRevision(ready)] - r.Stored = n.Value.Validation != nil && n.Value.Validation.Failed == "" - r.Answer = fmt.Sprintf("asked again by %s (%s): the validation is %s", by, why, validationWords(n.Value.Validation, n.Value.Refused)) - return r -} diff --git a/modules/mesh-delivery-indexer/internal/index/validation_wait_test.go b/modules/mesh-delivery-indexer/internal/index/validation_wait_test.go deleted file mode 100644 index 9588293b..00000000 --- a/modules/mesh-delivery-indexer/internal/index/validation_wait_test.go +++ /dev/null @@ -1,113 +0,0 @@ -package index - -import ( - "context" - "encoding/json" - "strings" - "testing" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer" -) - -type askRecorder struct{ asks []buildstep.StepAsk } - -func (r *askRecorder) Put(_ context.Context, a buildstep.StepAsk) error { - r.asks = append(r.asks, a) - return nil -} - -// Design 47 G §713, A-247: a pending revision names its ask on its newest value and resumes on that ask's end; an end of a -// replaced ask is dropped; a run that could not validate is no refusal and never passed; the start asks nothing again while -// the ask is out. -func TestAPendingRevisionWaitsForItsValidation(t *testing.T) { - ctx := context.Background() - b := newTakeInBed(t) - asks := &askRecorder{} - b.ix.ValidationAsks, b.ix.Inputs, b.ix.Revisions = asks, newMem(), newMemObjects(0) - // No entry of the revision claims the seat mesh-catalog: the validation could not be asked, so it is failed, said, and - // nothing is taken in. - b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "")) - entries := readPending(t, b) - if len(entries) != 1 || entries[0].Value.Validation == nil || !strings.Contains(entries[0].Value.Validation.Failed, "claims the seat mesh-catalog") || len(asks.asks) != 0 { - t.Fatalf("pending %+v, asks %d: want m.2 pending with failed and no ask", entries, len(asks.asks)) - } - if last := b.said[len(b.said)-1]; last.Kind != takein.TakeInPending || last.Failed == "" || last.Refused != "" { - t.Errorf("said %+v, want take-in.pending naming failed, never a refusal", last) - } - if w := PendingWaits(entries); len(w) != 1 || w[0].Validation == nil || w[0].Validation.Try != 1 { - t.Errorf("indexer_pending %+v, want the validation whole", w) - } - // An ask out: the newest value names it; the start asks nothing new while it is out. - loaded, _, _, _ := b.ix.loadNewest(ctx, entries) - req, _, err := b.ix.validationRequest(ctx, TrunkBranch, loaded) - if err != nil { - t.Fatal(err) - } - raw, _ := json.Marshal(req) - setValidation := func(v takein.Validation) { - e := readPending(t, b)[0] - p := e.Value - p.Validation = &v - data, _ := encodePending(p) - if err := b.pending.Update(ctx, e.Key, data, e.Revision); err != nil { - t.Fatal(err) - } - } - setValidation(takein.Validation{Ask: "mvl0frv18d1957", Input: runDigest(raw), Seq: 2, Try: 2}) - if _, _, err := b.ix.TakeInAtStart(ctx); err != nil || len(asks.asks) != 0 || len(b.keys()) != 1 { - t.Fatalf("the start: %v, asks %d, pending %v: want it waiting for its ask", err, len(asks.asks), b.keys()) - } - end := func(ask, outcome, said string, answer any) buildstep.StepEnded { - out, _ := json.Marshal(answer) - return buildstep.StepEnded{Ask: ask, ID: ask, Asker: ValidationAsker, Node: "ace", Outcome: outcome, Said: said, Output: string(out)} - } - // The answer is the manifest validator's verdict itself, the program's last line (A-254). - accepted := composer.Verdict{Cases: []composer.CaseVerdict{{Name: "change"}, {Name: "trunk"}}} - // The end of a replaced ask is dropped, written in the journal. - if err := b.ix.HearValidationEnded(ctx, end("mvl0frv18d1950", buildstep.OutcomePassed, "", accepted)); err != nil || len(b.keys()) != 1 || !strings.Contains(b.logs.String(), "is dropped") { - t.Fatalf("a replaced ask's end: %v, pending %v", err, b.keys()) - } - // could not run: pending with failed, never passed. - if err := b.ix.HearValidationEnded(ctx, end("mvl0frv18d1957", buildstep.OutcomeCouldNotRun, "no build node", nil)); err != nil { - t.Fatal(err) - } - if v := readPending(t, b)[0].Value.Validation; v == nil || v.Failed != "could not run: no build node" || v.Accepted != 0 { - t.Fatalf("after could not run: %+v", v) - } - // indexer_validate_again (§713): its hand act recorded first, then the next try asked; the answer says the ask's state. - var recorded []string - tools := ValidationTools(func(key string, value []byte) (TakeInAsk, error) { - a, err := DecodeTakeIn(value) - if err != nil || key != "pending.validate-again."+takein.PendingKey(2) || !a.ValidateAgain { - t.Fatalf("asked %s %+v %v", key, a, err) - } - r := b.ix.ValidateAgainByHand(ctx, a.Pending, a.By, a.Why) - a.Result = &r - return a, nil - }, b.ix.now, func(what, why string) error { recorded = append(recorded, what); return nil }) - if len(tools) != 1 || tools[0].Name != ValidateAgainTool { - t.Fatalf("tools = %v", tools) - } - if _, err := tools[0].Run(map[string]any{"key": takein.PendingKey(2)}); err == nil || len(recorded) != 0 { - t.Fatalf("asked again with no why: %v, %v", err, recorded) - } - got, err := tools[0].Run(map[string]any{"key": takein.PendingKey(2), "why": "the build node is back"}) - if v := readPending(t, b)[0].Value.Validation; err != nil || len(recorded) != 1 || v == nil || v.Try != 3 || v.Failed == "could not run: no build node" || - !strings.Contains(got.(*TakenIn).Answer, "try 3") { - t.Fatalf("asked again: %v, recorded %v, validation %+v, answer %+v: want the try after 2 asked", err, recorded, v, got) - } - setValidation(takein.Validation{Ask: "mvl0frv18d1959", Input: runDigest(raw), Seq: 2, Try: 3}) - if r := b.ix.ValidateAgainByHand(ctx, takein.PendingKey(2), "lead", "again"); r.Stored || !strings.Contains(r.Answer, "is out") || readPending(t, b)[0].Value.Validation.Try != 3 { - t.Fatalf("an ask out was asked again: %+v", r) - } - // Asked again (by hand or by a later merge) and accepted: the revision is taken in, and its keys go. - setValidation(takein.Validation{Ask: "mvl0frv18d1958", Input: runDigest(raw), Seq: 2, Try: 3}) - if err := b.ix.HearValidationEnded(ctx, end("mvl0frv18d1958", buildstep.OutcomePassed, "", accepted)); err != nil { - t.Fatal(err) - } - if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('c') { - t.Fatalf("accepted: pending %v %+v, avahi at %s: want it taken in\n%s", b.keys(), readPending(t, b)[0].Value, b.commit(t, "avahi"), b.logs.String()) - } -} diff --git a/modules/mesh-delivery-indexer/internal/index/verdicts.go b/modules/mesh-delivery-indexer/internal/index/verdicts.go index b8d0c6f0..526bb638 100644 --- a/modules/mesh-delivery-indexer/internal/index/verdicts.go +++ b/modules/mesh-delivery-indexer/internal/index/verdicts.go @@ -506,8 +506,8 @@ func (ix *Indexer) writeVerdict(ctx context.Context, digest string, v meshcatalo } // OwnBuildState is the indexer's state of the build it runs (the bucket mesh-delivery-indexer_own-build, one key, OwnBuildKey): -// written once, at the process's start by FixOwnBuild, so indexer_pending says whether this process knows its build and, when it -// does not, since when and why. +// written once, at the process's start by FixOwnBuild, so indexer_pending says which build this process knows it runs, and since +// when. The build only names the writer of what is written (A-476 b). const ( OwnBuildStateName = "own-build" OwnBuildKey = "state" @@ -526,8 +526,7 @@ type OwnBuild struct { // seat (OwnBuild; never the trunk's value of the indexer, which names a build that may be merged and not delivered), before // this process hears a merge. It is never asked again: while the node-engine replaces the indexer the seat already names the // successor, and an old process that learnt it later would take revisions in with its old judge. Where the start's ask is not -// answered the build stays unknown for the life of this process: every merge waits, loudly (a take-in.pending that says so, the -// own-build state, indexer_pending), and the module is started again to learn it. +// answered the build stays unknown for the life of this process, said in the own-build state and the journal; nothing waits for it. func (ix *Indexer) FixOwnBuild(ctx context.Context) { ix.ownMu.Lock() if ix.ownFixed || ix.OwnBuild == nil { @@ -543,7 +542,7 @@ func (ix *Indexer) FixOwnBuild(ctx context.Context) { st := OwnBuild{V: 1, Since: ix.now().UTC()} if err != nil { st.Why = err.Error() - ix.logf("judge: the build this program runs is not known (%v), so every merge waits and a manifest with a field it does not know waits and is not refused; it is learnt only at the start, so the indexer is started again", err) + ix.logf("judge: the build this program runs is not known (%v); merges are taken in all the same, and it is learnt only at the start", err) } else { st.Known, st.FP = true, fp ix.logf("judge: this program runs the indexer build %s, fixed here at its start", fp) diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 93082750..439a6b0b 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -179,8 +179,6 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a // commit is in another (issue 1900). if want := []string{"seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", - "seat:node-build-go.accept.check", // design 47 G A-231: the validation's run ask (§712) on the check queue - "seat:node-build-go.output", // design 47 G A-254: a validation's answer kept as an object "seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } @@ -204,9 +202,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered. index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool, index.EntryIndexTool, index.NamedDropTool, // design 47 G §675 - index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.UnpinTool, index.ConsumerTool, index.RunsTool, index.RunDropTool, // design 47 G §667, §8, §579, §64 (A-194) - index.ValidateAgainTool) // design 47 G §713 - if !reflect.DeepEqual(m.Tools, want) || len(want) != 52 || !slices.Contains(m.Tools, "modules") { + index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.UnpinTool, index.ConsumerTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194) + if !reflect.DeepEqual(m.Tools, want) || len(want) != 51 || !slices.Contains(m.Tools, "modules") { t.Errorf("tools = %v, want %v", m.Tools, want) } // Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section. diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index b7324498..49f95a7d 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -77,8 +77,7 @@ "indexer_unpin", "indexer_consumer", "indexer_runs", - "indexer_run_drop", - "indexer_validate_again" + "indexer_run_drop" ], "consumes": [ "go-package-registry.published", @@ -180,8 +179,6 @@ "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", - "seat:node-build-go.accept.check", - "seat:node-build-go.output", "seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions" diff --git a/modules/mesh-delivery-indexer/takein/takein.go b/modules/mesh-delivery-indexer/takein/takein.go index 66923591..351635cf 100644 --- a/modules/mesh-delivery-indexer/takein/takein.go +++ b/modules/mesh-delivery-indexer/takein/takein.go @@ -1,5 +1,5 @@ // Package takein is the indexer's take-in contract (novox/hq ADR 0633 §1, §5): the events it publishes on its own stream -// `take-in` while a source revision waits for the indexer it names or a delivery group is gathered, their keys, and the +// `take-in` while a source revision waits (its message unread, lost or held for room) or a delivery group is gathered, their keys, and the // answer of its tool indexer_pending. The indexer writes them; the watchman and the view read them. It is the indexer's // own package: one module owns these types, so they live with it and not in mesh-sdk (the operator, 2026-11-04). package takein @@ -118,7 +118,7 @@ func checkTakeInKey(key string) error { } // TakeIn is the body of a take-in event. Kind is the event's kind and Key its key. What says what waits: a revision -// (Key its oldest merge's PendingKey, WaitsFor the indexer build it names) or a group being gathered (Key its +// (Key its oldest merge's PendingKey) or a group being gathered (Key its // GroupTakeInKey, Members every approved member with its head, Merges the members held so far). Merges are the merges // it holds, oldest first; Since is when the wait began. A `taken` event names the same once the wait ended; By and Why // name a hand act (a hand take-in, or a drop, Dropped), and are carried by nothing else. @@ -138,16 +138,9 @@ type TakeIn struct { // Lost are the keys of merges whose message the forge bridge's stream no longer holds: they are not taken in, and the // revision waits until indexer_drop_pending drops them (issue 1585). Lost []string `json:"lost,omitempty"` - // Refused is why the manifest of the indexer the revision names was refused when the revision was worked out: with - // the seat definitions the revision carries (ADR 0633 §1), the manifest validator's problems. The revision waits for - // an indexer whose manifest is accepted, and says why here, as it says what it lost. + // Refused is why the revision is refused: the cycle with a build edge its catalog graph would hold (design 47 G §49). The + // revision waits, and says why here, as it says what it lost. Refused string `json:"refused,omitempty"` - // Failed is why the revision's validation could not validate (design 47 G §713): the build seat's words, never a refusal - // and never passed; it is asked again by a later merge of the branch and by indexer_validate_again. - Failed string `json:"failed,omitempty"` - // Unknown is why the indexer holds a merge it does not take in although no indexer build is named: it cannot learn the build - // it runs itself (ADR 0633 §1), so Since is when it began not to know. A pending revision names WaitsFor or Unknown. - Unknown string `json:"own-build-unknown,omitempty"` // Branch, Source and Entries are what a take-in took in (design 47 G §476, §493, §648): the watched branch, the source // revision it makes of that branch, and one entry per module it wrote. Every taken event of one take-in carries the same; // a taken event that ends a wait without a take-in (a group's end, a drop) carries none and starts no pipeline. @@ -184,8 +177,8 @@ func (t TakeIn) check() error { if !strings.HasPrefix(t.Key, "m.") { return fmt.Errorf("delivery: a revision's take-in key is its oldest merge's m., not %q", t.Key) } - if t.Kind == TakeInPending && t.WaitsFor == "" && t.Unknown == "" && len(t.Lost) == 0 && t.Refused == "" && t.Failed == "" { - return fmt.Errorf("delivery: a pending revision %s names the indexer build it waits for, why the indexer does not know its own, the merges it lost, its refusal or why it could not validate", t.Key) + if t.Kind == TakeInPending && len(t.Lost) == 0 && t.Refused == "" { + return fmt.Errorf("delivery: a pending revision %s names the merges it lost or its refusal", t.Key) } case TakeInGroup: if t.Group == "" { @@ -266,32 +259,15 @@ type PendingWait struct { Unread []string `json:"unread,omitempty"` // Refused is why the manifest of the indexer it names was refused (TakeIn.Refused). Refused string `json:"refused,omitempty"` - // Validation is the revision's validation whole, as its newest pending value keeps it (design 47 G §713, A-247). - Validation *Validation `json:"validation,omitempty"` -} - -// Validation is a pending revision's validation (design 47 G §712, §713, A-232, A-247), kept on the revision's newest pending -// value alone: the run ask's id, when it was asked, its input's digest, the sequence of the newest merge it covers and the try -// number for it, why it could not validate, and the sequence it was accepted at (never asked again once set). -type Validation struct { - Ask string `json:"ask,omitempty"` - Asked time.Time `json:"asked,omitzero"` - Input string `json:"input,omitempty"` - Seq uint64 `json:"seq"` - Try int `json:"try"` - Failed string `json:"failed,omitempty"` - Accepted uint64 `json:"accepted,omitempty"` } // PendingAnswer is indexer_pending's answer: every wait, oldest first, and the pending keys that do not read. type PendingAnswer struct { - Waits []PendingWait `json:"waits"` - Unread []string `json:"unread,omitempty"` - // OwnBuildUnknown is why the running indexer does not know its own build, and since when: while it is set, every merge waits - // (a take-in.pending with the same words) and the indexer is started again to learn it. Empty when the build is known. - OwnBuildUnknown string `json:"own-build-unknown,omitempty"` - OwnBuildSince time.Time `json:"own-build-since,omitzero"` - // OwnBuild is the build the running indexer knows it runs, fixed at its start; OwnBuildSince is since when. Empty when unknown. + Waits []PendingWait `json:"waits"` + Unread []string `json:"unread,omitempty"` + OwnBuildSince time.Time `json:"own-build-since,omitzero"` + // OwnBuild is the build the running indexer knows it runs, fixed at its start; OwnBuildSince is since when. Empty when unknown: + // it names only the writer of what the indexer writes, and nothing waits for it. OwnBuild string `json:"own-build,omitempty"` } diff --git a/modules/mesh-delivery-indexer/takein/takein_test.go b/modules/mesh-delivery-indexer/takein/takein_test.go index f69c1e49..a2376fe6 100644 --- a/modules/mesh-delivery-indexer/takein/takein_test.go +++ b/modules/mesh-delivery-indexer/takein/takein_test.go @@ -21,10 +21,7 @@ func sampleTakeIns(since time.Time) []TakeIn { Members: []meshdeliveryclient.Head{{Repository: "novox/mesh-delivery", Number: 9, Head: "ffff"}, {Repository: "novox/mesh-sdk", Number: 37, Head: "dddd"}}, Since: since}, {Kind: TakeInTaken, Key: PendingKey(12), What: TakeInRevision, Merges: m, WaitsFor: "src2:abc", Since: since, By: "lead-47@shanks", Why: "the named indexer can never run"}, - {Kind: TakeInPending, Key: PendingKey(13), What: TakeInRevision, Merges: m, Unknown: "the engine does not answer", Since: since}, // the indexer cannot learn its own build - // design 47 G §713: a revision refused by its validation, and one that could not validate. - {Kind: TakeInPending, Key: PendingKey(14), What: TakeInRevision, Merges: m, Refused: "a: names no version", Since: since}, - {Kind: TakeInPending, Key: PendingKey(15), What: TakeInRevision, Merges: m, Failed: "could not run: no build node", Since: since}, + {Kind: TakeInPending, Key: PendingKey(14), What: TakeInRevision, Merges: m, Refused: "a cycle with a build edge", Since: since}, } } diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/tiers_rows_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/tiers_rows_test.go index 8b0057c9..8a497a16 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/tiers_rows_test.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/tiers_rows_test.go @@ -108,7 +108,7 @@ func tiersToolSaysEachModulesWait(t *testing.T) { t.Fatalf("view_tiers with waits = %+v, %v", out, err) } web = held.Tiers[1][0] - if last := web.Waits[len(web.Waits)-1]; last.Step != view.TakeInStep || !last.Since.Equal(heldSince) || !strings.Contains(last.Words, "src2:abc") { + if last := web.Waits[len(web.Waits)-1]; last.Step != view.TakeInStep || !last.Since.Equal(heldSince) || !strings.Contains(last.Words, "unread or lost") { t.Errorf("web = %+v, want its last wait on the edge take-in since %s", web, heldSince) } if lib := held.Tiers[0][1]; lib.Module != "lib" || lib.Waits[len(lib.Waits)-1].Step != view.TakeInStep || !strings.Contains(lib.Waits[len(lib.Waits)-1].Words, "group feat") { diff --git a/modules/mesh-delivery-view/internal/view/tiers.go b/modules/mesh-delivery-view/internal/view/tiers.go index 1d98db2b..721b5da9 100644 --- a/modules/mesh-delivery-view/internal/view/tiers.go +++ b/modules/mesh-delivery-view/internal/view/tiers.go @@ -219,7 +219,7 @@ func (r Reader) heldBack(out *Tiers) (map[string]TierWait, []string) { return held, nil } for _, w := range a.Waits { - words := "the indexer holds back " + w.Key + ", which waits for the indexer build " + w.WaitsFor + words := "the indexer holds back " + w.Key + " (its message is unread or lost, or it is held for room)" if w.What == takein.TakeInGroup { words = "the indexer gathers the group " + w.Group + " and its end has not come" } else if w.Behind != "" { diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go index 75f1ff81..3843658f 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go @@ -132,7 +132,7 @@ func TestRunNeedsItsCredential(t *testing.T) { // 1135/AC6 } out.findings[old.Key()] = true w.SeedFinding(old.Key(), value) - body, err := takein.EncodeTakeIn(takein.TakeIn{Kind: takein.TakeInPending, Key: "m.00000000000000000043", What: takein.TakeInRevision, Since: time.Now(), WaitsFor: "src2:x", + body, err := takein.EncodeTakeIn(takein.TakeIn{Kind: takein.TakeInPending, Key: "m.00000000000000000043", What: takein.TakeInRevision, Since: time.Now(), Lost: []string{"m.00000000000000000043"}, Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}}) if err != nil { t.Fatal(err) diff --git a/modules/mesh-delivery-watchman/internal/watch/findings_test.go b/modules/mesh-delivery-watchman/internal/watch/findings_test.go index 9e292509..896c8807 100644 --- a/modules/mesh-delivery-watchman/internal/watch/findings_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/findings_test.go @@ -136,27 +136,27 @@ func TestTheWatchmansFindings(t *testing.T) { // began ten minutes ago, and the indexer still holds it. since := c.t.Add(-10 * time.Minute) w.SeedTakeIn(takein.PendingAnswer{Waits: []takein.PendingWait{{Key: revKey, What: takein.TakeInRevision, Keys: []string{revKey}, - Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}, WaitsFor: "src2:x", Since: since}}}) + Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}, Lost: []string{"m.00000000000000000042"}, Since: since}}}) w.TakeInCaughtUp() if len(o.findings) != 0 || o.ttls[signalKey("take-in", revKey)] != TakeInRevisionBound-10*time.Minute { t.Fatalf("within its bound: findings %v, ttl %v; want none yet and the rest of the bound", o.findings, o.ttls[signalKey("take-in", revKey)]) } expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound-10*time.Minute) }, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}}, - {"ADR 0633 §1 (issue 1921): a merge held because the indexer cannot learn its own build is raised past the revision bound from its since, naming why", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + {"ADR 0633 §1 (issue 1921): (A-476 b) a revision held because its message is lost or unread is raised past the revision bound from its since, naming why", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { w.TakeInCaughtUp() body, err := takein.EncodeTakeIn(takein.TakeIn{Kind: takein.TakeInPending, Key: revKey, What: takein.TakeInRevision, Since: c.t, - Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}, Unknown: "the engine does not answer"}) + Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}, Lost: []string{revKey}}) if err != nil { t.Fatal(err) } w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), body) if o.ttls[signalKey("take-in", revKey)] != TakeInRevisionBound || len(o.findings) != 0 { - t.Fatalf("an unknown own build: ttls %v, findings %v", o.ttls, o.findings) + t.Fatalf("a lost message: ttls %v, findings %v", o.ttls, o.findings) } expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound) for _, f := range o.findings { - if !strings.Contains(f.Summary, "cannot learn the build it runs") { + if !strings.Contains(f.Summary, "lost or unread") { t.Errorf("finding %q does not say why", f.Summary) } } @@ -573,7 +573,7 @@ func takeInBody(t *testing.T, kind, key string, since time.Time) []byte { if key[0] == 'g' { body.What, body.Group = takein.TakeInGroup, "feat/take-in" } else if kind == takein.TakeInPending { - body.WaitsFor = "src2:abc" + body.Lost = []string{key} } b, err := takein.EncodeTakeIn(body) if err != nil { diff --git a/modules/mesh-delivery-watchman/internal/watch/takein.go b/modules/mesh-delivery-watchman/internal/watch/takein.go index 9120b322..e534ac6c 100644 --- a/modules/mesh-delivery-watchman/internal/watch/takein.go +++ b/modules/mesh-delivery-watchman/internal/watch/takein.go @@ -135,10 +135,7 @@ func (w *Watchman) takeIn(kind, key string, t takein.TakeIn) { w.unfind(finding) return } - bound, what := TakeInRevisionBound, "a source revision waits for the indexer build "+t.WaitsFor - if t.Unknown != "" { - what = "a source revision is held because the indexer cannot learn the build it runs (" + t.Unknown + ")" - } + bound, what := TakeInRevisionBound, "a source revision is held (a message lost or unread, no room for it, or its refusal)" if t.What == takein.TakeInGroup { bound, what = TakeInGroupBound, "a delivery group is gathered and its end has not come" } @@ -152,7 +149,7 @@ func (w *Watchman) takeIn(kind, key string, t takein.TakeIn) { Headline: "A merged change is held back by the indexer", Explanation: "The indexer holds " + what + " and has not taken it in within " + bound.String() + ", so what the merge changes does " + "not reach the machines.", - Needs: "the indexer build it waits for to run, or the take-in by hand (indexer_take_in_pending), or a drop (indexer_drop_pending)"} + Needs: "the take-in by hand (indexer_take_in_pending), or a drop (indexer_drop_pending)"} left := bound - w.now().Sub(t.Since) w.unfind(finding) w.unsignal(takeInRow, key)