diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index 629dcee0..24400527 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -126,7 +126,8 @@ func TestManifest(t *testing.T) { if !slices.Equal(m.Consumes, []string{"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-delivery-indexer.views.graph", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", buckets.TellerModule + "." + tellerevents.AskEnded, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-engine.manifest-reads-changed"}) { t.Errorf("consumes = %v, want the indexer's views of graph, manifests and modules, the teller's ask-ended and the engines' reads changed", m.Consumes) } - if want := []string{"mesh-delivery-indexer.graph", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", + if want := []string{"mesh-delivery-indexer.graph", "mesh-delivery-indexer.manifests", + "seat:mesh-catalog.revision", // ADR 0663 §2: which modules exist "seat:mesh-catalog.seat-definitions", // ADR 0636 §4: the seat list "seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", // ADR 0565 §9, §11: the seat alone "seat:node-backup.backed-up", "seat:node-backup.now", diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index d81cca86..500602d6 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -52,7 +52,7 @@ "invokes": [ "mesh-delivery-indexer.graph", "mesh-delivery-indexer.manifests", - "mesh-delivery-indexer.modules", + "seat:mesh-catalog.revision", "seat:mesh-catalog.seat-definitions", "seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go index 7b080273..511ca138 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go @@ -13,8 +13,9 @@ import ( "mesh-delivery-indexer/internal/index" ) -// catalogRead reads the indexer's modules, manifests and graph states for the seat's verbs. -func catalogRead(modules, manifests, nodes stdio.KeptState, seatDefinitions, marks views.Source) index.CatalogRead { +// catalogRead reads the indexer's view of the trunk (its modules and module graph, from the newest catalog graph +// revision) and its manifests state for the seat's verbs. +func catalogRead(modules views.Source, manifests stdio.KeptState, nodes views.Source, seatDefinitions, marks views.Source) index.CatalogRead { return index.CatalogRead{ SeatDefinitions: seatDefinitions, Marks: marks, Modules: func() (map[string]meshcatalog.ModulesByModuleValue, error) { @@ -24,11 +25,11 @@ func catalogRead(modules, manifests, nodes stdio.KeptState, seatDefinitions, mar } out := map[string]meshcatalog.ModulesByModuleValue{} for _, k := range keys { - e, err := modules.Get(k) + e, ok, err := modules.Get(k) if err != nil { return nil, fmt.Errorf("modules.%s: %w", k, err) } - if e == nil { + if !ok { continue } if m, err := meshcatalogclient.DecodeModule(e.Value); err == nil { @@ -59,11 +60,11 @@ func catalogRead(modules, manifests, nodes stdio.KeptState, seatDefinitions, mar } g := graph.Graph{} for _, k := range keys { - e, err := nodes.Get(k) + e, ok, err := nodes.Get(k) if err != nil { return nil, fmt.Errorf("graph.%s: %w", k, err) } - if e == nil { + if !ok { continue } if n, err := graph.Decode(e.Value); err == nil { @@ -76,7 +77,7 @@ func catalogRead(modules, manifests, nodes stdio.KeptState, seatDefinitions, mar } // catalogTools are the seat's verbs over the indexer's own states, served beside the indexer's own tools, since the indexer's manifest claims the seat (ADR 0604 §2). -func catalogTools() []stdio.Tool { - return index.CatalogTools(catalogRead(stdio.State("modules"), stdio.State("manifests"), stdio.State(graph.State), +func catalogTools(state func(string) views.Source) []stdio.Tool { + return index.CatalogTools(catalogRead(state("modules"), stdio.State("manifests"), state(graph.State), keptSource(index.SeatDefinitionsState), keptSource(views.MarksState))) } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index e4c89f9c..574441c6 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -17,6 +17,7 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/busstate" "mesh-delivery-indexer/internal/index" ) @@ -56,7 +57,10 @@ func Tools(ask asker, now func() time.Time, read stdio.KeptState, counts func() "source names for that repository, for a merge whose merged message the indexer never heard (the forge bridge has no " + "tool): a head whose trunk holds no module.json is taken in as a merge that leaves none (every module of the repository " + "is dropped, one another repository's trunk holds handed over from there first, listed in handed and dropped); a head " + - "that holds modules is answered \"take each module in by name\".", + "that holds modules is answered \"take each module in by name\". With neither module nor repository it is the indexer's " + + "catch-up tool, named in its manifest beside the catalog graph revisions: it takes in the head the gate's source names for " + + "every repository, as merges, and keeps the view as the next revision when an entry moved; the node agent calls it so " + + "after restoring them. Answers the repositories taken in and, in unread, those whose merged message was not read.", Input: map[string]any{"type": "object", "properties": map[string]any{ "module": map[string]any{"type": "string", "description": "the module's name, as the catalogue registers it"}, "repository": map[string]any{"type": "string", "description": "instead of module: the repository (owner/name) whose head is taken in"}, @@ -65,9 +69,12 @@ func Tools(ask asker, now func() time.Time, read stdio.KeptState, counts func() var key string var value []byte var err error - if str(args, "repository") != "" && str(args, "module") == "" { + switch { + case str(args, "repository") == "" && str(args, "module") == "": + key, value, err = index.NewCatchUp(str(args, "why"), stdio.CallerOf(args), now()) + case str(args, "repository") != "" && str(args, "module") == "": key, value, err = index.NewTakeInRepository(str(args, "repository"), str(args, "why"), stdio.CallerOf(args), now()) - } else { + default: key, value, err = index.NewTakeIn(str(args, "module"), str(args, "why"), stdio.CallerOf(args), now()) } if err != nil { @@ -275,16 +282,26 @@ func main() { asks := askTakeIn(stdio.State(index.TakeInsState), 25*time.Second) acts := meshhandactsclient.Recorder{Ask: stdio.Ask} tools := Tools(asks, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState)), acts) - tools = append(tools, catalogTools()...) - tools = append(tools, viewTools(os.Getenv, keptSource, time.Now)...) + // The modules and the module graph are the newest catalog graph revision's (milestone 47/G): revision.go. + revisions := bundleObjects{busstate.ObjectsOf(index.CatalogRevisionsObjects)} + trunk := index.TrunkStates{Objects: revisions, Under: keptSource} + tools = append(tools, catalogTools(trunk.State)...) + // The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly + // the parts whose entry declares it (novox/hq ADR 0663). + tools = append(tools, index.RevisionTool(revisions, index.Module)) + tools = append(tools, index.CatalogRevisionTools(revisions, asks, time.Now, func(what, why string) error { + _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.DropCandidateGraphTool, Verb: "drop-candidate-graph"}) + return err + })...) + tools = append(tools, viewTools(os.Getenv, trunk.State, time.Now)...) tools = append(tools, index.PendingTools(keptRead{stdio.State(index.PendingState)}, keptRead{stdio.State(index.OwnBuildStateName)}, asks, time.Now)...) tools = append(tools, index.FingerprintsTools(asks, time.Now, func(what, why string) error { _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.FingerprintsAgainTool, Verb: "fingerprints-again"}) return err }, 10*time.Second)...) tools = append(tools, index.TryCandidateTools(keptRead{stdio.State("candidates")}, keptRead{stdio.State(index.TryRecoveryState)}, asks, time.Now)...) - tools = append(tools, configReadersTool(stdio.NewWatchedStates()), busStateTool(keptSource), requirementsTool(keptSource), - packageUsersTool(keptSource), seatDefinitionTool(keptSource), outOfStepTool(keptSource), + tools = append(tools, configReadersTool(stdio.NewWatchedStates(), trunk.State("modules")), busStateTool(trunk.State), requirementsTool(keptSource), + packageUsersTool(trunk.State), seatDefinitionTool(keptSource), outOfStepTool(keptSource), holdsTool(stdio.State(index.HoldsState), asks, time.Now, acts), engineReadsTool(time.Now)) if err := stdio.Serve("", tools); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/revision.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/revision.go new file mode 100644 index 00000000..71407173 --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/revision.go @@ -0,0 +1,50 @@ +package main + +import ( + "context" + + "git.novox.be/novox/mesh-sdk/go/busstate" + + "mesh-delivery-indexer/internal/index" +) + +// bundleObjects is the objects catalog-revisions as the bundle reaches them through its runtime (the old objects section +// of a foundation part, reached by the module's own name). +type bundleObjects struct{ o busstate.Objects } + +func (b bundleObjects) Put(_ context.Context, name string, data []byte) error { + _, err := b.o.Put(name, data, "") + return err +} + +func (b bundleObjects) Get(_ context.Context, name string) ([]byte, error) { + d, err := b.o.Get(name) + if err != nil { + return nil, err + } + if d.Info.Name == "" || d.Info.Deleted { + return nil, index.ErrNoObject + } + return d.Data, nil +} + +func (b bundleObjects) Delete(_ context.Context, name string) error { return b.o.Delete(name) } + +func (b bundleObjects) Names(context.Context) ([]string, error) { + infos, err := b.o.List() + if err != nil { + return nil, err + } + names := []string{} + for _, i := range infos { + if !i.Deleted { + names = append(names, i.Name) + } + } + return names, nil +} + +func (b bundleObjects) Fill(context.Context) (int64, int64, error) { + f, err := b.o.Fill() + return int64(f.Bytes), int64(f.MaxBytes), err +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/views.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/views.go index 3b287093..ecd4431e 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/views.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/views.go @@ -37,9 +37,9 @@ func viewTools(getenv func(string) string, state func(string) views.Source, now func keptSource(name string) views.Source { return views.KeptSource{State: stdio.State(name)} } // configReadersTool is indexer_config_readers (design 32 *A value no longer read*; DLV-I10): read from one watched copy -// of the indexer's modules, manifests and config-candidates states, so an ask on each manifest-accepted reads no state +// of the indexer's manifests and config-candidates states and its view of the trunk (the newest catalog graph revision), so an ask on each manifest-accepted reads no state // whole again. -func configReadersTool(watched *stdio.WatchedStates) stdio.Tool { +func configReadersTool(watched *stdio.WatchedStates, modules views.Source) stdio.Tool { return stdio.Tool{Name: index.ConfigReadersTool, Description: "Who reads a module's config keys now (design 32): every value key and own secret any manifest of the module " + "the indexer keeps ever declared, each with its readers — the module itself where its trunk's manifest declares it, and " + @@ -53,13 +53,15 @@ func configReadersTool(watched *stdio.WatchedStates) stdio.Tool { return nil, fmt.Errorf("%s names the module", index.ConfigReadersTool) } trunk := map[string]meshcatalog.ModulesByModuleValue{} - mods, err := watched.Values("modules") + keys, err := modules.Keys() if err != nil { - return nil, fmt.Errorf("the modules state cannot be read: %w", err) + return nil, fmt.Errorf("the view of the trunk cannot be read: %w", err) } - for k, v := range mods { - if m, err := meshcatalogclient.DecodeModule(v); err == nil { - trunk[k] = m + for _, k := range keys { + if e, ok, err := modules.Get(k); err == nil && ok { + if m, err := meshcatalogclient.DecodeModule(e.Value); err == nil { + trunk[k] = m + } } } kept := map[string]meshcatalog.ManifestsByDigestValue{} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index 2d127854..a0860700 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -39,6 +39,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-delivery/sdk/graph" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/views" "mesh-delivery-indexer/internal/index" @@ -86,7 +87,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 }, func(ctx context.Context, js jetstream.JetStream) { // The states and the stream the controller makes from the indexer's manifest (old sections, ADR 0595 §11). for bucket, history := range map[string]uint8{index.PendingBucket: 1, index.VerdictsBucket: index.VerdictsHistory, - index.GraphBucket: 1, index.TakenBucket: 1, index.PackagesBucket: 1, + index.TakenBucket: 1, index.PackagesBucket: 1, index.SeatDefinitionsBucket: 1, index.RequirementsBucket: 1, findingsBucket: 1, index.HoldsBucket: 1, index.UnjudgedBucket: 1} { if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucket, History: history}); err != nil { t.Fatal(err) @@ -262,6 +263,13 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 named := held.WaitsFor get := func(bucket, key string) ([]byte, bool) { t.Helper() + if bucket == index.ModulesState { // the view of the trunk, from the newest catalog graph revision + e, err := revisionView{js: js}.Get(ctx, key) + if err != nil { + return nil, false + } + return e.Value(), true + } kv, err := js.KeyValue(ctx, bucket) if err != nil { t.Fatal(err) @@ -273,7 +281,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 return e.Value(), true } // Only what building and moving the named indexer needs is written. - raw, ok := get(index.ModulesBucket, ix) + raw, ok := get(index.ModulesState, ix) own, err := meshcatalogclient.DecodeModule(raw) if !ok || err != nil || own.Fp2 != named || own.Commit != m1.Merge { t.Fatalf("modules.%s = %+v (%v), want the named build %s at the merge", ix, own, err, named) @@ -284,11 +292,8 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 if _, ok := get(index.VerdictsBucket, own.Manifest); !ok { t.Errorf("the verdict on the indexer's manifest %s is not written", own.Manifest) } - if gk, _ := graph.Key(ix); !func() bool { _, ok := get(index.GraphBucket, gk); return ok }() { - t.Errorf("the indexer's key %s of the module graph is not written", gk) - } repoKey, _ := trunk.RepositoryKey(m1.Repository) - if _, ok := get(index.ModulesBucket, "a"); ok { + if _, ok := get(index.ModulesState, "a"); ok { t.Errorf("modules.a is written while the revision is pending") } if _, ok := get(index.TakenBucket, repoKey); ok { @@ -348,7 +353,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 break } } - if raw, ok := get(index.ModulesBucket, "a"); !ok || !strings.Contains(string(raw), m1.Merge) { + if raw, ok := get(index.ModulesState, "a"); !ok || !strings.Contains(string(raw), m1.Merge) { t.Errorf("modules.a = %s after the take-in at the start, want it at the merge %s", raw, m1.Merge) } if _, ok := get(index.TakenBucket, repoKey); !ok { @@ -357,7 +362,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 if len(w) != 1 || w[0].Key != groupKey || len(w[0].Keys) != 1 || w[0].Keys[0] != g1Key { t.Fatalf("indexer_pending after the start = %+v, want only the group gathered, its first member %s", w, g1Key) } - if _, ok := get(index.ModulesBucket, "c"); ok { + if _, ok := get(index.ModulesState, "c"); ok { t.Errorf("modules.c is written while its group is gathered") } @@ -376,8 +381,8 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 } var hasC, hasK bool for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && !(hasC && hasK && len(w) == 0); time.Sleep(50 * time.Millisecond) { - _, hasC = get(index.ModulesBucket, "c") - _, hasK = get(index.ModulesBucket, "k") + _, hasC = get(index.ModulesState, "c") + _, hasK = get(index.ModulesState, "k") w = waits() } if !hasC || !hasK || len(w) != 0 { @@ -471,7 +476,75 @@ var acceptedHeard atomic.Int64 // startIndexerRun starts the bus, the stream FORGE with the indexer's durable consumer, the buckets, lets put fill // mesh-controller_builds, and runs the program; restart starts it again. mesh-delivery-gate_source is never created. -func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, jetstream.KeyValue, func()) { +// revisionView reads the indexer's view of the trunk as the indexer itself does at a restart (milestone 47/G): the newest +// trunk revision its objects catalog-revisions keep, a module by its key. +type revisionView struct{ js jetstream.JetStream } + +func (v revisionView) Get(ctx context.Context, key string) (jetstream.KeyValueEntry, error) { + view, _, _, err := index.TrunkView(ctx, newBusObjects(v.js, testAccount, index.CatalogRevisionsObjects)) + if err != nil { + return nil, err + } + value, rev, found, err := view.Get(ctx, key) + if err != nil { + return nil, err + } + if !found { + return nil, jetstream.ErrKeyNotFound + } + return viewEntry{key: key, value: value, revision: rev}, nil +} + +// History is the module's value in every trunk revision the objects keep that holds it, oldest first. +func (v revisionView) History(ctx context.Context, key string) ([]jetstream.KeyValueEntry, error) { + o := newBusObjects(v.js, testAccount, index.CatalogRevisionsObjects) + names, err := o.Names(ctx) + if err != nil { + return nil, err + } + var revs []int64 + for _, n := range names { + if r, ok := index.TrunkRevisionOf(n); ok { + revs = append(revs, r) + } + } + slices.Sort(revs) + var out []jetstream.KeyValueEntry + for _, r := range revs { + _, entries, err := index.ReadRevision(ctx, o, fmt.Sprintf("trunk.%d", r)) + if err != nil { + return nil, err + } + for _, e := range entries { + if e.Module != key { + continue + } + data, err := meshcatalogclient.EncodeModule(index.ModuleValueOf(e)) + if err != nil { + return nil, err + } + out = append(out, viewEntry{key: key, value: data, revision: uint64(r)}) + } + } + return out, nil +} + +// viewEntry is one module's value of the view. +type viewEntry struct { + key string + value []byte + revision uint64 +} + +func (e viewEntry) Bucket() string { return index.CatalogRevisionsObjects } +func (e viewEntry) Key() string { return e.key } +func (e viewEntry) Value() []byte { return e.value } +func (e viewEntry) Revision() uint64 { return e.revision } +func (e viewEntry) Created() time.Time { return time.Time{} } +func (e viewEntry) Delta() uint64 { return 0 } +func (e viewEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut } + +func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, revisionView, func()) { dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i") if err != nil { t.Fatal(err) @@ -526,10 +599,11 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before . if err != nil { t.Fatal(err) } - modules, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.ModulesBucket, History: index.ModulesHistory}) - if err != nil { + // The indexer's objects catalog-revisions, which the grants part creates from its manifest's old objects section. + if _, err := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: newBusObjects(nil, testAccount, index.CatalogRevisionsObjects).bucket()}); err != nil { t.Fatal(err) } + modules := revisionView{js: js} // The bucket of the manifests, made by the controller from the manifest's state entry manifests (history 1, issue 1449). if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.ManifestsBucket, History: index.ManifestsHistory}); err != nil { t.Fatal(err) @@ -583,7 +657,7 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before . } // startIndexer is startIndexerRun for a test that never starts the program again. -func startIndexer(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, jetstream.KeyValue) { +func startIndexer(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, revisionView) { js, ctx, candidates, modules, _ := startIndexerRun(t, put, before...) return js, ctx, candidates, modules } @@ -612,8 +686,14 @@ func (l buildLog) end(t *testing.T, repo, module, path string) { // message on FORGE makes modules. within 20 s, the fingerprint of the merge's tree and the recorded inputs. func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9; I1, I3: the modules are the merged message's, no build record names them (issue 1851) // The bucket holds only build-agent's record, which no fingerprint reads since R4g (ADR 0530 §1). + var asks jetstream.KeyValue js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds buildLog) { builds.end(t, "novox/mesh-controller", "build-agent", "modules/build-agent") + }, func(ctx context.Context, js jetstream.JetStream) { + var err error + if asks, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.TakeInsBucket, History: 1}); err != nil { + t.Fatal(err) + } }) const merge = "89abcdef0123456789abcdef0123456789abcdef" data, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, @@ -636,6 +716,60 @@ func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9; I1, I3: the m if m.Repository != "novox/mesh-delivery" || m.Commit != merge || m.Path != "modules/a" || m.Manifest == "" || m.Fp != "" || m.None != "" { // 1606/AC2 t.Errorf("module = %+v within 20 s, want path modules/a, its manifest, and no value of today's form", m) } + // Milestone 47/G W4.13 (design 47, the indexer whole): the source revision taken in is one catalog graph revision kept + // as objects, said as catalog-revision on the seat's subject, every entry it lists stored once by digest. + objects := newBusObjects(js, testAccount, index.CatalogRevisionsObjects) + list, err := index.ListRevisions(ctx, objects, false) + if err != nil || len(list.Revisions) != 1 || list.Revisions[0].Name != "trunk.1" || !list.Revisions[0].Current || + list.Entries != list.Revisions[0].Entries || len(list.Unlisted) != 0 || list.Fill.Bytes == 0 { + t.Fatalf("revisions kept = %+v (%v): want trunk.1 alone, current, each entry it lists stored once", list, err) + } + events, err := js.Stream(ctx, "EVENTS") + if err != nil { + t.Fatal(err) + } + if msg, err := events.GetLastMsgForSubject(ctx, "mesh.seat.mesh-catalog.event."+meshcatalog.EventCatalogRevision); err != nil || + !strings.Contains(string(msg.Data), `"revision":1`) { + t.Errorf("catalog-revision said: %v (%v): want revision 1", msg, err) + } + // W4.12 through the process: a recompute by hand moves nothing and answers equal; a candidate's half dropped by hand + // leaves the trunk's revision and its entries. + if err := index.KeepRevision(ctx, objects, meshcatalog.CatalogRevision{Kind: "candidate", Group: "g1", Heads: "h1", Revision: 1, + Entries: []meshcatalog.EntryRef{{Module: "x", Digest: "dx"}}}, map[string][]byte{"dx": []byte(`{"module":"x"}`)}); err != nil { + t.Fatal(err) + } + answer := func(key string, value []byte) index.TakenIn { + t.Helper() + w, err := asks.Watch(ctx, key, jetstream.UpdatesOnly()) + if err != nil { + t.Fatal(err) + } + defer w.Stop() + if _, err := asks.Put(ctx, key, value); err != nil { + t.Fatal(err) + } + for e := range w.Updates() { + if e == nil { + continue + } + if a, err := index.DecodeTakeIn(e.Value()); err == nil && a.Done() { + return *a.Result + } + } + t.Fatalf("%s is not answered", key) + return index.TakenIn{} + } + key, value, _ := index.NewRecomputeAsk("a check", "ctl", time.Now()) + if r := answer(key, value); !r.Stored || !strings.Contains(r.Answer, "equal to the kept revision 1") { + t.Errorf("indexer_recompute answered %+v: want equal to the kept revision 1", r) + } + key, value, _ = index.NewDropCandidateGraphAsk("g1", "h1", "its gate event never came", "ctl", time.Now()) + if r := answer(key, value); !r.Stored || !slices.Equal(r.Dropped, []string{"entry.dx"}) { + t.Errorf("indexer_drop_candidate_graph answered %+v: want candidate.g1.h1 dropped with entry.dx", r) + } + if after, err := index.ListRevisions(ctx, objects, false); err != nil || len(after.Revisions) != 1 || after.Entries != list.Entries { + t.Errorf("revisions after the drop = %+v (%v): want trunk.1 and its entries alone", after, err) + } } // TestIndexerKeepsManifestsAtAMerge: a merged message of mesh-catalog carrying the real manifest of the module avahi and the @@ -960,17 +1094,16 @@ func TestIndexerJudgesAnUnreadableVerdictAgainAtStart(t *testing.T) { // 1923/AC if _, err := manifests.Put(ctx, mkey, mdata); err != nil { t.Fatal(err) } - stale, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "89abcdef0123456789abcdef0123456789abcdef", Path: "modules/avahi", + // The view the indexer starts from is the newest trunk revision its objects keep (milestone 47/G). + stale := meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "89abcdef0123456789abcdef0123456789abcdef", Path: "modules/avahi", Manifest: digest, None2: "the manifest cannot be read: claims node-build-go, which is a seat in the mesh's own namespace that it does not define", - Inputs2: &meshcatalog.ModulesByModuleValueInputs2{Tree: "tree-avahi", Locks: map[string]string{"go.sum": goSum}}}) + Inputs2: &meshcatalog.ModulesByModuleValueInputs2{Tree: "tree-avahi", Locks: map[string]string{"go.sum": goSum}}} + rev, objects, err := index.CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk"}, + map[string]meshcatalog.CatalogEntry{"avahi": index.CatalogEntryOf("avahi", stale, catalogue.Manifest{}, graph.Node{None: stale.None2})}) if err != nil { t.Fatal(err) } - held, err := js.KeyValue(ctx, index.ModulesBucket) - if err != nil { - t.Fatal(err) - } - if _, err := held.Put(ctx, "avahi", stale); err != nil { + if err := index.KeepRevision(ctx, newBusObjects(js, testAccount, index.CatalogRevisionsObjects), rev, objects); err != nil { t.Fatal(err) } kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.ToolchainsBucket("mesh-build-go"), History: 1}) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 8bb6284f..2a3c7452 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -375,9 +375,21 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } - modules, err := process.EssentialBucket(ctx, js, "indexer", index.ModulesBucket, "write a module's fingerprint") - if err != nil { - return err + // The view of the trunk is the newest trunk revision of the catalog graph the objects keep (milestone 47/G, ADR 0656 §6: + // one form of the fact), read once here and kept as a revision after every write; objects not on the bus yet, or none + // kept, start it empty, and the first start's catch-up takes the trunk in once the gate's source is read (catchUp). + revisions := newBusObjects(js, cred, index.CatalogRevisionsObjects) + modules, graph, at, err := index.TrunkView(ctx, revisions) + switch { + case errors.Is(err, process.ErrNotYet): + logger.Printf("the view of the trunk starts empty: %v; the trunk is taken in once they are there and the gate's source is read", err) + modules, graph = index.MemoryStore(), index.MemoryStore() + case err != nil: + return fmt.Errorf("the view of the trunk cannot be read from the newest catalog graph revision: %w", err) + case at == 0: + logger.Printf("the view of the trunk starts empty: %s keeps no trunk revision yet; the trunk is taken in once the gate's source is read", index.CatalogRevisionsObjects) + default: + logger.Printf("the view of the trunk read from trunk revision %d", at) } manifests, err := process.EssentialBucket(ctx, js, "indexer", index.ManifestsBucket, "keep a module's manifest") if err != nil { @@ -413,8 +425,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // What each consumer's go.sum requires of each mesh Go module (DLV-I7): a state this delivery brings, bound when it // appears; until then indexer_requirements answers none, and the builder's floor waits on its answer. requirements := process.AwaitBucket(ctx, js, conn, index.RequirementsBucket, "what consumers require is not kept", logger.Printf) - // The module graph (ADR 0603 §2): a state this delivery brings, bound when it appears; until then no graph is written. - graph := process.AwaitBucket(ctx, js, conn, index.GraphBucket, "no module graph is written", logger.Printf) + // The module graph (ADR 0603 §2) is held in memory, read with the view: each revision's entries carry its nodes. ownBuildState := process.AwaitBucket(ctx, js, conn, index.OwnBuildBucket, "which build this indexer runs is not kept", logger.Printf) // The manifests a take-in wrote and no judge pass judged yet (issue 1966): a state this delivery brings, bound when it // appears; until then a take-in's new manifests are kept for their judging in its memory alone. @@ -427,14 +438,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro seatBus := &seatBus{defs: awaitedSource{ctx, seatDefs}, marks: awaitedSource{ctx, marks}} seatReader, saySeats := followSeatList(seatBus, saySeatDefinition(js, cred), logger.Printf) readWhenBound(seatReader, seatDefs, marks) - ix := &index.Indexer{Findings: awaitedStore{b: findings}, OldSections: awaitedStore{b: oldSections}, Unjudged: awaitedStore{b: unjudged}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known, + ix := &index.Indexer{Findings: awaitedStore{b: findings}, OldSections: awaitedStore{b: oldSections}, Unjudged: awaitedStore{b: unjudged}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: modules, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known, Verdicts: awaitedStore{b: verdicts}, SayRefused: sayRefused(conn, cred, logger.Printf), SayAccepted: sayAccepted(conn, cred), Predefined: awaitedStore{b: predefined}, ConfigCandidates: awaitedStore{b: configCandidates}, Logf: logger.Printf, Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call), ReadOpen: openVia(calls.Call), Packages: awaitedStore{b: packages, js: js, name: index.PackagesBucket}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call), - OwnBuild: ownBuildVia(calls.Call, cred.Node), OpenTries: openTriesVia(calls.Call), OwnBuildState: awaitedStore{b: ownBuildState}, TryRecovery: awaitedStore{b: tryRecovery}, Graph: awaitedStore{b: graph}, + OwnBuild: ownBuildVia(calls.Call, cred.Node), OpenTries: openTriesVia(calls.Call), OwnBuildState: awaitedStore{b: ownBuildState}, TryRecovery: awaitedStore{b: tryRecovery}, Graph: graph, SeatDefinitions: awaitedStore{b: seatDefs}, ReadFile: readFileVia(calls.Call), SaySeatDefinition: saySeats, - Message: messageVia(calls.CallKept), // a kept call: a read, called again through a transport fault (ADR 0668 §1) + Message: messageVia(calls.CallKept), Revisions: revisions, // Message a kept call: a read, called again through a transport fault (ADR 0668 §1) + SayCatalogRevision: sayCatalogRevision(conn, cred), + SayFill: sayFill(conn, cred), // The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear; // until then every merge is taken in at once, as before. Pending: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.PendingBucket, "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)}, @@ -744,10 +757,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // said up to; a first view (the tools bundle) answers at that mark. startViews(ctx, js, cred, marks, findings, - map[string]readyOf{"manifests": boundNow(manifests), "modules": boundNow(modules), "candidates": boundNow(candidates), + map[string]readyOf{"manifests": boundNow(manifests), "candidates": boundNow(candidates), index.VerdictsState: awaited(verdicts), index.PredefinedState: awaited(predefined), index.ConfigCandidatesState: awaited(configCandidates), index.PackagesState: awaited(packages), - graphState: awaited(graph), index.HeardState: awaited(heard.b), "taken": awaited(taken), "findings": awaited(findings), + index.HeardState: awaited(heard.b), "taken": awaited(taken), "findings": awaited(findings), index.SeatDefinitionsState: awaited(seatDefs)}, errs, logger.Printf) cc, err := process.HearEvents(cons, func(m jetstream.Msg) { handle(ctx, m, ix, heard, logger) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go index 1450b161..11264cd3 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go @@ -79,6 +79,9 @@ func (m *memStore) Delete(_ context.Context, k string, _ uint64) error { delete( const head = "0123456789abcdef0123456789abcdef01234567" +// testAccount is the bus account the tests run the indexer as. +var testAccount = process.Account{Node: "n1", Module: "mesh-delivery-indexer"} + func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part) // Review round 2: the indexer's views emitter writes its finding view-unsaid (Emitter.Findings set). if em := viewsEmitter(context.Background(), nil, process.Account{}, nil, &process.AwaitedBucket{}, nil, t.Logf); em.Findings == nil { @@ -143,6 +146,10 @@ func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part) t.Errorf("answered leaves %v, presence asked %v", held.vals, asked) } }) + // Milestone 47/G: the object buckets are the module's own by the name its account runs as, never a fixed name. + if b := newBusObjects(nil, process.Account{Module: "renamed-indexer"}, index.CatalogRevisionsObjects).bucket(); b != "renamed-indexer_catalog-revisions" { + t.Errorf("the catalog graph revisions open as %s, want the account's module's", b) + } t.Run("not a forge message: terminated and logged", func(t *testing.T) { // 1238 (never redelivered) var logs bytes.Buffer st := &memStore{vals: map[string][]byte{}} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/objects.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/objects.go new file mode 100644 index 00000000..7b250e47 --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/objects.go @@ -0,0 +1,166 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "sync" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-indexer/internal/index" +) + +// busObjects is one of the indexer's own object buckets in the old `objects` section (novox/hq ADR 0619 §2), which the +// grants part creates: it is opened at its first use and kept; until the bucket exists every call is an error saying so, +// and the next call opens it again (no timer: the next revision or tool call asks). +type busObjects struct { + js jetstream.JetStream + // module is the module this program runs as, read from its bus account (process.Account.Module), never a fixed name. + module string + local string + mu sync.Mutex + o jetstream.ObjectStore +} + +// newBusObjects is the object bucket of this local name of the module the account runs as. +func newBusObjects(js jetstream.JetStream, cred process.Account, local string) *busObjects { + return &busObjects{js: js, module: cred.Module, local: local} +} + +// bucket is the bus's name of the object bucket: the module's own name and the local name. +func (b *busObjects) bucket() string { return catalogue.ObjectsBucket(b.module, b.local) } + +func (b *busObjects) open(ctx context.Context) (jetstream.ObjectStore, error) { + b.mu.Lock() + defer b.mu.Unlock() + if b.o != nil { + return b.o, nil + } + o, err := b.js.ObjectStore(ctx, b.bucket()) + if errors.Is(err, jetstream.ErrBucketNotFound) || errors.Is(err, jetstream.ErrStreamNotFound) { + return nil, fmt.Errorf("the object bucket %s is not on the bus yet (the grants part creates it from the manifest's objects): %w", b.local, process.ErrNotYet) + } + if err != nil { + return nil, fmt.Errorf("the object bucket %s is not open: %w", b.local, err) + } + b.o = o + return o, nil +} + +func (b *busObjects) Put(ctx context.Context, name string, data []byte) error { + o, err := b.open(ctx) + if err != nil { + return err + } + _, err = o.Put(ctx, jetstream.ObjectMeta{Name: name}, bytes.NewReader(data)) + return err +} + +func (b *busObjects) Get(ctx context.Context, name string) ([]byte, error) { + o, err := b.open(ctx) + if err != nil { + return nil, err + } + data, err := o.GetBytes(ctx, name) + if errors.Is(err, jetstream.ErrObjectNotFound) { + return nil, index.ErrNoObject + } + return data, err +} + +func (b *busObjects) Delete(ctx context.Context, name string) error { + o, err := b.open(ctx) + if err != nil { + return err + } + if err := o.Delete(ctx, name); err != nil && !errors.Is(err, jetstream.ErrObjectNotFound) { + return err + } + return nil +} + +func (b *busObjects) Names(ctx context.Context) ([]string, error) { + o, err := b.open(ctx) + if err != nil { + return nil, err + } + infos, err := o.List(ctx) + if errors.Is(err, jetstream.ErrNoObjectsFound) { + return []string{}, nil + } + if err != nil { + return nil, err + } + names := make([]string, 0, len(infos)) + for _, i := range infos { + names = append(names, i.Name) + } + return names, nil +} + +func (b *busObjects) Fill(ctx context.Context) (int64, int64, error) { + o, err := b.open(ctx) + if err != nil { + return 0, 0, err + } + st, err := o.Status(ctx) + if err != nil { + return 0, 0, err + } + info, err := b.js.Stream(ctx, "OBJ_"+st.Bucket()) + if err != nil { + return int64(st.Size()), 0, err + } + return int64(st.Size()), info.CachedInfo().Config.MaxBytes, nil +} + +// sayCatalogRevision publishes the seat mesh-catalog's event catalog-revision under the duplicate id of its kind and +// number, so a revision said twice is heard once (design 47, Two graphs). +func sayCatalogRevision(conn *nats.Conn, cred process.Account) func(context.Context, meshcatalog.CatalogRevisionEvent) error { + return func(_ context.Context, e meshcatalog.CatalogRevisionEvent) error { + data, err := json.Marshal(e) + if err != nil { + return err + } + msg := nats.NewMsg(seatEventSubject(meshcatalog.EventCatalogRevision)) + id := fmt.Sprintf("%s-%s-%d%s", meshcatalog.EventCatalogRevision, e.Kind, e.Revision, e.Heads) + msg.Header.Set("x-event-id", id) + msg.Header.Set("x-source", cred.Module) + msg.Header.Set("x-node", cred.Node) + msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano)) + msg.Header.Set("content-type", "application/json") + msg.Header.Set(nats.MsgIdHdr, id) + msg.Data = data + return conn.PublishMsg(msg) + } +} + +// sayFill publishes the seat mesh-catalog's event filling, full or room of the catalog graph revisions under the duplicate +// id given (milestone 47/G W4.6), so one said twice is heard once. +func sayFill(conn *nats.Conn, cred process.Account) func(context.Context, string, string, meshcatalog.Fill) error { + return func(_ context.Context, event, id string, f meshcatalog.Fill) error { + data, err := json.Marshal(f) + if err != nil { + return err + } + msg := nats.NewMsg(seatEventSubject(event)) + msg.Header.Set("x-event-id", id) + msg.Header.Set("x-source", cred.Module) + msg.Header.Set("x-node", cred.Node) + msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano)) + msg.Header.Set("content-type", "application/json") + msg.Header.Set(nats.MsgIdHdr, id) + msg.Data = data + return conn.PublishMsg(msg) + } +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go index af5f2e8a..3f010eba 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go @@ -15,6 +15,7 @@ import ( type takerIn interface { TakeIn(ctx context.Context, module string) index.TakenIn TakeInRepository(ctx context.Context, repository string) index.TakenIn + CatchUp(ctx context.Context) index.TakenIn JudgeAgain(ctx context.Context, module, digest string) index.TakenIn SayAcceptedAgain(ctx context.Context, module string) index.TakenIn SeatDefinitionsAgain(ctx context.Context, module string, pull int, head string) index.TakenIn @@ -25,6 +26,8 @@ type takerIn interface { DropTryCandidate(ctx context.Context, id, by, why string) index.TakenIn RecoverTriesByHand(ctx context.Context, by, why string) index.TakenIn FingerprintsAgain(ctx context.Context, module string) index.TakenIn + RecomputeByHand(ctx context.Context, by, why string) index.TakenIn + DropCandidateGraph(ctx context.Context, group, heads, by, why string) index.TakenIn } // takenIn answers one value of the takeins state (the tool indexer_take_in, issue 1945): a value not yet answered takes the @@ -43,7 +46,13 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream return } asked, r := "take-in", index.TakenIn{} - if a.Fingerprints { + if a.Recompute { + asked, r = "recompute of the catalog graph", ix.RecomputeByHand(ctx, a.By, a.Why) + a.Module = r.Module + } else if a.CandidateGroup != "" && a.Drop { + asked, r = "drop of a candidate's catalog half", ix.DropCandidateGraph(ctx, a.CandidateGroup, a.CandidateHeads, a.By, a.Why) + a.Module = r.Module + } else if a.Fingerprints { asked, r = "computing of fingerprints again", ix.FingerprintsAgain(ctx, a.Module) } else if a.RecoverTries { asked, r = "recovery of the tries", ix.RecoverTriesByHand(ctx, a.By, a.Why) @@ -60,6 +69,9 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream } else if a.Pending != "" { asked, r = "hand take-in of what waits", ix.TakeInPendingByHand(ctx, a.Pending, a.By, a.Why) a.Module = a.Pending + } else if a.CatchUp { + asked, r = "catch-up of every repository", ix.CatchUp(ctx) + a.Module = r.Module } else if a.Repository != "" { asked, r = "take-in of a repository", ix.TakeInRepository(ctx, a.Repository) a.Module = a.Repository diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 54463a6b..5fcfa42b 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -97,6 +97,14 @@ type Indexer struct { ReadFile func(ctx context.Context, repository, sha, path string) ([]byte, error) // SaySeatDefinition says seat-definition-changed (seatdefs.go); nil says nothing. SaySeatDefinition func(ctx context.Context, c SeatDefinitionChanged) error + // Revisions is the object bucket catalog-revisions (catalog_revisions.go; design 47, Two graphs); nil keeps no + // catalog graph revision. + Revisions Objects + // SayCatalogRevision says the seat mesh-catalog's event catalog-revision for a revision kept; nil says nothing. + SayCatalogRevision func(ctx context.Context, e meshcatalog.CatalogRevisionEvent) error + // SayFill says the seat mesh-catalog's event filling, full or room of the catalog graph revisions, under the duplicate + // id given (W4.6). + SayFill func(ctx context.Context, event, id string, f meshcatalog.Fill) error // Graph is the module graph's state (GraphBucket, graph.go; novox/hq ADR 0603 §2); nil writes no graph. Graph Store // Files asks the forge for every blob under each directory at a commit, by directory then path relative to it, each @@ -164,12 +172,15 @@ type Indexer struct { failedAt map[string]string // the head of a repository whose tree lacks a directory asked, by repository: not asked again until the head changes // OwnBuild answers the fingerprint of the build this program runs, from the node-engine's seat; nil knows none. OwnBuild func(ctx context.Context) (string, error) - underSeats bool // a dry run inside judge.WithSeats: its judge passes do not hold the seat list (verdicts.go) - judgeMu sync.Mutex // one judge pass at a time (verdicts.go) - ownMu sync.Mutex // guards ownFP - ownFP string // the fingerprint of the build this program runs, fixed once at the start (FixOwnBuild) - ownFixed bool // FixOwnBuild asked already: it asks once - ownSince time.Time // when the build was asked and not known, with ownWhy + underSeats bool // a dry run inside judge.WithSeats: its judge passes do not hold the seat list (verdicts.go) + judgeMu sync.Mutex // one judge pass at a time (verdicts.go) + ownMu sync.Mutex // guards ownFP + keepMu sync.Mutex // one catalog graph revision kept at a time (keepEntries) + catchingUp atomic.Bool // the first start's catch-up runs: its merges keep no revision until it ends (CatchUp) + catchUpWG sync.WaitGroup // the first start's catch-up in the background (kickCatchUp), for tests to wait on + ownFP string // the fingerprint of the build this program runs, fixed once at the start (FixOwnBuild) + ownFixed bool // FixOwnBuild asked already: it asks once + ownSince time.Time // when the build was asked and not known, with ownWhy ownWhy string // OwnBuildState keeps which build this process knows it runs, or since when and why not (FixOwnBuild); nil keeps none. OwnBuildState Store diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_graph.go b/modules/mesh-delivery-indexer/internal/index/catalog_graph.go new file mode 100644 index 00000000..d1e6f001 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/catalog_graph.go @@ -0,0 +1,141 @@ +package index + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "maps" + "reflect" + "slices" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/sdk/graph" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// The catalog graph (novox/hq ADR 0656; design 47, *Two graphs, one source of truth*): at each revision one entry per +// module, as its manifest declares it, in the seat mesh-catalog's layout (its definition's CatalogEntry), and the +// revision a list of its entries' digests, each entry kept once as an object named by its digest +// (catalog-revisions: trunk., candidate.., entry.). + +// CatalogEntryLayout is the layout version an entry and a revision are written in. +const CatalogEntryLayout = 1 + +// CatalogEntryOf is the catalog graph entry of one module: the module graph's node of it as a recompute computed it +// (its build, provision and verb edges, the verb edges left out on a cycle, ADR 0614), with where the trunk holds it, +// its manifest's digest and verdict, the main package its process runs (MainOf), the package versions it builds against +// and what the indexer computes its next revision from (Inputs: the lead's ruling on milestone 47/G finding 1), so that +// ModuleValueOf gives the module's value back from the entry alone. The acting rules are the revision's to add. +func CatalogEntryOf(module string, cur meshcatalog.ModulesByModuleValue, m catalogue.Manifest, n graph.Node) meshcatalog.CatalogEntry { + e := meshcatalog.CatalogEntry{V: CatalogEntryLayout, Module: module, Repository: cur.Repository, Path: cur.Path, + Commit: cur.Commit, Manifest: cur.Manifest, Verdict: cur.Verdict, Fp: n.FP, None: n.None, Bases: n.Bases, + Packages: maps.Clone(cur.Packages), Provides: n.Provides, Requires: n.Requires, Awaits: n.Awaits, Invokes: n.Invokes, + Serves: n.Serves, LeftOut: n.LeftOut, Main: MainOf(m)} + if n.FP == "" && n.None == "" { + // A module the recompute computed no node for (its manifest cannot be read) keeps what its value holds. + e.Fp, e.None = cur.Fp2, cur.None2 + } + in := meshcatalog.EntryInputs{Toolchains: maps.Clone(cur.Toolchains), Paths: maps.Clone(cur.Paths), Problems: cur.Problems, + Warnings: cur.Warnings, Unknown: cur.Unknown, Src: cur.Src, FpSrc: cur.FpSrc} + if cur.Inputs2 != nil { + from := meshcatalog.FingerprintInputs(*cur.Inputs2) + in.From = &from + } + if !reflect.ValueOf(in).IsZero() { + e.Inputs = &in + } + for language, t := range n.Toolchains { + if e.Toolchains == nil { + e.Toolchains = map[string]meshcatalog.BuildEdge{} + } + e.Toolchains[language] = meshcatalog.BuildEdge{Module: t.Module, Version: t.Version, Recipe: t.Recipe} + } + return e +} + +// ModuleValueOf is the module's value as the indexer computes with it, read back from its catalog graph entry: the +// inverse of CatalogEntryOf, by which the indexer's view of the trunk is the newest trunk revision's entries (the lead's +// ruling on milestone 47/G finding 1; ADR 0656 §6: one form of the fact). The old line's fields (the old-form +// fingerprint, the reader needs) are not kept. +func ModuleValueOf(e meshcatalog.CatalogEntry) meshcatalog.ModulesByModuleValue { + v := meshcatalog.ModulesByModuleValue{V: keys.Version, Repository: e.Repository, Path: e.Path, Commit: e.Commit, Manifest: e.Manifest, + Verdict: e.Verdict, Fp2: e.Fp, None2: e.None, Packages: maps.Clone(e.Packages)} + if in := e.Inputs; in != nil { + v.Toolchains, v.Paths, v.Problems, v.Warnings, v.Unknown = maps.Clone(in.Toolchains), maps.Clone(in.Paths), in.Problems, in.Warnings, in.Unknown + v.Src, v.FpSrc = in.Src, in.FpSrc + if in.From != nil { + from := meshcatalog.ModulesByModuleValueInputs2(*in.From) + v.Inputs2 = &from + } + } + return v +} + +// NodeOf is the node of the module graph an entry carries, as CatalogEntryOf took it in: the graph the indexer holds in +// memory is read back from the newest trunk revision at its start (TrunkView), so a revision kept before its first +// recompute loses no edge. +func NodeOf(e meshcatalog.CatalogEntry) graph.Node { + n := graph.Node{V: graph.Version, FP: e.Fp, None: e.None, Repository: e.Repository, Path: e.Path, Bases: e.Bases, + Provides: e.Provides, Requires: e.Requires, Awaits: e.Awaits, Invokes: e.Invokes, Serves: e.Serves, LeftOut: e.LeftOut} + if len(e.Packages) > 0 { + n.Packages = slices.Sorted(maps.Keys(e.Packages)) + } + for language, t := range e.Toolchains { + if n.Toolchains == nil { + n.Toolchains = map[string]graph.Toolchain{} + } + n.Toolchains[language] = graph.Toolchain{Module: t.Module, Version: t.Version, Recipe: t.Recipe} + } + return n +} + +// EncodeEntry is an entry's bytes and their sha256 in hex, the name of its object: the same entry in two revisions is +// one object (design 47, *An unchanged entry is stored once*). +func EncodeEntry(e meshcatalog.CatalogEntry) (string, []byte, error) { + data, err := json.Marshal(e) + if err != nil { + return "", nil, fmt.Errorf("the entry of %s: %w", e.Module, err) + } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]), data, nil +} + +// CatalogRevisionOf is a revision of entries, listed by module name with each entry's digest, and the entry objects it +// lists by digest, each once. The caller numbers the revision, names its kind (trunk or candidate) and what it was read +// from. +func CatalogRevisionOf(rev meshcatalog.CatalogRevision, entries map[string]meshcatalog.CatalogEntry) (meshcatalog.CatalogRevision, map[string][]byte, error) { + rev.V, rev.Entries = CatalogEntryLayout, make([]meshcatalog.EntryRef, 0, len(entries)) + objects := map[string][]byte{} + for _, module := range slices.Sorted(maps.Keys(entries)) { + digest, data, err := EncodeEntry(entries[module]) + if err != nil { + return meshcatalog.CatalogRevision{}, nil, err + } + objects[digest] = data + rev.Entries = append(rev.Entries, meshcatalog.EntryRef{Module: module, Digest: digest}) + } + return rev, objects, nil +} + +// MainOf is the Go main package the check's first tier builds when the module acts (the seat mesh-catalog's entry +// field main): the from of the first Go artifact a process resource of the manifest runs, as the manifest names it; +// empty when it runs none. +func MainOf(m catalogue.Manifest) string { + if m.Build == nil { + return "" + } + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) != "process" { + continue + } + name := fmt.Sprint(r["artifact"]) + for _, a := range m.Build.Artifacts { + if a.Name == name && a.Language == "go" && a.From != "" { + return a.From + } + } + } + return "" +} diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go new file mode 100644 index 00000000..b044b4bc --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -0,0 +1,483 @@ +package index + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "strconv" + "strings" + "time" + + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/sdk/graph" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/views" +) + +// The indexer's object bucket catalog-revisions (design 47, Two graphs; novox/hq ADR 0619 §2: declared in the old +// `objects` section, its cap there): a revision is its list of entries' digests under trunk. or +// candidate.., and each entry one object entry., written once and shared by every revision that +// lists it. + +// CatalogRevisionsObjects is the bucket's local name in the indexer's manifest. +const CatalogRevisionsObjects = "catalog-revisions" + +// ErrNoObject is an object the bucket does not hold. +var ErrNoObject = errors.New("no such object") + +// Objects is an object bucket as the indexer keeps its revisions in it. +type Objects interface { + // Put stores data whole under name, replacing one of that name. + Put(ctx context.Context, name string, data []byte) error + // Get is the object's bytes; ErrNoObject when it holds none of that name. + Get(ctx context.Context, name string) ([]byte, error) + // Delete removes the object; one it does not hold is no error. + Delete(ctx context.Context, name string) error + // Names is every object's name; none is an empty list. + Names(ctx context.Context) ([]string, error) + // Fill is the bucket's bytes used and its cap in bytes. + Fill(ctx context.Context) (used, capacity int64, err error) +} + +// EntryObject is the object name of the entry of that digest. +func EntryObject(digest string) string { return "entry." + digest } + +// RevisionObject is the object name of a revision: trunk., or candidate.. for a candidate, +// which keeps its newest revision only. +func RevisionObject(rev meshcatalog.CatalogRevision) string { + if rev.Kind == "candidate" { + return "candidate." + rev.Group + "." + rev.Heads + } + return "trunk." + strconv.FormatInt(rev.Revision, 10) +} + +// TrunkRevisionOf is the revision number a trunk object's name holds, and whether it is one. +func TrunkRevisionOf(name string) (int64, bool) { + n, ok := strings.CutPrefix(name, "trunk.") + if !ok { + return 0, false + } + r, err := strconv.ParseInt(n, 10, 64) + return r, err == nil +} + +// KeepRevision stores a revision as CatalogRevisionOf made it: each entry object the bucket does not hold yet, then the +// revision's list, so a list is never kept whose entries are not; an entry already held is not written again. +func KeepRevision(ctx context.Context, o Objects, rev meshcatalog.CatalogRevision, entries map[string][]byte) error { + held, err := o.Names(ctx) + if err != nil { + return fmt.Errorf("the objects %s: %w", CatalogRevisionsObjects, err) + } + have := make(map[string]bool, len(held)) + for _, n := range held { + have[n] = true + } + for _, ref := range rev.Entries { + name := EntryObject(ref.Digest) + if have[name] { + continue + } + data, ok := entries[ref.Digest] + if !ok { + return fmt.Errorf("revision %s lists the entry of %s (%s), which it does not carry", RevisionObject(rev), ref.Module, ref.Digest) + } + if err := o.Put(ctx, name, data); err != nil { + return fmt.Errorf("the entry of %s in %s: %w", ref.Module, CatalogRevisionsObjects, err) + } + have[name] = true + } + list, err := json.Marshal(rev) + if err != nil { + return fmt.Errorf("revision %s: %w", RevisionObject(rev), err) + } + if err := o.Put(ctx, RevisionObject(rev), list); err != nil { + return fmt.Errorf("revision %s in %s: %w", RevisionObject(rev), CatalogRevisionsObjects, err) + } + return nil +} + +// ReadRevision is the revision kept under name and its entries whole, in the revision's order; an entry it lists that +// the bucket lost is an error naming it, never an entry left out. +func ReadRevision(ctx context.Context, o Objects, name string) (meshcatalog.CatalogRevision, []meshcatalog.CatalogEntry, error) { + var rev meshcatalog.CatalogRevision + data, err := o.Get(ctx, name) + if err != nil { + return rev, nil, fmt.Errorf("revision %s: %w", name, err) + } + if err := json.Unmarshal(data, &rev); err != nil { + return rev, nil, fmt.Errorf("revision %s: %w", name, err) + } + entries := make([]meshcatalog.CatalogEntry, 0, len(rev.Entries)) + for _, ref := range rev.Entries { + data, err := o.Get(ctx, EntryObject(ref.Digest)) + if err != nil { + return rev, nil, fmt.Errorf("revision %s, the entry of %s (%s): %w", name, ref.Module, ref.Digest, err) + } + var e meshcatalog.CatalogEntry + if err := json.Unmarshal(data, &e); err != nil { + return rev, nil, fmt.Errorf("revision %s, the entry of %s: %w", name, ref.Module, err) + } + entries = append(entries, e) + } + return rev, entries, nil +} + +// newestTrunk is the newest trunk revision the bucket keeps, its list only; none is a zero revision and false. +func newestTrunk(ctx context.Context, o Objects) (meshcatalog.CatalogRevision, bool, error) { + names, err := o.Names(ctx) + if err != nil { + return meshcatalog.CatalogRevision{}, false, err + } + var newest int64 + for _, n := range names { + if r, ok := TrunkRevisionOf(n); ok && r > newest { + newest = r + } + } + if newest == 0 { + return meshcatalog.CatalogRevision{}, false, nil + } + data, err := o.Get(ctx, "trunk."+strconv.FormatInt(newest, 10)) + if err != nil { + return meshcatalog.CatalogRevision{}, false, err + } + var rev meshcatalog.CatalogRevision + if err := json.Unmarshal(data, &rev); err != nil { + return rev, false, fmt.Errorf("trunk.%d: %w", newest, err) + } + return rev, true, nil +} + +// changedEntries are the modules whose entry differs between two revisions' lists, one gone or added included, by name. +func changedEntries(before, after []meshcatalog.EntryRef) []string { + was := map[string]string{} + for _, r := range before { + was[r.Module] = r.Digest + } + var changed []string + for _, r := range after { + if d, ok := was[r.Module]; !ok || d != r.Digest { + changed = append(changed, r.Module) + } + delete(was, r.Module) + } + for m := range was { + changed = append(changed, m) + } + slices.Sort(changed) + return changed +} + +// keepTrunkRevision keeps the catalog graph revision a merge's or a recompute's module graph makes (design 47, Two +// graphs: a revision for each source revision taken in, and for a fingerprint moved with no merge), numbered after the +// newest kept, and says catalog-revision; a graph whose entries are the newest revision's makes none. A revision not +// kept is said in the log and returned, so the take-in that made it is not said taken. +func (ix *Indexer) keepTrunkRevision(ctx context.Context, edges map[string]graphEdges, whole graph.Graph) error { + if ix.Revisions == nil { + return nil + } + // Each entry is the module's value as this recompute left it in modules (its fingerprint, toolchains, package versions + // and inputs written), so the revision holds what the next one is computed from. + held, err := ix.heldModules(ctx) + if err != nil { + return err + } + // Every module the view holds is an entry, one the recompute computed no edges for (a manifest it cannot read) too. + entries := make(map[string]meshcatalog.CatalogEntry, len(held)) + for name, cur := range held { + entries[name] = CatalogEntryOf(name, cur, edges[name].manifest, whole[name]) + } + for name, e := range edges { + if _, ok := held[name]; !ok { + entries[name] = CatalogEntryOf(name, e.cur, e.manifest, whole[name]) + } + } + return ix.keepEntries(ctx, entries) +} + +// KeepView keeps the view of the trunk as a trunk revision after a write to it outside a recompute (the lead's ruling on +// milestone 47/G: the judge pass's verdicts, a build's reported imports, the source stamp, the modules the trunk no longer +// holds, the indexer's own entry a held revision writes): each entry is the module's value in the view with its node of +// the module graph in memory, so what the view holds is never lost at a restart. A view whose entries are the newest +// revision's keeps none. +func (ix *Indexer) KeepView(ctx context.Context) error { + if ix.Revisions == nil || ix.Modules == nil || ix.catchingUp.Load() { + return nil + } + held, err := ix.heldModules(ctx) + if err != nil { + return err + } + nodes := graph.Graph{} + if ix.Graph != nil { + keys, err := ix.Graph.Keys(ctx) + if err != nil && !errors.Is(err, process.ErrNotYet) { + return fmt.Errorf("the module graph: %w", err) + } + for _, k := range keys { + if value, _, found, err := ix.Graph.Get(ctx, k); err == nil && found { + if n, err := graph.Decode(value); err == nil { + nodes[k] = n + } + } + } + } + entries := make(map[string]meshcatalog.CatalogEntry, len(held)) + for name, cur := range held { + var m catalogue.Manifest + if cur.Manifest != "" { + if _, read, err := ix.manifestRaw(ctx, cur.Manifest); err == nil { + m = read + } + } + entries[name] = CatalogEntryOf(name, cur, m, nodes[name]) + } + return ix.keepEntries(ctx, entries) +} + +// keepEntries keeps entries as the trunk revision after the newest, says catalog-revision, and keeps none when no entry +// moved; one keep at a time, so two never take the same number. +func (ix *Indexer) keepEntries(ctx context.Context, entries map[string]meshcatalog.CatalogEntry) error { + if ix.catchingUp.Load() { + return nil // the catch-up of a first start keeps its one revision at its end (CatchUp) + } + ix.keepMu.Lock() + defer ix.keepMu.Unlock() + ix.mu.Lock() + src := int64(ix.source.rev) + ix.mu.Unlock() + newest, any, err := newestTrunk(ctx, ix.Revisions) + if errors.Is(err, process.ErrNotYet) { + ix.logf("catalog graph: no revision is kept: its objects %s are not on the bus yet", CatalogRevisionsObjects) + return nil + } + if err != nil { + ix.logf("catalog graph: no revision is kept: the newest is not read: %v", err) + return fmt.Errorf("the newest catalog graph revision: %w", err) + } + at := ix.now().UTC() + rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: newest.Revision + 1, Kind: "trunk", Source: src, At: &at}, entries) + if err != nil { + return err + } + changed := changedEntries(newest.Entries, rev.Entries) + if (any && len(changed) == 0) || (!any && len(entries) == 0) { + return nil // nothing moved; an empty first revision is never kept, so the first start's catch-up still runs + } + used, capacity, fillErr := ix.Revisions.Fill(ctx) + if fillErr == nil && capacity > 0 && used+newBytes(ctx, ix.Revisions, rev, objects) > capacity { + // W4.6 (design 47, Two graphs): a full store refuses the next revision, said once per source revision, and the + // take-in that made it holds its pending merge with the reason objects full until room is said. + refused := fmt.Sprintf("the next revision; source revision %d held pending with the reason %s", src, HeldObjectsFull) + ix.logf("catalog graph: revision %d of source revision %d is not kept: %s is full (%d of %d bytes)", rev.Revision, src, CatalogRevisionsObjects, used, capacity) + ix.sayFill(ctx, meshcatalog.EventFull, fmt.Sprintf("%d", src), meshcatalog.Fill{Bytes: used, Cap: capacity, Refused: refused}) + return fmt.Errorf("revision %d of source revision %d: %w", rev.Revision, src, ErrObjectsFull) + } + if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil { + ix.logf("catalog graph: revision %d of source revision %d is not kept: %v", rev.Revision, src, err) + return err + } + if after, capacity, err := ix.Revisions.Fill(ctx); fillErr == nil && err == nil && capacity > 0 && used*5 < capacity*4 && after*5 >= capacity*4 { + ix.sayFill(ctx, meshcatalog.EventFilling, fmt.Sprintf("%d", rev.Revision), meshcatalog.Fill{Bytes: after, Cap: capacity}) + } + ix.logf("catalog graph: revision %d of source revision %d kept, %d entries, %d changed", rev.Revision, src, len(rev.Entries), len(changed)) + if ix.SayCatalogRevision != nil { + if err := ix.SayCatalogRevision(ctx, meshcatalog.CatalogRevisionEvent{V: CatalogEntryLayout, Revision: rev.Revision, Kind: rev.Kind, + Source: src, At: &at, Changed: changed, Entries: int64(len(rev.Entries))}); err != nil { + ix.logf("catalog graph: revision %d is not said: %v", rev.Revision, err) + } + } + return nil +} + +// HeldObjectsFull is the reason a pending merge is held while the catalog graph revisions are full (Pending.Held). +const HeldObjectsFull = "objects full" + +// ErrObjectsFull is a revision refused because the catalog graph revisions are at their cap. +var ErrObjectsFull = errors.New("the catalog graph revisions are full") + +// newBytes is the size of what keeping rev adds: its list and each entry the bucket does not hold yet. +func newBytes(ctx context.Context, o Objects, rev meshcatalog.CatalogRevision, entries map[string][]byte) int64 { + held, _ := o.Names(ctx) + have := make(map[string]bool, len(held)) + for _, n := range held { + have[n] = true + } + list, _ := json.Marshal(rev) + size := int64(len(list)) + for _, ref := range rev.Entries { + if !have[EntryObject(ref.Digest)] { + size += int64(len(entries[ref.Digest])) + } + } + return size +} + +// sayFill says the seat's event filling, full or room of the catalog graph revisions, under the duplicate id of the +// event and what it is about, so the same one said twice is heard once. +func (ix *Indexer) sayFill(ctx context.Context, event, about string, f meshcatalog.Fill) { + if ix.SayFill == nil { + return + } + at := ix.now().UTC() + f.At, f.Store = &at, CatalogRevisionsObjects + if err := ix.SayFill(ctx, event, event+"-"+about, f); err != nil { + ix.logf("catalog graph: %s is not said: %v", event, err) + } +} + +// RoomAgain says room and evaluates pending again when a merge held with the reason objects full finds the store +// below its filling mark (80 % of its cap): called after anything removes catalog graph revisions (no timer). It answers whether room was said. +func (ix *Indexer) RoomAgain(ctx context.Context) (bool, error) { + if ix.Revisions == nil || ix.Pending == nil { + return false, nil + } + entries, err := ReadPending(ctx, ix.Pending, ix.logf) + if err != nil || !slices.ContainsFunc(entries, func(e PendingEntry) bool { return e.Value.Held == HeldObjectsFull }) { + return false, err + } + used, capacity, err := ix.Revisions.Fill(ctx) + if err != nil || (capacity > 0 && used*5 >= capacity*4) { + return false, err // room is said below the filling mark, 80 %, so one refusal is not followed by another at once + } + ix.sayFill(ctx, meshcatalog.EventRoom, fmt.Sprintf("%d", used), meshcatalog.Fill{Bytes: used, Cap: capacity}) + return true, ix.PendingAgain(ctx) // room in catalog-revisions: pending evaluated again +} + +// TrunkView is the indexer's view of the trunk read from the newest trunk revision the bucket keeps (the lead's ruling +// on milestone 47/G finding 1; ADR 0656 §6, one form of the fact): a store in memory holding each module's value as +// ModuleValueOf reads it back from its entry, under its modules key, a store in memory of each module's node of the module +// graph (NodeOf), and the revision it was read at (0 when the bucket keeps none yet: both start empty and the first start's +// catch-up fills them). A lost entry is an error naming the module (ReadRevision), never a module left out of the view. +func TrunkView(ctx context.Context, o Objects) (modules, nodes Store, at int64, err error) { + view, held := newOverlay(nil), newOverlay(nil) + newest, any, err := newestTrunk(ctx, o) + if err != nil || !any { + return view, held, 0, err + } + _, entries, err := ReadRevision(ctx, o, RevisionObject(newest)) + if err != nil { + return nil, nil, 0, err + } + for _, e := range entries { + key, err := meshcatalogclient.ModulesKey(e.Module) + if err != nil { + return nil, nil, 0, fmt.Errorf("trunk.%d: the entry of %q: %w", newest.Revision, e.Module, err) + } + data, err := meshcatalogclient.EncodeModule(ModuleValueOf(e)) + if err != nil { + return nil, nil, 0, fmt.Errorf("trunk.%d: the entry of %s: %w", newest.Revision, e.Module, err) + } + if err := view.Create(ctx, key, data); err != nil { + return nil, nil, 0, fmt.Errorf("trunk.%d: the entry of %s: %w", newest.Revision, e.Module, err) + } + gk, err := graph.Key(e.Module) + if err != nil { + continue // a module the graph cannot name has no node, as writeGraph writes none + } + node, err := graph.Encode(NodeOf(e)) + if err != nil { + return nil, nil, 0, fmt.Errorf("trunk.%d: the node of %s: %w", newest.Revision, e.Module, err) + } + if err := held.Create(ctx, gk, node); err != nil { + return nil, nil, 0, fmt.Errorf("trunk.%d: the node of %s: %w", newest.Revision, e.Module, err) + } + } + return view, held, newest.Revision, nil +} + +// MemoryStore is a store held in memory alone: the module graph's nodes, which each revision's entries carry, so the +// indexer keeps no state of them on the bus (milestone 47/G). +func MemoryStore() Store { return newOverlay(nil) } + +// The view of the trunk is the newest catalog graph revision the indexer's objects catalog-revisions keep (milestone +// 47/G, ADR 0656 §6): the tools bundle answers the modules and the module graph it once read from the states modules and +// graph from that revision, at its number, and says it waits while the indexer keeps none yet (ADR 0615 §6), never an +// empty catalog. + +// ErrNoRevision is a view asked before the indexer kept its first trunk revision. +var ErrNoRevision = errors.New("view-waits: the indexer keeps no catalog graph revision yet (its first start takes the trunk in); ask again on catalog-revision") + +// TrunkStates reads the view of the trunk from the newest trunk revision; under answers every other state. +type TrunkStates struct { + Objects Objects + Under func(string) views.Source +} + +// read is the view's modules and nodes, and the revision they were read at; none kept yet is ErrNoRevision. +func (t TrunkStates) read(nodes bool) (Store, int64, error) { + mods, held, at, err := TrunkView(context.Background(), t.Objects) + if err == nil && at == 0 { + err = ErrNoRevision + } + if nodes { + return held, at, err + } + return mods, at, err +} + +// state is the source of a state by its name: modules and graph from the revision, their marks at its number, every +// other the indexer's own state. +func (t TrunkStates) State(name string) views.Source { + switch name { + case "modules", graph.State: + return revisionSource{t: t, nodes: name == graph.State} + case views.MarksState: + return markSource{t: t, under: t.Under(name)} + } + return t.Under(name) +} + +// revisionSource is the modules or the module graph of the newest trunk revision, each value at its number. +type revisionSource struct { + t TrunkStates + nodes bool +} + +func (r revisionSource) Keys() ([]string, error) { + s, _, err := r.t.read(r.nodes) + if err != nil { + return nil, err + } + return s.Keys(context.Background()) +} + +func (r revisionSource) Get(key string) (*views.Entry, bool, error) { + s, at, err := r.t.read(r.nodes) + if err != nil { + return nil, false, err + } + value, _, found, err := s.Get(context.Background(), key) + if err != nil || !found { + return nil, false, err + } + return &views.Entry{Value: value, Revision: uint64(at)}, true, nil +} + +// markSource answers the mark of modules and graph as the newest trunk revision's number, which catalog-revision says +// the changes after; every other mark is the views family's. +type markSource struct { + t TrunkStates + under views.Source +} + +func (m markSource) Keys() ([]string, error) { return m.under.Keys() } + +func (m markSource) Get(state string) (*views.Entry, bool, error) { + if state != "modules" && state != graph.State { + return m.under.Get(state) + } + _, at, err := m.t.read(false) + if err != nil { + return nil, false, err + } + data, err := json.Marshal(views.Mark{V: views.Version, State: state, Revision: uint64(at), At: time.Now().UTC()}) + if err != nil { + return nil, false, err + } + return &views.Entry{Value: data, Revision: uint64(at)}, true, nil +} diff --git a/modules/mesh-delivery-indexer/internal/index/graph.go b/modules/mesh-delivery-indexer/internal/index/graph.go index 199f9f7d..2d37097f 100644 --- a/modules/mesh-delivery-indexer/internal/index/graph.go +++ b/modules/mesh-delivery-indexer/internal/index/graph.go @@ -81,22 +81,7 @@ func (ix *Indexer) writeGraph(ctx context.Context, edges map[string]graphEdges) GraphMost, len(names), names[GraphMost], len(names)-GraphMost-1) names = names[:GraphMost] } - providers := func(provision string) []catalogue.Manifest { - var out []catalogue.Manifest - for _, name := range slices.Sorted(maps.Keys(edges)) { - if slices.Contains(edges[name].manifest.Offers(), provision) { - out = append(out, edges[name].manifest) - } - } - return out - } - // The verb edges on a cycle are left out over the whole graph (ADR 0614 §3), every module it knows, so a module past - // GraphMost still closes the cycle it lies on. - whole := graph.Graph{} - for name, e := range edges { - whole[name] = graphNode(e, providers) - } - whole.LeaveOut() + whole := wholeGraph(edges) for _, name := range names { key, err := graph.Key(name) if err != nil { @@ -141,3 +126,23 @@ func (ix *Indexer) writeGraph(ctx context.Context, edges map[string]graphEdges) } return nil } + +// wholeGraph is the module graph of every module of edges, the verb edges on a cycle left out over the whole graph +// (ADR 0614 §3), so a module past GraphMost still closes the cycle it lies on. +func wholeGraph(edges map[string]graphEdges) graph.Graph { + providers := func(provision string) []catalogue.Manifest { + var out []catalogue.Manifest + for _, name := range slices.Sorted(maps.Keys(edges)) { + if slices.Contains(edges[name].manifest.Offers(), provision) { + out = append(out, edges[name].manifest) + } + } + return out + } + whole := graph.Graph{} + for name, e := range edges { + whole[name] = graphNode(e, providers) + } + whole.LeaveOut() + return whole +} diff --git a/modules/mesh-delivery-indexer/internal/index/helpers_test.go b/modules/mesh-delivery-indexer/internal/index/helpers_test.go index 59cb9423..4c458ce4 100644 --- a/modules/mesh-delivery-indexer/internal/index/helpers_test.go +++ b/modules/mesh-delivery-indexer/internal/index/helpers_test.go @@ -1,6 +1,12 @@ package index -import "encoding/json" +import ( + "context" + "encoding/json" + "maps" + "slices" + "sync" +) // rawList is the list a kept predefined value holds, as it was written. func rawList(value []byte) json.RawMessage { @@ -10,3 +16,69 @@ func rawList(value []byte) json.RawMessage { _ = json.Unmarshal(value, &kept) return kept.List } + +// memObjects is an object bucket in memory with a cap: a put past the cap is refused as the bus's DiscardNew refuses it. +type memObjects struct { + mu sync.Mutex + objs map[string][]byte + puts int + capacity int64 +} + +func newMemObjects(capacity int64) *memObjects { + return &memObjects{objs: map[string][]byte{}, capacity: capacity} +} + +func (m *memObjects) used() int64 { + var n int64 + for _, d := range m.objs { + n += int64(len(d)) + } + return n +} + +func (m *memObjects) Put(_ context.Context, name string, data []byte) error { + m.mu.Lock() + defer m.mu.Unlock() + if m.capacity > 0 && m.used()-int64(len(m.objs[name]))+int64(len(data)) > m.capacity { + return errObjectsFullForTest + } + m.objs[name] = slices.Clone(data) + m.puts++ + return nil +} + +func (m *memObjects) Get(_ context.Context, name string) ([]byte, error) { + m.mu.Lock() + defer m.mu.Unlock() + d, ok := m.objs[name] + if !ok { + return nil, ErrNoObject + } + return slices.Clone(d), nil +} + +func (m *memObjects) Delete(_ context.Context, name string) error { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.objs, name) + return nil +} + +func (m *memObjects) Names(context.Context) ([]string, error) { + m.mu.Lock() + defer m.mu.Unlock() + return slices.Sorted(maps.Keys(m.objs)), nil +} + +func (m *memObjects) Fill(context.Context) (int64, int64, error) { + m.mu.Lock() + defer m.mu.Unlock() + return m.used(), m.capacity, nil +} + +var errObjectsFullForTest = &objectsFullError{} + +type objectsFullError struct{} + +func (*objectsFullError) Error() string { return "maximum bytes exceeded" } diff --git a/modules/mesh-delivery-indexer/internal/index/imports.go b/modules/mesh-delivery-indexer/internal/index/imports.go index 50095db6..acd6e152 100644 --- a/modules/mesh-delivery-indexer/internal/index/imports.go +++ b/modules/mesh-delivery-indexer/internal/index/imports.go @@ -27,6 +27,13 @@ import ( // Imports that cannot be resolved (a package the forge cannot list) are kept with no ImportForm: the next recompute that // resolves them writes the import form, one rebuild too many, never a stale module. func (ix *Indexer) ApplyBuildEnd(ctx context.Context, value []byte) error { + if err := ix.applyBuildEnd(ctx, value); err != nil { + return err + } + return ix.KeepView(ctx) // the imports kept with the view (KeepView), so a restart reads them back +} + +func (ix *Indexer) applyBuildEnd(ctx context.Context, value []byte) error { if !builds.IsBuildEntry(value) { return nil } diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index 219cbe85..ffe1ffce 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -292,6 +292,13 @@ func (ix *Indexer) dropRemoved(ctx context.Context, repository string, held map[ // message (applyMerged). A repository whose trunk cannot be read, or reads as holding no module at all, is left as it is // and said: a failure never empties the bucket. func (ix *Indexer) Reconcile(ctx context.Context) error { + if err := ix.reconcile(ctx); err != nil { + return err + } + return ix.KeepView(ctx) // the modules dropped leave the view's next revision too +} + +func (ix *Indexer) reconcile(ctx context.Context) error { if ix.Modules == nil || ix.Trunk == nil { return nil } diff --git a/modules/mesh-delivery-indexer/internal/index/names.go b/modules/mesh-delivery-indexer/internal/index/names.go index e3d860d9..7a53689a 100644 --- a/modules/mesh-delivery-indexer/internal/index/names.go +++ b/modules/mesh-delivery-indexer/internal/index/names.go @@ -1,7 +1,5 @@ package index -import "git.novox.be/novox/mesh-delivery/sdk/graph" - // The holders, states and buckets the indexer names, taken from its own manifest (module.json: its name, its `state`, its // `reads`) and from the manifests of the holders it reads; a unit row checks them against the manifest // (TestTheNamesAreTheManifests). No contract names a holder (novox/hq ADR 0625 §4, ADR 0634 §1). @@ -14,13 +12,9 @@ const ( CandidatesBucket = Module + "_" + CandidatesState CandidatesHistory = 10 ModulesState = "modules" - ModulesBucket = Module + "_" + ModulesState - ModulesHistory = 20 ManifestsState = "manifests" ManifestsBucket = Module + "_" + ManifestsState ManifestsHistory = 1 - // GraphBucket is the bucket of the state graph (sdk/graph names the state, not its holder). - GraphBucket = Module + "_" + graph.State // VerdictsState and VerdictsBucket are in verdicts.go. VerdictsHistory = 5 PredefinedState = "predefined" diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index 0f6515c6..17b32ef0 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -95,6 +95,9 @@ type Pending struct { // UnreadSince is when the message was first found unread: past meshconditionsclient.CallUnansweredAfter, with the forge // heard since, it is the finding call-unanswered (unreadcall.go, ADR 0668 §3). UnreadSince time.Time `json:"unread-since,omitzero"` + // Held is why a ready merge is not taken in although its revision names this build: objects full (HeldObjectsFull), + // until room is said (RoomAgain). + Held string `json:"held,omitempty"` } // PendingEntry is one key of the pending state as read: its key, revision, value and the merged message decoded. For a @@ -782,11 +785,32 @@ func (ix *Indexer) evaluate(ctx context.Context, own string, start bool) (Applie if errors.Is(err, errVerdictsNotYet) { return a, "the verdicts state is not on the bus yet: the revision waits for it, taken in when it binds", nil } + if errors.Is(err, ErrObjectsFull) { + ix.holdFull(ctx, ready) + return a, ix.waitingLine(w) + "; held: " + HeldObjectsFull, nil + } return a, ix.waitingLine(w), err } return Applied{}, ix.waitingLine(w), ix.holdFor(ctx, ready, names, own, dry) } +// holdFull marks the ready merges held with the reason objects full (Pending.Held), read again at their revisions now +// (the take-in's claim moved them), so indexer_pending shows why they wait. +func (ix *Indexer) holdFull(ctx context.Context, ready []PendingEntry) { + entries, err := ReadPending(ctx, ix.Pending, ix.logf) + if err != nil { + ix.logf("take-in: the merges held by a full %s are not marked: %v", CatalogRevisionsObjects, err) + return + } + var held []PendingEntry + for _, e := range entries { + if slices.ContainsFunc(ready, func(r PendingEntry) bool { return r.Key == e.Key }) && e.Value.Held != HeldObjectsFull { + held = append(held, e) + } + } + ix.markSaid(ctx, held, nil, func(p *Pending) { p.Held = HeldObjectsFull }) +} + // holdLost holds a revision whose newest merge of a repository cannot be read (the forge bridge's stream no longer holds // it, or the bus refuses the read): said once on take-in.pending with the lost keys (TakeIn.Lost), and on each lost entry // (Pending.Lost, the reason, which indexer_pending shows), until a newer merge of its repository covers it or @@ -884,7 +908,9 @@ func (ix *Indexer) dry() *dryRun { // requirements, findings, seat definitions, holds and their bounds are not overlaid, nothing is said, and no hold is // weighed (Holds and Counted nil hold nothing); ReadFile only reads. Requirements: nil, Findings: nil, SeatDefinitions: nil, ReadFile: ix.ReadFile, SaySeatDefinition: nil, - Holds: nil, Counted: nil, AskEngines: nil, HoldBounds: nil, OwnBuildState: nil, underSeats: true} + Holds: nil, Counted: nil, AskEngines: nil, HoldBounds: nil, OwnBuildState: nil, underSeats: true, + // A dry run keeps no catalog graph revision (catalog_revisions.go): its revision is the take-in's to keep. + Revisions: nil, SayCatalogRevision: nil, SayFill: nil} x.Logf = func(format string, args ...any) { ix.logf("take-in, computed and not written: "+format, args...) } x.Store, d.candidates = over(ix.Store) x.Modules, d.modules = over(ix.Modules) @@ -1187,6 +1213,11 @@ func (ix *Indexer) holdFor(ctx context.Context, ready []PendingEntry, names, own } } } + // The indexer's own entry the held revision names is kept as the trunk revision after the newest, every other entry as + // it was (milestone 47/G W4.7: a merged revision naming a newer indexer replaces only the indexer's entry). + if err := ix.KeepView(ctx); err != nil { + return fmt.Errorf("the revision with the indexer's own entry: %w", err) + } t := revisionWait(takein.TakeInPending, ready, names) t.Refused = refused var ended, unmarked []PendingEntry diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index 0d62d0c5..86cdaabf 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -18,6 +18,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" "git.novox.be/novox/mesh-delivery/sdk/trunk" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/busretry" "git.novox.be/novox/mesh-sdk/go/catalogue" @@ -744,6 +745,141 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) } } }) + t.Run("W4.6 a full store holds the merge with the reason objects full, says full once, and room takes it in", func(t *testing.T) { + b := newTakeInBed(t) + b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) + named := b.fp(t, Module) + b.ix.ownMu.Lock() + b.ix.ownFP = named + b.ix.ownMu.Unlock() + store := newMemObjects(100) // 90 bytes held by another revision: full for this one + _ = store.Put(ctx, "trunk.0", make([]byte, 90)) + b.ix.Revisions = store + var said []string + b.ix.SayFill = func(_ context.Context, event, id string, f meshcatalog.Fill) error { + said = append(said, event+" "+f.Refused) + return nil + } + if _, _, err := b.ix.TakeInAtStart(ctx); err != nil { + t.Fatal(err) + } + var p Pending + _ = json.Unmarshal(b.pending.vals[b.keys()[0]], &p) + if len(b.keys()) != 1 || p.Held != HeldObjectsFull || len(said) != 1 || !strings.HasPrefix(said[0], "full ") || !strings.Contains(said[0], HeldObjectsFull) { + t.Fatalf("pending %v held %q, said %q: want the merge held with %q and full said", b.keys(), p.Held, said, HeldObjectsFull) + } + if roomed, err := b.ix.RoomAgain(ctx); err != nil || roomed { + t.Fatalf("room while still full: %v %v", roomed, err) + } + _ = store.Delete(ctx, "trunk.0") // the other revision removed: room + store.capacity = 0 + if roomed, err := b.ix.RoomAgain(ctx); err != nil || !roomed || len(b.keys()) != 0 || said[len(said)-1] != "room " { + t.Errorf("room said %v (%v), said %q, pending %v: want room said and the merge taken in", roomed, err, said, b.keys()) + } + // The same take-in into a store it fills to 85 % of its cap says filling once, and is taken in. + kept := store.used() + b2 := newTakeInBed(t) + b2.arrive(t, 2, b2.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) + b2.ix.ownMu.Lock() + b2.ix.ownFP = b2.fp(t, Module) + b2.ix.ownMu.Unlock() + b2.ix.Revisions = newMemObjects(kept * 100 / 85) + var filling []meshcatalog.Fill + b2.ix.SayFill = func(_ context.Context, event, _ string, f meshcatalog.Fill) error { + if event == meshcatalog.EventFilling { + filling = append(filling, f) + } + return nil + } + if _, _, err := b2.ix.TakeInAtStart(ctx); err != nil || len(b2.keys()) != 0 || len(filling) != 1 || filling[0].Store != CatalogRevisionsObjects { + t.Errorf("a take-in to 85 %% said filling %+v (%v), pending %v: want it said once and the merge taken in", filling, err, b2.keys()) + } + }) + t.Run("W4.12 a candidate's half dropped by hand frees its own entries, says room and takes the held merge in; the read tools list it; a recompute that moves nothing is equal", func(t *testing.T) { + bed := func(store *memObjects) *takeInBed { + b := newTakeInBed(t) + b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) + b.ix.ownMu.Lock() + b.ix.ownFP = b.fp(t, Module) + b.ix.ownMu.Unlock() + b.ix.Revisions = store + return b + } + free := newMemObjects(0) + if _, _, err := bed(free).ix.TakeInAtStart(ctx); err != nil { + t.Fatal(err) + } + size := free.used() // what the merge's revision takes + store := newMemObjects(2 * size) + shared := []byte(`{"module":"b"}`) + if err := KeepRevision(ctx, store, meshcatalog.CatalogRevision{Kind: "candidate", Group: "g1", Heads: "h1", Revision: 1, + Entries: []meshcatalog.EntryRef{{Module: "a", Digest: "da"}, {Module: "b", Digest: "db"}}}, + map[string][]byte{"da": make([]byte, size+10), "db": shared}); err != nil { + t.Fatal(err) + } + if err := KeepRevision(ctx, store, meshcatalog.CatalogRevision{Kind: "candidate", Group: "g2", Heads: "h2", Revision: 1, + Entries: []meshcatalog.EntryRef{{Module: "b", Digest: "db"}}}, map[string][]byte{"db": shared}); err != nil { + t.Fatal(err) + } + b := bed(store) + var said []string + b.ix.SayFill = func(_ context.Context, event, _ string, _ meshcatalog.Fill) error { said = append(said, event); return nil } + if _, _, err := b.ix.TakeInAtStart(ctx); err != nil || len(b.keys()) != 1 { + t.Fatalf("pending %v (%v): want the merge held for objects full", b.keys(), err) + } + var recorded []string + asks := 0 + tools := map[string]stdio.Tool{} + for _, tool := range CatalogRevisionTools(store, func(_ string, value []byte) (TakeInAsk, error) { + asks++ + a, err := DecodeTakeIn(value) + if err != nil { + return a, err + } + var r TakenIn + if a.Recompute { + r = b.ix.RecomputeByHand(ctx, a.By, a.Why) + } else { + r = b.ix.DropCandidateGraph(ctx, a.CandidateGroup, a.CandidateHeads, a.By, a.Why) + } + a.Result = &r + return a, nil + }, time.Now, func(what, why string) error { + recorded = append(recorded, what) + if why == "unrecorded" { + return errors.New("mesh-hand-acts did not answer") + } + return nil + }) { + tools[tool.Name] = tool + } + if _, err := tools[DropCandidateGraphTool].Run(map[string]any{"group": "g1", "heads": "h1", "why": "unrecorded"}); err == nil || asks != 0 { + t.Fatalf("a drop whose hand act is not recorded: %v, %d asks: want refused before it is asked", err, asks) + } + got, err := tools[DropCandidateGraphTool].Run(map[string]any{"group": "g1", "heads": "h1", "why": "its gate event never came"}) + r, _ := got.(*TakenIn) + if err != nil || r == nil || !r.Stored || !slices.Equal(r.Dropped, []string{"entry.da"}) || len(recorded) != 2 { + t.Fatalf("drop answered %+v (%v), recorded %q: want candidate.g1.h1 dropped with entry.da alone, after its hand act", r, err, recorded) + } + if _, err := store.Get(ctx, "entry.db"); err != nil || len(b.keys()) != 0 || !slices.Contains(said, meshcatalog.EventRoom) { + t.Errorf("entry.db %v, pending %v, said %q: want the shared entry kept, room said and the merge taken in", err, b.keys(), said) + } + list, err := tools[CatalogRevisionsTool].Run(map[string]any{}) + l, _ := list.(RevisionsAnswer) + if err != nil || len(l.Revisions) != 2 || l.Revisions[0].Name != "candidate.g2.h2" || l.Revisions[1].Name != "trunk.1" || !l.Revisions[1].Current || + l.Fill.Cap != 2*size || l.Fill.Bytes != store.used() || len(l.Unlisted) != 0 { + t.Errorf("indexer_catalog_revisions = %+v (%v): want candidate.g2.h2 and the current trunk.1, the fill against its cap", l, err) + } + if c, err := tools[CandidateGraphsTool].Run(map[string]any{}); err != nil || len(c.(RevisionsAnswer).Revisions) != 1 { + t.Errorf("indexer_candidate_graphs = %+v (%v): want g2's alone", c, err) + } + if one, err := tools[CatalogRevisionTool].Run(map[string]any{}); err != nil || one.(map[string]any)["name"] != "trunk.1" { + t.Errorf("indexer_catalog_revision = %v (%v): want the newest trunk revision whole", one, err) + } + if got, err := tools[RecomputeTool].Run(map[string]any{"why": "a check"}); err != nil || !strings.Contains(got.(*TakenIn).Answer, "equal to the kept revision 1") { + t.Errorf("indexer_recompute = %+v (%v): want equal to the kept revision 1", got, err) + } + }) t.Run("T7 a toolchain report evaluates pending again: the revision then names the running build and is taken in", func(t *testing.T) { b := newTakeInBed(t) b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) @@ -970,7 +1106,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) for _, f := range []string{"Store", "Modules", "Manifests", "Predefined", "ConfigCandidates", "Taken", "Verdicts", "OldSections", "Unjudged", "JudgeID", "SayRefused", "SayAccepted", "Packages", "Graph", "Files", "GoSum", "GoMod", "Now", "Builds", "Named", "Trunk", "Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn", - "Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "Holds", "Counted", "AskEngines", "HoldBounds", "OpenTries", "TryRecovery", "OwnBuildState"} { + "Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "Holds", "Counted", "AskEngines", "HoldBounds", "OpenTries", "TryRecovery", "OwnBuildState", "Revisions", "SayCatalogRevision", "SayFill"} { named[f] = true } ty := reflect.TypeOf(Indexer{}) diff --git a/modules/mesh-delivery-indexer/internal/index/registered.go b/modules/mesh-delivery-indexer/internal/index/registered.go index 2f30556c..5bf426d7 100644 --- a/modules/mesh-delivery-indexer/internal/index/registered.go +++ b/modules/mesh-delivery-indexer/internal/index/registered.go @@ -220,6 +220,8 @@ type TakenIn struct { // and deleted. Handed []string `json:"handed,omitempty"` Dropped []string `json:"dropped,omitempty"` + // Unread are the repositories a catch-up (CatchUp) could not take in: their merged message was not read. + Unread []string `json:"unread,omitempty"` // Fingerprints are the modules whose fingerprint a recompute by hand moved (FingerprintsAgain), each old and new. Fingerprints []FingerprintMove `json:"fingerprints,omitempty"` } diff --git a/modules/mesh-delivery-indexer/internal/index/revision_tools.go b/modules/mesh-delivery-indexer/internal/index/revision_tools.go new file mode 100644 index 00000000..afe1860f --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/revision_tools.go @@ -0,0 +1,326 @@ +package index + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "regexp" + "slices" + "strconv" + "strings" + "time" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// The indexer's tools of its catalog graph revisions (design 47, the indexer's tools; the instruction "tools are how a +// module is spoken to"): indexer_catalog_revision and indexer_catalog_revisions read the objects catalog-revisions, +// indexer_candidate_graphs every candidate's catalog half; indexer_recompute and indexer_drop_candidate_graph steer the +// indexer by an ask in takeins, as indexer_take_in does, since the indexer's process serves no request subject. +const ( + CatalogRevisionTool = "indexer_catalog_revision" + CatalogRevisionsTool = "indexer_catalog_revisions" + CandidateGraphsTool = "indexer_candidate_graphs" + RecomputeTool = "indexer_recompute" + DropCandidateGraphTool = "indexer_drop_candidate_graph" +) + +// RecomputeKey is the takeins key of a recompute by hand: one at a time. +const RecomputeKey = "recompute" + +var candidateToken = regexp.MustCompile(`^[A-Za-z0-9_=-]+$`) + +// NewRecomputeAsk is the value indexer_recompute writes: its key in takeins and the value. +func NewRecomputeAsk(why, by string, at time.Time) (string, []byte, error) { + data, err := json.Marshal(TakeInAsk{V: 1, Recompute: true, Why: strings.TrimSpace(why), By: by, At: at.UTC()}) + return RecomputeKey, data, err +} + +// NewDropCandidateGraphAsk is the value indexer_drop_candidate_graph writes: its key in takeins, +// candidate-graph.., and the value. +func NewDropCandidateGraphAsk(group, heads, why, by string, at time.Time) (string, []byte, error) { + group, heads = strings.TrimSpace(group), strings.TrimSpace(heads) + switch { + case !candidateToken.MatchString(group) || !candidateToken.MatchString(heads): + return "", nil, errors.New("name the candidate by its group and its heads, as indexer_candidate_graphs names them") + case strings.TrimSpace(why) == "": + return "", nil, errors.New("say why (why): a drop is a hand act, recorded through mesh-hand-acts") + } + data, err := json.Marshal(TakeInAsk{V: 1, CandidateGroup: group, CandidateHeads: heads, Drop: true, Why: strings.TrimSpace(why), By: by, At: at.UTC()}) + return "candidate-graph." + group + "." + heads, data, err +} + +// RevisionSummary is one kept catalog graph revision as indexer_catalog_revisions and indexer_candidate_graphs list it. +type RevisionSummary struct { + Name string `json:"name"` + Kind string `json:"kind"` + Revision int64 `json:"revision"` + Source int64 `json:"source"` + Group string `json:"group,omitempty"` + Heads string `json:"heads,omitempty"` + At *time.Time `json:"at,omitempty"` + Entries int `json:"entries"` + // Current says it is the trunk's newest revision, which is kept whatever pins it. + Current bool `json:"current,omitempty"` +} + +// RevisionsAnswer is what indexer_catalog_revisions answers: every kept revision, the entries stored once by digest, the +// store's fill against its cap, and each object it could not read. +type RevisionsAnswer struct { + Revisions []RevisionSummary `json:"revisions"` + Entries int `json:"entries"` + Unlisted []string `json:"unlisted,omitempty"` + Fill meshcatalog.Fill `json:"fill"` + FillUnread string `json:"fill-unread,omitempty"` + Unread []string `json:"unread,omitempty"` +} + +// ListRevisions reads every revision the objects keep, candidates included (kind "candidate" alone when candidates). +func ListRevisions(ctx context.Context, o Objects, candidates bool) (RevisionsAnswer, error) { + out := RevisionsAnswer{Revisions: []RevisionSummary{}} + names, err := o.Names(ctx) + if err != nil { + return out, fmt.Errorf("the objects %s cannot be listed: %w", CatalogRevisionsObjects, err) + } + listed := map[string]bool{} + var newest int64 + entries := []string{} + for _, n := range names { + if strings.HasPrefix(n, "entry.") { + entries = append(entries, n) + continue + } + data, err := o.Get(ctx, n) + var rev meshcatalog.CatalogRevision + if err == nil { + err = json.Unmarshal(data, &rev) + } + if err != nil { + out.Unread = append(out.Unread, n+": "+err.Error()) + continue + } + for _, ref := range rev.Entries { + listed[EntryObject(ref.Digest)] = true + } + if rev.Kind == "trunk" && rev.Revision > newest { + newest = rev.Revision + } + if candidates && rev.Kind != "candidate" { + continue + } + out.Revisions = append(out.Revisions, RevisionSummary{Name: n, Kind: rev.Kind, Revision: rev.Revision, Source: rev.Source, + Group: rev.Group, Heads: rev.Heads, At: rev.At, Entries: len(rev.Entries)}) + } + for i := range out.Revisions { + out.Revisions[i].Current = out.Revisions[i].Kind == "trunk" && out.Revisions[i].Revision == newest + } + slices.SortFunc(out.Revisions, func(a, b RevisionSummary) int { return strings.Compare(a.Name, b.Name) }) + out.Entries = len(entries) + for _, e := range entries { + if !listed[e] { + out.Unlisted = append(out.Unlisted, e) + } + } + slices.Sort(out.Unlisted) + used, capacity, err := o.Fill(ctx) + out.Fill = meshcatalog.Fill{Bytes: used, Cap: capacity, Store: CatalogRevisionsObjects} + if err != nil { + out.FillUnread = err.Error() + } + return out, nil +} + +// removeUnlisted deletes every entry no kept revision lists any more: an entry is stored once and shared, so it goes +// only with the last revision that lists it. It answers the entries removed. +func removeUnlisted(ctx context.Context, o Objects) ([]string, error) { + list, err := ListRevisions(ctx, o, false) + if err != nil { + return nil, err + } + if len(list.Unread) > 0 { + // A revision that cannot be read may list any entry: none is removed rather than one it still needs. + return nil, fmt.Errorf("no entry is removed while a revision cannot be read: %s", strings.Join(list.Unread, "; ")) + } + var removed []string + for _, e := range list.Unlisted { + if err := o.Delete(ctx, e); err != nil { + return removed, fmt.Errorf("%s: %w", e, err) + } + removed = append(removed, e) + } + return removed, nil +} + +// DropCandidateGraph drops a candidate's catalog half by hand (indexer_drop_candidate_graph, design 47: for a candidate +// whose gate event never came), the entries no other revision lists with it, and says room when that takes the store +// below its filling mark, which takes in a merge held for objects full (RoomAgain). +func (ix *Indexer) DropCandidateGraph(ctx context.Context, group, heads, by, why string) TakenIn { + name := RevisionObject(meshcatalog.CatalogRevision{Kind: "candidate", Group: group, Heads: heads}) + r := TakenIn{Module: name} + if ix.Revisions == nil { + r.Answer = "the indexer keeps no catalog graph revisions: nothing dropped" + return r + } + if _, err := ix.Revisions.Get(ctx, name); err != nil { + r.Answer = name + " is not kept (" + err.Error() + "): nothing dropped" + return r + } + ix.keepMu.Lock() + err := ix.Revisions.Delete(ctx, name) + var removed []string + if err == nil { + removed, err = removeUnlisted(ctx, ix.Revisions) + } + ix.keepMu.Unlock() + if err != nil { + r.Answer = name + " could not be dropped whole: " + err.Error() + ix.logf("%s", r.Answer) + return r + } + r.Stored, r.Dropped = true, removed + r.Answer = fmt.Sprintf("%s is dropped by %s (%s), with %d entries no other revision lists", name, by, why, len(removed)) + if roomed, err := ix.RoomAgain(ctx); err != nil { + r.Answer += "; room is not evaluated: " + err.Error() + } else if roomed { + r.Answer += "; room said, what waited for it evaluated again" + } + ix.logf("%s", r.Answer) + return r +} + +// RecomputeByHand computes the catalog graph again now from the view of the trunk, as a toolchain report or a merge +// does, and compares it with the newest revision kept: the modules whose entry moved, and the revision kept for them +// (none when the recompute equals the one kept). +func (ix *Indexer) RecomputeByHand(ctx context.Context, by, why string) TakenIn { + r := TakenIn{Module: "catalog graph"} + if ix.Revisions == nil { + r.Answer = "the indexer keeps no catalog graph revisions: nothing recomputed" + return r + } + before, _, err := newestTrunk(ctx, ix.Revisions) + if err != nil { + r.Answer = "the newest catalog graph revision cannot be read: " + err.Error() + return r + } + if err := ix.Recompute(ctx); err != nil { + r.Answer = "the recompute failed: " + err.Error() + return r + } + after, _, err := newestTrunk(ctx, ix.Revisions) + if err != nil { + r.Answer = "the newest catalog graph revision cannot be read after the recompute: " + err.Error() + return r + } + r.Stored = true + if after.Revision == before.Revision { + r.Answer = fmt.Sprintf("recomputed by %s (%s): equal to the kept revision %d", by, why, before.Revision) + } else { + r.Handed = changedEntries(before.Entries, after.Entries) + r.Answer = fmt.Sprintf("recomputed by %s (%s): revision %d differs from the kept %d in %s; revision %d is kept", by, why, + after.Revision, before.Revision, strings.Join(r.Handed, ", "), after.Revision) + } + ix.logf("%s", r.Answer) + return r +} + +// CatalogRevisionTools are the tools of the catalog graph revisions; ask writes an ask in takeins and waits for the +// indexer's answer, record records a hand act through mesh-hand-acts before a drop is asked. +func CatalogRevisionTools(o Objects, ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time, record func(what, why string) error) []stdio.Tool { + str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } + asked := func(key string, value []byte, err error) (any, error) { + if err != nil { + return nil, err + } + a, err := ask(key, value) + if err != nil { + return nil, err + } + return a.Result, nil + } + return []stdio.Tool{ + {Name: CatalogRevisionTool, + Description: "One catalog graph revision the indexer keeps in its objects catalog-revisions, whole: its list (kind, number, " + + "source revision, trunk heads, a candidate's group and heads) and every entry it lists, each module as its manifest " + + "declares it at that source revision (bases, packages, toolchains, provides, requires, invokes, serves, fingerprint, " + + "acting rules, inputs). The trunk's newest with no input; revision a kept trunk revision; group and heads a candidate's. " + + "An entry the store lost is an error naming it. indexer_catalog_revisions lists what is kept.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "revision": map[string]any{"type": "integer", "description": "a kept trunk revision's number"}, + "group": map[string]any{"type": "string", "description": "a candidate's group, with heads"}, + "heads": map[string]any{"type": "string", "description": "a candidate's heads, with group"}}}, + Run: func(args map[string]any) (any, error) { + ctx := context.Background() + n, _ := args["revision"].(float64) + name := "" + switch { + case str(args, "group") != "" || str(args, "heads") != "": + name = RevisionObject(meshcatalog.CatalogRevision{Kind: "candidate", Group: str(args, "group"), Heads: str(args, "heads")}) + case n > 0: + name = "trunk." + strconv.FormatInt(int64(n), 10) + default: + newest, ok, err := newestTrunk(ctx, o) + if err != nil { + return nil, fmt.Errorf("the catalog graph revisions cannot be read: %w", err) + } + if !ok { + return nil, ErrNoRevision + } + name = RevisionObject(newest) + } + rev, entries, err := ReadRevision(ctx, o, name) + if err != nil { + return nil, err + } + return map[string]any{"name": name, "revision": rev, "entries": entries}, nil + }}, + {Name: CatalogRevisionsTool, + Description: "Every catalog graph revision the indexer keeps in its objects catalog-revisions, trunk and candidate: each " + + "one's name, kind, number, source revision, when, how many entries it lists and whether it is the trunk's newest " + + "(current); how many entries are stored once by digest, those no revision lists (unlisted), and the store's fill " + + "in bytes against its cap (filling is said at 80 %, full refuses the next revision). An object it cannot read is " + + "named in unread. indexer_catalog_revision reads one whole.", + Input: map[string]any{"type": "object", "properties": map[string]any{}}, + Run: func(map[string]any) (any, error) { + return ListRevisions(context.Background(), o, false) + }}, + {Name: CandidateGraphsTool, + Description: "Every candidate's catalog half the indexer keeps (a delivery group's or a pull request's heads merged onto " + + "a trunk source revision): its group, its heads, its revision, the source revision it is merged onto, since when, " + + "and how many entries it lists; and the store's fill. indexer_drop_candidate_graph drops one whose gate event never came.", + Input: map[string]any{"type": "object", "properties": map[string]any{}}, + Run: func(map[string]any) (any, error) { + return ListRevisions(context.Background(), o, true) + }}, + {Name: RecomputeTool, + Description: "Compute the catalog graph again now, as the indexer does on a toolchain report or a merge, from the view of " + + "the trunk (each module's manifest, its bases, the agreed toolchains and the published packages), and compare it with " + + "the newest revision kept: answers equal, or the modules whose entry differs and the revision kept for them.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "why": map[string]any{"type": "string", "description": "why it is recomputed now, in plain words"}}}, + Run: func(args map[string]any) (any, error) { + key, value, err := NewRecomputeAsk(str(args, "why"), stdio.CallerOf(args), now()) + return asked(key, value, err) + }}, + {Name: DropCandidateGraphTool, + Description: "Drop one candidate's catalog half by hand, for a candidate whose gate event never came: its revision and the " + + "entries no other revision lists. A hand act, recorded through mesh-hand-acts before it is asked. When the drop takes " + + "the store below its filling mark, room is said and a merge held for objects full is taken in. indexer_candidate_graphs " + + "names the candidates.", + Input: map[string]any{"type": "object", "required": []string{"group", "heads", "why"}, "properties": map[string]any{ + "group": map[string]any{"type": "string", "description": "the candidate's group"}, + "heads": map[string]any{"type": "string", "description": "the candidate's heads"}, + "why": map[string]any{"type": "string", "description": "why it is dropped, in plain words"}}}, + Run: func(args map[string]any) (any, error) { + key, value, err := NewDropCandidateGraphAsk(str(args, "group"), str(args, "heads"), str(args, "why"), stdio.CallerOf(args), now()) + if err != nil { + return nil, err + } + if err := record("dropped the catalog half of the candidate "+str(args, "group")+" at "+str(args, "heads")+" by hand", str(args, "why")); err != nil { + return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was dropped: %w", err) + } + return asked(key, value, nil) + }}, + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/revision_verb.go b/modules/mesh-delivery-indexer/internal/index/revision_verb.go new file mode 100644 index 00000000..82ffac84 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/revision_verb.go @@ -0,0 +1,102 @@ +package index + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "strconv" + "strings" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/views" +) + +// RevisionInvoke is how a part declares the seat mesh-catalog's verb revision in its manifest's invokes, the one way a +// part reads the catalog half (novox/hq ADR 0663 §2). +const RevisionInvoke = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbRevision + +// RevisionCallers are the parts the newest trunk revision's entries say declare the verb revision in their invokes, +// sorted, the holder itself never among them (ADR 0663 §2: granted to exactly the parts that declare it). +func RevisionCallers(entries []meshcatalog.CatalogEntry, holder string) stdio.Callers { + var out stdio.Callers + for _, e := range entries { + if e.Module != holder && slices.Contains(e.Invokes, RevisionInvoke) { + out = append(out, e.Module) + } + } + slices.Sort(out) + return out +} + +// Revision answers the seat mesh-catalog's verb revision from the indexer's own objects (ADR 0663 §1, §3): one catalog +// graph revision, the trunk's newest when the input names none, a kept trunk revision by its number, or a candidate's by +// its group and heads, with one page of its entries whole, by module name. The caller is the runtime's proof: it is +// answered only when the newest trunk revision's entry of its module declares the verb in invokes, and the holder is +// refused too. With no trunk revision kept yet nobody is answered and the reader waits (ErrNoRevision), never an empty +// catalog. +func Revision(ctx context.Context, o Objects, holder, caller string, in meshcatalog.RevisionInput) (meshcatalog.RevisionAnswer, error) { + var out meshcatalog.RevisionAnswer + newest, ok, err := newestTrunk(ctx, o) + if err != nil { + return out, fmt.Errorf("the catalog graph revisions cannot be read: %w", err) + } + if !ok { + return out, ErrNoRevision + } + _, trunkEntries, err := ReadRevision(ctx, o, RevisionObject(newest)) + if err != nil { + return out, err + } + if err := views.Refusal(meshcatalog.Seat+"."+meshcatalog.VerbRevision, RevisionCallers(trunkEntries, holder), caller); err != nil { + return out, fmt.Errorf("%w (novox/hq ADR 0663 §2: a part reads it by declaring %s in invokes)", err, RevisionInvoke) + } + name := RevisionObject(newest) + switch { + case in.Group != "" || in.Heads != "": + if in.Group == "" || in.Heads == "" { + return out, fmt.Errorf("a candidate's revision is named by its group and its heads, both") + } + name = RevisionObject(meshcatalog.CatalogRevision{Kind: "candidate", Group: in.Group, Heads: in.Heads}) + case in.Revision != 0 && in.Revision != newest.Revision: + name = "trunk." + strconv.FormatInt(in.Revision, 10) + } + rev, entries := newest, trunkEntries + if name != RevisionObject(newest) { + if rev, entries, err = ReadRevision(ctx, o, name); err != nil { + return out, err + } + if in.Revision != 0 && rev.Revision != in.Revision { + return out, fmt.Errorf("the candidate %s.%s keeps its newest revision only, %d, not %d", in.Group, in.Heads, rev.Revision, in.Revision) + } + } + slices.SortFunc(entries, func(a, b meshcatalog.CatalogEntry) int { return strings.Compare(a.Module, b.Module) }) + out.Revision, out.Entries = rev, []meshcatalog.CatalogEntry{} + size := 0 + for _, e := range entries { + if e.Module <= in.After { + continue + } + data, _ := json.Marshal(e) + if len(out.Entries) > 0 && size+len(data) > views.PageBytes { + out.Next = out.Entries[len(out.Entries)-1].Module + break + } + size += len(data) + out.Entries = append(out.Entries, e) + } + return out, nil +} + +// RevisionTool is the verb revision as the tool mesh-catalog.revision of the indexer's own runtime (ADR 0663 §1): the +// tool runner describes it with the seat's definition, so it carries no description or input of its own. +func RevisionTool(o Objects, holder string) stdio.Tool { + return stdio.Tool{Name: meshcatalog.Seat + "." + meshcatalog.VerbRevision, Run: func(args map[string]any) (any, error) { + var in meshcatalog.RevisionInput + if raw, err := json.Marshal(args); err == nil { + _ = json.Unmarshal(raw, &in) + } + return Revision(context.Background(), o, holder, stdio.CallerOf(args), in) + }} +} diff --git a/modules/mesh-delivery-indexer/internal/index/source.go b/modules/mesh-delivery-indexer/internal/index/source.go index 62123afe..702990d1 100644 --- a/modules/mesh-delivery-indexer/internal/index/source.go +++ b/modules/mesh-delivery-indexer/internal/index/source.go @@ -2,9 +2,14 @@ package index import ( "context" + "errors" "fmt" "maps" "slices" + "strings" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" + "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" @@ -56,6 +61,9 @@ func (ix *Indexer) ApplySource(ctx context.Context, a meshdelivery.SourceAnswer) if ix.Modules == nil { return ix.RecomputeCandidates(ctx) } + if err := ix.kickCatchUp(ctx, s.Heads); err != nil { + return err + } // A registered module the forge could not read is read again at each new source revision (issue 1945), in the // background: a forge that does not answer must not hold this revision. One taken in later is stamped when it is written // (applyManifests reads srcOf). @@ -100,6 +108,124 @@ func (ix *Indexer) ApplySource(ctx context.Context, a meshdelivery.SourceAnswer) return fmt.Errorf("modules.%s: %w", key, err) } } + // The stamps are kept with the view (KeepView), so a restart reads them back. + if err := ix.KeepView(ctx); err != nil { + return err + } // A trunk that moved makes the previews made onto the old one out of date. return ix.RecomputeCandidates(ctx) } + +// kickCatchUp takes the trunk in at a first start, when the objects keep no trunk revision yet (the lead's ruling A on +// milestone 47/G: the catch-up path): the head the source names for every repository, by the merged message the forge +// bridge gives for it, each applied as its merge, and revision 1 kept from them all at its end. Until then the revision +// the tools answer waits, never an empty catalog. The catch-up runs off the source handler, in its own goroutine, said +// when it ends (lead-47's ruling R1 on the rebase): a bridge that does not answer holds the kept call (ADR 0668) to its +// bound, and must not hold this source revision's stamps and the pulls' candidates with it. A repository whose message is +// not read keeps no revision, said: the next source revision runs the catch-up again (no timer). Objects not on the bus +// yet run none, said, until then. +func (ix *Indexer) kickCatchUp(ctx context.Context, heads map[string]string) error { + if ix.Revisions == nil || ix.Message == nil || len(heads) == 0 { + return nil + } + if _, any, err := newestTrunk(ctx, ix.Revisions); errors.Is(err, process.ErrNotYet) { + ix.logf("catalog graph: the trunk is not taken in yet: its objects %s are not on the bus", CatalogRevisionsObjects) + return nil + } else if err != nil || any { + return err + } + if !ix.catchingUp.CompareAndSwap(false, true) { + return nil + } + heads = maps.Clone(heads) + ix.catchUpWG.Add(1) + go func() { + defer ix.catchUpWG.Done() + unread := ix.takeInHeads(ctx, heads) + ix.catchingUp.Store(false) + if len(unread) > 0 { + ix.logf("catalog graph: the first start's catch-up ended and no first revision is kept: %d repositories were not taken in (%s); taken in again at the next source revision", + len(unread), strings.Join(unread, ", ")) + return + } + if err := ix.KeepView(ctx); err != nil { + ix.logf("catalog graph: the first start's catch-up took in the trunk of %d repositories, but its revision could not be kept: %v; taken in again at the next source revision", len(heads), err) + return + } + ix.logf("catalog graph: the first start's catch-up ended: the trunk of %d repositories taken in", len(heads)) + }() + return nil +} + +// takeInHeads applies the forge bridge's merged message of each repository's head, as its merge, and answers the +// repositories whose message was not read or not applied. +func (ix *Indexer) takeInHeads(ctx context.Context, heads map[string]string) []string { + var unread []string + for _, repository := range slices.Sorted(maps.Keys(heads)) { + head := heads[repository] + if head == "" { + continue + } + f, err := ix.mergedAt(ctx, repository, head) + if err == nil { + _, err = ix.ApplyMessage(ctx, f) + } + if err != nil { + unread = append(unread, repository) + ix.logf("catalog graph: the trunk of %s at %.12s is not taken in: %v", repository, head, err) + } + } + return unread +} + +// CatchUp is the indexer's catch-up tool's work (indexer_take_in with no module and no repository, the catch-up its +// manifest names beside the catalog graph revisions): after a restore point of them is restored, the node agent calls it +// with no arguments, and the indexer takes in the head the gate's source names for every repository, as the first +// start's catch-up does, whatever revision the objects keep, then keeps the view as the next revision when an entry +// moved. Its answer names the repositories taken in and those whose merged message could not be read. +func (ix *Indexer) CatchUp(ctx context.Context) TakenIn { + out := TakenIn{Module: "catch-up"} + ix.mu.Lock() + heads := maps.Clone(ix.source.heads) + ix.mu.Unlock() + switch { + case len(heads) == 0: + out.Answer = "the gate's source names no head yet, so nothing is taken in; the next source revision is taken in as it comes" + return out + case ix.Modules == nil || ix.Message == nil: + out.Answer = "the indexer holds no view of the trunk or reads no merged message, so nothing is taken in" + return out + case !ix.catchingUp.CompareAndSwap(false, true): + out.Answer = "a catch-up runs already; its end keeps the revision" + return out + } + unread := ix.takeInHeads(ctx, heads) + ix.catchingUp.Store(false) + if err := ix.KeepView(ctx); err != nil { + out.Answer = "the repositories were taken in but the revision could not be kept: " + err.Error() + return out + } + out.Stored = len(unread) == 0 + out.Unread = unread + out.Answer = fmt.Sprintf("the heads the gate's source names for %d repositories taken in", len(heads)-len(unread)) + if len(unread) > 0 { + out.Answer += fmt.Sprintf("; %d not (%s): their merged message was not read, and the call may be made again", len(unread), strings.Join(unread, ", ")) + } + return out +} + +// mergedAt is the forge bridge's merged message of a repository's head. +func (ix *Indexer) mergedAt(ctx context.Context, repository, head string) (forgemsg.Forge, error) { + data, err := ix.Message(ctx, repository, head, 0) + if err != nil { + return forgemsg.Forge{}, fmt.Errorf("the forge bridge could not answer: %w", err) + } + f, err := forgemsg.DecodeForge(data) + if err != nil { + return f, fmt.Errorf("the forge bridge's answer is not a forge message: %w", err) + } + if f.Kind != forgemsg.ForgeMerged || f.Merge != head || !trunk.SameRepository(f.Repository, repository) { + return f, fmt.Errorf("the forge bridge's answer is not the merge of %s at %s", repository, head) + } + return f, nil +} diff --git a/modules/mesh-delivery-indexer/internal/index/takein.go b/modules/mesh-delivery-indexer/internal/index/takein.go index 89505157..31e4ef4e 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein.go +++ b/modules/mesh-delivery-indexer/internal/index/takein.go @@ -32,6 +32,9 @@ type TakeInAsk struct { // Repository asks the head the gate's source names for a repository to be taken in (TakeInRepository) instead of a module. Repository string `json:"repository,omitempty"` Why string `json:"why,omitempty"` + // CatchUp asks the head the gate's source names for every repository to be taken in (CatchUp): the catch-up tool, + // indexer_take_in called with no module and no repository, as the node agent calls it after a restore. + CatchUp bool `json:"catch-up,omitempty"` // Judge asks the manifests to be judged again now (indexer_judge_again) instead of the module to be taken in. Judge bool `json:"judge,omitempty"` // Digest names, with Judge, the one manifest digest of the module to judge (indexer_judge with a digest). @@ -63,12 +66,17 @@ type TakeInAsk struct { // packages as they are read (indexer_fingerprints_again, fingerprints.go). Fingerprints bool `json:"fingerprints,omitempty"` // RecoverTries asks the open tries' candidates to be recovered now (indexer_recover_tries). - RecoverTries bool `json:"recover-tries,omitempty"` - Drop bool `json:"drop,omitempty"` - By string `json:"by,omitempty"` - At time.Time `json:"at"` - Result *TakenIn `json:"result,omitempty"` - Answered time.Time `json:"answered,omitzero"` + RecoverTries bool `json:"recover-tries,omitempty"` + // Recompute asks the catalog graph to be computed again now and compared with the newest kept revision (indexer_recompute). + Recompute bool `json:"recompute,omitempty"` + // CandidateGroup and CandidateHeads, with Drop, ask that candidate's catalog half to be dropped (indexer_drop_candidate_graph). + CandidateGroup string `json:"candidate-group,omitempty"` + CandidateHeads string `json:"candidate-heads,omitempty"` + Drop bool `json:"drop,omitempty"` + By string `json:"by,omitempty"` + At time.Time `json:"at"` + Result *TakenIn `json:"result,omitempty"` + Answered time.Time `json:"answered,omitzero"` } // Done says the ask is answered. @@ -83,7 +91,7 @@ func DecodeTakeIn(data []byte) (TakeInAsk, error) { if a.V != 1 { return a, fmt.Errorf("a takeins value of version %d, not 1", a.V) } - if a.Module == "" && a.Repository == "" && a.Pending == "" && a.Merge == "" && a.TryCandidate == "" && !a.RecoverTries && !a.Fingerprints { + if a.Module == "" && a.Repository == "" && a.Pending == "" && a.Merge == "" && a.TryCandidate == "" && !a.RecoverTries && !a.Fingerprints && !a.CatchUp && !a.Recompute && a.CandidateGroup == "" { return a, errors.New("a takeins value naming no module, repository, pending wait, merge or try") } return a, nil @@ -114,6 +122,20 @@ func NewTakeInRepository(repository, why, by string, at time.Time) (string, []by return "repository." + key, data, err } +// CatchUpKey is the takeins key of a catch-up: one at a time, a second written over an unanswered first. +const CatchUpKey = "catch-up" + +// NewCatchUp is the value indexer_take_in writes when it names no module and no repository: the catch-up of every +// repository the gate's source names (CatchUp). +func NewCatchUp(why, by string, at time.Time) (string, []byte, error) { + why = strings.TrimSpace(why) + if why == "" { + why = "the catch-up after a restore" + } + data, err := json.Marshal(TakeInAsk{V: 1, CatchUp: true, Why: why, By: by, At: at.UTC()}) + return CatchUpKey, data, err +} + // TakeInMergeTool takes one merge in by hand, its merged message asked of the forge bridge (ADR 0633 §5, issue 1585). const TakeInMergeTool = "indexer_take_in_merge" diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index 2ffb0617..0f8b0f29 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -372,7 +372,10 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc if err := ix.writeOverlay(ctx, overlay, finals, w); err != nil { return err } - return ix.writeGraph(ctx, edges) + if err := ix.writeGraph(ctx, edges); err != nil { + return err + } + return ix.keepTrunkRevision(ctx, edges, wholeGraph(edges)) } // writeOverlay writes the records of a merge, each once and only if it differs from what the bucket holds; finals has diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index 17667eef..5de024cf 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -2,12 +2,15 @@ package index import ( "context" + "encoding/json" "errors" "fmt" "maps" + "os" "reflect" "slices" "strings" + "sync" "testing" "time" @@ -18,8 +21,10 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/keys" "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/views" "git.novox.be/novox/mesh-delivery/sdk/graph" @@ -285,16 +290,128 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // if g := graphOf(t, ix, "other"); g.FP != "" || !strings.Contains(g.None, "ts-base") || !reflect.DeepEqual(g.Bases, []string{"ts-base"}) { t.Errorf("graph.other = %+v, want its base and its reason", g) } + // Milestone 47/G W4.1, W4.2 (design 47, Two graphs): the catalog graph entry of a module is its graph node with where the + // trunk holds it; a revision lists one entry per module by digest, an unchanged entry has the same digest in the next + // revision, and a changed one only its own. + entries := map[string]meshcatalog.CatalogEntry{} + for name := range mods.vals { + m, _ := moduleOf(t, mods, name) + entries[name] = CatalogEntryOf(name, m, catalogue.Manifest{}, graphOf(t, ix, name)) + } + // W3.1's main: the Go main package the module's process runs, read from its manifest (here the indexer's own); a + // manifest that runs no process names none. + own, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + if m, err := catalogue.Decode(own); err != nil || MainOf(m) != "cmd/mesh-delivery-indexer" || MainOf(catalogue.Manifest{}) != "" { + t.Errorf("the indexer's main = %q (%v), want cmd/mesh-delivery-indexer", MainOf(m), err) + } + if e := entries["avahi"]; e.Fp != want["avahi"] || e.Commit == "" || e.Manifest == "" || !reflect.DeepEqual(e.Bases, []string{"mesh-tools"}) || + e.Toolchains["go"] != (meshcatalog.BuildEdge{Module: "mesh-build-go", Version: "go1.27.1", Recipe: "1"}) { + t.Errorf("the catalog entry of avahi = %+v", e) + } + // W3.1's inputs (the lead's ruling on finding 1): an entry carries what the next revision is computed from (the + // fingerprint's inputs, the toolchains used, the package versions), so the module's value read back from it is the + // value modules held, but for the old line's fields. + for name := range mods.vals { + m, _ := moduleOf(t, mods, name) + m.Fp, m.None, m.Needs = "", "", nil + if back := ModuleValueOf(entries[name]); !reflect.DeepEqual(back, m) { + t.Errorf("%s read back from its entry = %+v, want %+v", name, back, m) + } + } + if e := entries["avahi"]; e.Inputs == nil || e.Inputs.From == nil || e.Inputs.Toolchains["go"].Version != "go1.27.1" { + t.Errorf("the catalog entry of avahi carries inputs %+v, want what its fingerprint is computed from", e.Inputs) + } + rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk", Source: 7}, entries) + if err != nil || len(rev.Entries) != len(mods.vals) || len(objects) != len(mods.vals) || rev.V != CatalogEntryLayout { + t.Fatalf("revision %+v with %d objects (%v), want one entry per module of %d", rev, len(objects), err, len(mods.vals)) + } + moved := maps.Clone(entries) + tools := moved["mesh-tools"] + tools.Fp = "src2:" + strings.Repeat("b", 64) + moved["mesh-tools"] = tools + next, _, _ := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 2, Kind: "trunk", Source: 8}, moved) + for i, ref := range next.Entries { + if same := ref.Digest == rev.Entries[i].Digest; same == (ref.Module == "mesh-tools") { + t.Errorf("entry %s: digest %s against %s before; only mesh-tools' moves", ref.Module, ref.Digest, rev.Entries[i].Digest) + } + } + // W4.2's store: both revisions kept in catalog-revisions hold one object per entry once, the moved entry twice, and + // each list; a revision read back is its entries whole; an entry the bucket lost is an error, never left out. + kept := newMemObjects(0) + _, nextObjects, _ := CatalogRevisionOf(next, moved) + if err := KeepRevision(context.Background(), kept, rev, objects); err != nil { + t.Fatal(err) + } + if err := KeepRevision(context.Background(), kept, next, nextObjects); err != nil { + t.Fatal(err) + } + if names, _ := kept.Names(context.Background()); len(names) != len(mods.vals)+1+2 || kept.puts != len(names) { + t.Errorf("catalog-revisions holds %v after %d puts, want each entry once, the moved one twice, and two lists", names, kept.puts) + } + if back, got, err := ReadRevision(context.Background(), kept, "trunk.2"); err != nil || back.Source != 8 || len(got) != len(mods.vals) || + !slices.ContainsFunc(got, func(e meshcatalog.CatalogEntry) bool { return e.Module == "mesh-tools" && e.Fp == tools.Fp }) { + t.Errorf("trunk.2 read back = %+v, %d entries, %v", back, len(got), err) + } + // W4.3's second part: the view of the trunk read at start is the newest trunk revision, each module's value as its + // entry gives it back; a bucket that keeps none is an empty view at 0. + view, nodes, at, err := TrunkView(context.Background(), kept) + if err != nil || at != 2 { + t.Fatalf("the view is read at %d (%v), want 2", at, err) + } + for name, e := range moved { + key, _ := meshcatalogclient.ModulesKey(name) + raw, _, found, _ := view.Get(context.Background(), key) + got, err := meshcatalogclient.DecodeModule(raw) + if want := ModuleValueOf(e); !found || err != nil || !reflect.DeepEqual(got, want) { + t.Errorf("the view's %s = %+v (%v), want %+v", name, got, err, want) + } + // The node of the module graph the entry carries is read back with the view (NodeOf), so a revision kept before the + // first recompute after a start loses no edge. + gk, _ := graph.Key(name) + rawNode, _, found, _ := nodes.Get(context.Background(), gk) + if back, err := graph.Decode(rawNode); !found || err != nil || !reflect.DeepEqual(NodeOf(e), back) || back.FP != e.Fp || + !reflect.DeepEqual(back.Bases, e.Bases) || !reflect.DeepEqual(CatalogEntryOf(name, ModuleValueOf(e), catalogue.Manifest{}, back).Bases, e.Bases) { + t.Errorf("the view's node of %s = %+v (%v), want %+v", name, back, err, NodeOf(e)) + } + } + if empty, _, at, err := TrunkView(context.Background(), newMemObjects(0)); err != nil || at != 0 || func() bool { k, _ := empty.Keys(context.Background()); return len(k) != 0 }() { + t.Errorf("an empty bucket's view is read at %d (%v), want an empty view at 0", at, err) + } + _ = kept.Delete(context.Background(), EntryObject(next.Entries[0].Digest)) + if _, _, err := ReadRevision(context.Background(), kept, "trunk.2"); err == nil || !strings.Contains(err.Error(), next.Entries[0].Module) { + t.Errorf("trunk.2 with a lost entry read without error naming it: %v", err) + } + // W4.3: a recompute keeps a trunk revision of the module graph and says catalog-revision; one that moves no entry keeps + // none; a module leaving makes the next revision, naming it changed. + ix.Revisions = newMemObjects(0) + var revisions []meshcatalog.CatalogRevisionEvent + ix.SayCatalogRevision = func(_ context.Context, e meshcatalog.CatalogRevisionEvent) error { + revisions = append(revisions, e) + return nil + } + if err := ix.Recompute(context.Background()); err != nil || len(revisions) != 1 || revisions[0].Revision != 1 || revisions[0].Kind != "trunk" || + revisions[0].Entries != int64(len(mods.vals)) || len(revisions[0].Changed) != len(mods.vals) { + t.Fatalf("the first recompute said %+v (%v), want trunk revision 1 with every module", revisions, err) + } graph := ix.Graph.(*memStore) written := graph.writes - if err := ix.Recompute(context.Background()); err != nil || graph.writes != written { - t.Errorf("a recompute that moves nothing wrote the graph %d times (%v)", graph.writes-written, err) + if err := ix.Recompute(context.Background()); err != nil || graph.writes != written || len(revisions) != 1 { + t.Errorf("a recompute that moves nothing wrote the graph %d times and said %d revisions (%v)", graph.writes-written, len(revisions), err) } // A module the modules state no longer holds (a merge dropped it, issue 1900) leaves the graph at the next recompute. delete(mods.vals, "other") if err := ix.Recompute(context.Background()); err != nil { t.Fatal(err) } + if len(revisions) != 2 || revisions[1].Revision != 2 || !reflect.DeepEqual(revisions[1].Changed, []string{"other"}) { + t.Errorf("after other left, said %+v, want revision 2 naming other changed", revisions) + } + if _, kept, err := ReadRevision(context.Background(), ix.Revisions, "trunk.2"); err != nil || len(kept) != len(mods.vals) { + t.Errorf("trunk.2 holds %d entries (%v), want %d", len(kept), err, len(mods.vals)) + } if _, held := graph.vals["other"]; held { t.Error("graph.other is still held after its module went") } @@ -339,6 +456,137 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // t.Errorf("%s moved with a build of build-agent: fp2 = %q, want %s", name, m.Fp2, fp) } } + // The lead's ruling A on milestone 47/G: a first start whose objects keep no trunk revision takes in the head the source + // names for every repository, by the forge bridge's merged message, and keeps revision 1 with every module; a message + // the bridge does not give keeps none, and the next source revision takes the trunk in again. + first, _, merged := newFormIndexerUnmerged(t, builderModules(t)...) + first.Revisions = newMemObjects(0) + var firstSaid []meshcatalog.CatalogRevisionEvent + first.SayCatalogRevision = func(_ context.Context, e meshcatalog.CatalogRevisionEvent) error { + firstSaid = append(firstSaid, e) + return nil + } + // Lead-47's ruling R1: the catch-up runs off the source handler, so a bridge that has not answered yet holds no source + // revision (the first message is answered only once ApplySource returned; held in the handler it is answered as failed). + answers, applied := 0, make(chan struct{}) + first.Message = func(_ context.Context, repository, head string, _ int) ([]byte, error) { + if answers++; answers == 1 { + select { + case <-applied: + return nil, errors.New("the forge bridge does not answer") + case <-time.After(10 * time.Second): + return nil, errors.New("the catch-up held the source handler") + } + } + if repository != catalogueRepo || head != merged.Merge { + return nil, fmt.Errorf("no merge of %s at %s", repository, head) + } + return json.Marshal(merged) + } + heads := meshdelivery.SourceByMainValue{Heads: map[string]string{catalogueRepo: merged.Merge}} + var firstLog []string + var logMu sync.Mutex + first.Logf = func(f string, a ...any) { + logMu.Lock() + firstLog = append(firstLog, fmt.Sprintf(f, a...)) + logMu.Unlock() + } + err = first.ApplySource(context.Background(), 5, heads) + close(applied) + first.catchUpWG.Wait() + if err != nil || len(firstSaid) != 0 || !strings.Contains(strings.Join(firstLog, "\n"), "the forge bridge does not answer") { + t.Fatalf("a catch-up whose message is not read said %+v (%v), log %q, want no revision and the bridge's silence said", firstSaid, err, firstLog) + } + err = first.ApplySource(context.Background(), 6, heads) + first.catchUpWG.Wait() + if err != nil || len(firstSaid) != 1 || firstSaid[0].Revision != 1 || + firstSaid[0].Entries != int64(len(builderModules(t))) || firstSaid[0].Source != 6 { + t.Fatalf("the first start's catch-up said %+v (%v), want trunk revision 1 of source 6 with every module", firstSaid, err) + } + // KeepView: a write to the view outside a recompute (here a verdict carried into one module) is the next revision. + viewKey, _ := meshcatalogclient.ModulesKey("avahi") + _ = write(context.Background(), first.Modules, viewKey, func(value []byte, _ bool) ([]byte, bool, error) { + m, _ := meshcatalogclient.DecodeModule(value) + m.Verdict, m.Problems = meshcatalogclient.VerdictRefused, []string{"a field the judge refuses"} + data, err := meshcatalogclient.EncodeModule(m) + return data, false, err + }) + if err := first.KeepView(context.Background()); err != nil || len(firstSaid) != 2 || !reflect.DeepEqual(firstSaid[1].Changed, []string{"avahi"}) { + t.Errorf("KeepView after a write said %+v (%v), want revision 2 naming avahi", firstSaid, err) + } + if view, _, at, err := TrunkView(context.Background(), first.Revisions); err != nil || at != 2 { + t.Errorf("the view at a restart is read at %d (%v), want 2", at, err) + } else if raw, _, _, _ := view.Get(context.Background(), viewKey); !strings.Contains(string(raw), "a field the judge refuses") { + t.Errorf("the view at a restart lost the verdict: %s", raw) + } + // W4.14: the catch-up tool (indexer_take_in with no module and no repository, as the node agent calls it after a + // restore) takes in the head the source names for every repository again, whatever revision is kept, and keeps none + // when nothing moved; a repository whose message is not read is named in unread. + if key, value, err := NewCatchUp("", "node-engine@ace", time.Now()); err != nil || key != CatchUpKey { + t.Fatalf("NewCatchUp = %q (%v), want key %q", key, err, CatchUpKey) + } else if a, err := DecodeTakeIn(value); err != nil || !a.CatchUp { + t.Fatalf("a catch-up ask decodes to %+v (%v), want a catch-up", a, err) + } + asked := answers + if got := first.CatchUp(context.Background()); !got.Stored || answers != asked+1 || len(firstSaid) != 2 || len(got.Unread) != 0 { + t.Errorf("the catch-up answered %+v after %d asks, said %d revisions, want the one repository taken in, no new revision", got, answers-asked, len(firstSaid)) + } + first.Message = func(context.Context, string, string, int) ([]byte, error) { + return nil, errors.New("the forge bridge does not answer") + } + if got := first.CatchUp(context.Background()); got.Stored || !reflect.DeepEqual(got.Unread, []string{catalogueRepo}) { + t.Errorf("a catch-up whose message is not read answered %+v, want %s unread", got, catalogueRepo) + } + // The tools bundle answers modules and graph from the newest trunk revision, their marks at its number; with none kept + // it says it waits (view-waits), never an empty catalog. + trunkStates := TrunkStates{Objects: first.Revisions, Under: func(string) views.Source { return nil }} + if keys, err := trunkStates.State("modules").Keys(); err != nil || len(keys) != len(builderModules(t)) { + t.Errorf("the bundle's modules = %v (%v), want every module of revision 2", keys, err) + } + if e, ok, err := trunkStates.State(views.MarksState).Get("graph"); err != nil || !ok || !strings.Contains(string(e.Value), `"revision":2`) { + t.Errorf("the bundle's mark of graph = %+v %v (%v), want revision 2", e, ok, err) + } + if _, err := (TrunkStates{Objects: newMemObjects(0)}).State("graph").Keys(); !errors.Is(err, ErrNoRevision) { + t.Errorf("the bundle's graph with no revision kept answered %v, want it to wait", err) + } + // W4.4, ADR 0663: the seat mesh-catalog's verb revision answers, from the indexer's own objects, exactly the parts whose + // entry in the newest trunk revision declares it, the holder refused too; a candidate's by its group and heads; none + // kept, everyone waits. + verbObjects := newMemObjects(0) + keep := func(rev meshcatalog.CatalogRevision, declares map[string]bool) { + entries := map[string][]byte{} + for _, name := range []string{"avahi", Module, "mesh-grants", "node-tools"} { + e := meshcatalog.CatalogEntry{Module: name, Repository: catalogueRepo} + if declares[name] { + e.Invokes = []string{RevisionInvoke} + } + data, _ := json.Marshal(e) + digest := fmt.Sprintf("%s-%d-%s", rev.Kind, rev.Revision, name) + entries[digest] = data + rev.Entries = append(rev.Entries, meshcatalog.EntryRef{Module: name, Digest: digest}) + } + if err := KeepRevision(context.Background(), verbObjects, rev, entries); err != nil { + t.Fatal(err) + } + } + if _, err := Revision(context.Background(), verbObjects, Module, "n1.node-tools", meshcatalog.RevisionInput{}); !errors.Is(err, ErrNoRevision) { + t.Errorf("revision with none kept answered %v, want it to wait", err) + } + keep(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, map[string]bool{"node-tools": true, Module: true}) + keep(meshcatalog.CatalogRevision{Revision: 4, Kind: "candidate", Group: "feat/x", Heads: "0123456789ab"}, map[string]bool{"node-tools": true, "mesh-grants": true}) + for caller, answered := range map[string]bool{"n1.node-tools": true, "n1.mesh-grants": false, "n1." + Module: false, "person.someone": false, "unproven": false} { + got, err := Revision(context.Background(), verbObjects, Module, caller, meshcatalog.RevisionInput{}) + if (err == nil) != answered || (answered && (got.Revision.Revision != 3 || len(got.Entries) != 4 || got.Next != "")) { + t.Errorf("revision to %s answered %+v (%v), want answered %v with trunk revision 3 whole", caller, got, err, answered) + } + } + if got, err := Revision(context.Background(), verbObjects, Module, "n1.node-tools", meshcatalog.RevisionInput{Group: "feat/x", Heads: "0123456789ab", After: "mesh-grants"}); err != nil || + got.Revision.Kind != "candidate" || len(got.Entries) != 1 || got.Entries[0].Module != "node-tools" { + t.Errorf("a candidate's revision after mesh-grants answered %+v (%v), want its one entry node-tools", got, err) + } + if _, err := Revision(context.Background(), verbObjects, Module, "n1.node-tools", meshcatalog.RevisionInput{Revision: 9}); err == nil { + t.Error("a trunk revision not kept was answered") + } // ADR 0614 §2, §3: the graph names what each module invokes and serves, and leaves out, over the whole graph, the // verb edges of two modules calling each other, from either side; a third calling one of them keeps its edge. invoking := func(name, invokes string) mod { diff --git a/modules/mesh-delivery-indexer/internal/index/verdicts.go b/modules/mesh-delivery-indexer/internal/index/verdicts.go index 24ce8159..22bc5764 100644 --- a/modules/mesh-delivery-indexer/internal/index/verdicts.go +++ b/modules/mesh-delivery-indexer/internal/index/verdicts.go @@ -252,6 +252,10 @@ func (ix *Indexer) judgeOverHeld(ctx context.Context, fresh map[string]bool, ove ix.logf("judge: modules.%s does not carry its verdict: %v", name, err) } } + // The verdicts carried are kept with the view (KeepView), so a restart reads them back. + if err := ix.KeepView(ctx); err != nil { + ix.logf("judge: the verdicts carried are not kept in a catalog graph revision: %v", err) + } if over != nil { return out, nil } diff --git a/modules/mesh-delivery-indexer/internal/index/views.go b/modules/mesh-delivery-indexer/internal/index/views.go index 7bcffe38..a428ba9d 100644 --- a/modules/mesh-delivery-indexer/internal/index/views.go +++ b/modules/mesh-delivery-indexer/internal/index/views.go @@ -11,10 +11,11 @@ import ( const MarksBucket = Module + "_" + views.MarksState // FirstViewTools are the indexer's first-view tools (ADR 0621 §1, ADR 0622 §1, ADR 0631 §1): the tool, the state it answers, and -// the words of what that state holds. Each answers only its callers list (CallersEnv), which the manifest writes. +// the words of what that state holds. Each answers only its callers list (CallersEnv), which the manifest writes. The fourth +// first view, of which modules exist, is the seat mesh-catalog's verb revision (ADR 0663 §1, RevisionTool), granted by +// declaration, not by a list. var FirstViewTools = []ViewedTool{ {Tool: "manifests", State: "manifests", Words: "every manifest the indexer keeps, by its digest."}, - {Tool: "modules", State: "modules", Words: "every module the indexer holds, its fingerprint, inputs and manifest digest."}, {Tool: "verdicts", State: VerdictsState, Words: "the judge's verdict on each manifest: accepted, waiting or refused."}, {Tool: "graph", State: graph.State, Words: "the module graph: each module's build, provision and verb edges, whose verb edges a want waits on."}, } @@ -43,13 +44,11 @@ type ViewedTool struct { // ViewedStates are the states the indexer's views family says, each by its bucket: the ones its readers read. var ViewedStates = map[string]string{ "manifests": ManifestsBucket, - "modules": ModulesBucket, VerdictsState: VerdictsBucket, PredefinedState: PredefinedBucket, ConfigCandidatesState: ConfigCandidatesBucket, "candidates": CandidatesBucket, PackagesState: PackagesBucket, - graph.State: GraphBucket, HeardState: HeardBucket, "taken": TakenBucket, mcc.FindingsState: mcc.FindingsBucket(Module), diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index cb7db957..8707e686 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -16,7 +16,6 @@ import ( "git.novox.be/novox/mesh-sdk/go/catalogue" mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" - "git.novox.be/novox/mesh-delivery/sdk/graph" "git.novox.be/novox/mesh-sdk/go/views" "mesh-delivery-indexer/internal/index" @@ -57,7 +56,13 @@ type manifest struct { Slug string `json:"slug"` RunsAs string `json:"runs-as"` Streams []stream `json:"streams"` - Claims []struct { + Objects []struct { + Name string `json:"name"` + MaxMiB int `json:"max-mib"` + Form int `json:"form"` + CatchUp string `json:"catch-up"` + } `json:"objects"` + Claims []struct { Name string `json:"name"` Scope string `json:"scope"` Serves []string `json:"serves"` @@ -113,19 +118,26 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build t.Errorf("emits = %v, want the two take-in events (the verdict events are the seat mesh-catalog's)", m.Emits) } // 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard. - if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: "try-recovery", History: 1}, {Name: "own-build", History: 1}, {Name: index.PackagesState, History: index.PackagesHistory}, - {Name: graph.State, History: graph.History}, {Name: views.MarksState, History: 1}, {Name: mcc.FindingsState, History: 5}, {Name: views.StatusState, History: 1}, + if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: "try-recovery", History: 1}, {Name: "own-build", History: 1}, {Name: index.PackagesState, History: index.PackagesHistory}, + {Name: views.MarksState, History: 1}, {Name: mcc.FindingsState, History: 5}, {Name: views.StatusState, History: 1}, {Name: index.RequirementsState, History: 5}, {Name: index.SeatDefinitionsState, History: 5}, {Name: index.HoldsState, History: 5}, {Name: index.EngineReadsState, History: 1}, {Name: index.HoldBoundsState, History: 1, KeyTTL: true}, {Name: index.PendingState, History: 1}, {Name: index.UnjudgedState, History: 1}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17, ADR 0632 §2 t.Errorf("state = %+v, want %+v", m.State, want) } + // Milestone 47/G W4.2 (design 47, Two graphs; ADR 0619 §2): the catalog graph revisions are objects in the old section, + // capped at 512 MiB, under the local name the indexer opens. W4.14 (design 47, Reverted): their form is 1, and the + // catch-up tool a restore of them is followed by is indexer_take_in, which the bundle serves. + if len(m.Objects) != 1 || m.Objects[0].Name != index.CatalogRevisionsObjects || m.Objects[0].MaxMiB != 512 || + m.Objects[0].Form != 1 || m.Objects[0].CatchUp != index.TakeInTool || !slices.Contains(m.Tools, index.TakeInTool) { + t.Errorf("objects = %+v, want %s capped at 512 MiB, form 1, catch-up %s (a tool of its own)", m.Objects, index.CatalogRevisionsObjects, index.TakeInTool) + } // D-11 (ADR 0634 §1): the holders and buckets the indexer names (internal/index/names.go) are its manifest's: its own // name and states with their revisions, and the states it reads, each as `.`. if index.Module != m.Module { t.Errorf("the indexer names itself %q, its manifest's module is %q", index.Module, m.Module) } - for bucket, history := range map[string]int{index.CandidatesBucket: index.CandidatesHistory, index.ModulesBucket: index.ModulesHistory, index.ManifestsBucket: index.ManifestsHistory, - index.VerdictsBucket: index.VerdictsHistory, index.PackagesBucket: index.PackagesHistory, index.TakenBucket: 1, index.PredefinedBucket: 5, index.ConfigCandidatesBucket: 5, index.GraphBucket: graph.History} { + for bucket, history := range map[string]int{index.CandidatesBucket: index.CandidatesHistory, index.ManifestsBucket: index.ManifestsHistory, + index.VerdictsBucket: index.VerdictsHistory, index.PackagesBucket: index.PackagesHistory, index.TakenBucket: 1, index.PredefinedBucket: 5, index.ConfigCandidatesBucket: 5} { if !slices.ContainsFunc(m.State, func(st state) bool { return m.Module+"_"+st.Name == bucket && st.History == history }) { t.Errorf("the indexer names its bucket %s with %d revisions, its manifest's states are %+v", bucket, history, m.State) } @@ -163,25 +175,30 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none. The same bundle // reads the verdicts state (indexer_verdicts) and asks the indexer to judge again through the takeins state // (indexer_judge; issue 774, clause 3). 1966/AC11: indexer_old_sections reads the count of the old sections (state oldsections). - // ADR 0621 §1, ADR 0622 §1, ADR 0631 §1: the first-view tools manifests, modules, verdicts and graph, and design 25 § 12.2's read + // ADR 0621 §1, ADR 0622 §1, ADR 0631 §1, ADR 0663 §1: the first-view tools manifests, verdicts and graph (modules is the seat + // verb revision's, served by the claim), and design 25 § 12.2's read // tools, all of the tools bundle, which runs in a runtime of its own on the indexer's account (runs-as). want := []string{"indexer_take_in", "indexer_old_sections", "indexer_verdicts", "indexer_judge"} for _, v := range append(append([]index.ViewedTool{}, index.FirstViewTools...), index.ReadTools...) { want = append(want, v.Tool) - if _, ok := index.ViewedStates[v.State]; !ok { + // Milestone 47/G: modules and graph are the newest catalog graph revision's, which catalog-revision says the changes of. + if _, ok := index.ViewedStates[v.State]; !ok && v.State != index.ModulesState && v.State != "graph" { t.Errorf("%s answers the state %s, which the views family does not say", v.Tool, v.State) } } want = append(want, "indexer_first_views", "indexer_first_view_again", index.ConfigReadersTool, index.BusStateTool, index.RequirementsTool, index.PackageUsersTool, index.SayAcceptedTool, // DLV-F3r, DLV-I10, DLV-I11, DLV-I7, DLV-I8, DLV-I9c index.SeatDefinitionTool, index.SeatDefinitionsAgainTool, index.OutOfStepTool, // DLV-I19 index.HoldsTool, index.EngineReadsTool, // ADR 0632 §2: the mesh-wide holds and the engines' reads they weigh - index.PendingTool, index.TakeInPendingTool, index.DropPendingTool, index.TryCandidatesTool, index.DropTryCandidateTool, index.RecoverTriesTool, index.TakeInMergeTool, index.FingerprintsAgainTool) // issue 1966; 1585/T12, T13 (ADR 0633 §5) - if !reflect.DeepEqual(m.Tools, want) || len(want) != 40 { + index.PendingTool, index.TakeInPendingTool, index.DropPendingTool, index.TryCandidatesTool, index.DropTryCandidateTool, index.RecoverTriesTool, index.TakeInMergeTool, index.FingerprintsAgainTool, // issue 1966; 1585/T12, T13 (ADR 0633 §5) + // milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered. + index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool) + if !reflect.DeepEqual(m.Tools, want) || len(want) != 45 || slices.Contains(m.Tools, "modules") { t.Errorf("tools = %v, want %v", m.Tools, want) } // Every read tool of the indexer a reader of this repository invokes is one it serves (the checker's seventh first // view, indexer_read_seat_definitions, was invoked and not served). readers, _ := filepath.Glob("../*/module.json") + var declaring []string for _, p := range readers { raw, err := os.ReadFile(p) if err != nil { @@ -194,24 +211,36 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build t.Fatalf("%s: %v", p, err) } for _, inv := range r.Invokes { + // The grants part reads modules until its own subtask (W8) moves it to revision; the join removes this line. + if p == "../mesh-grants/module.json" && inv == "mesh-delivery-indexer.modules" { + continue + } if tool, ok := strings.CutPrefix(inv, "mesh-delivery-indexer."); ok && !slices.Contains(m.Tools, tool) { t.Errorf("%s invokes %s, which the indexer does not serve", p, inv) } + if inv == index.RevisionInvoke { + declaring = append(declaring, filepath.Base(filepath.Dir(p))) + } } } + // ADR 0663 §2: the assigner reads which modules exist by declaring the seat's verb revision (the grants part too, from W8). + if !slices.Contains(declaring, "mesh-delivery-assigner") { + t.Errorf("the parts of this repository declaring %s are %v, want the assigner", index.RevisionInvoke, declaring) + } if m.RunsAs != m.Module || m.SecretsOwner != m.RunsAs { t.Errorf("runs-as = %q, secrets-owner %q: the tools bundle runs on the indexer's own account (ADR 0621 §1)", m.RunsAs, m.SecretsOwner) } // ADR 0621 §2, ADR 0622 §1, ADR 0631 §1: each first-view tool answers exactly its readers; the indexer is on none, and // graph answers the assigner alone. - callers := map[string]string{"manifests": "mesh-grants,node-tools,mesh-delivery-assigner", "modules": "mesh-grants,node-tools,mesh-delivery-assigner", "verdicts": "mesh-grants", + // ADR 0663 §2: revision has no callers list; it answers the parts whose entry declares it. + callers := map[string]string{"manifests": "mesh-grants,node-tools,mesh-delivery-assigner", "verdicts": "mesh-grants", "graph": "mesh-delivery-assigner"} for _, a := range m.Build.Artifacts { if a.Name != "tools" { continue } if len(a.Env) != len(callers) { - t.Errorf("tools env = %v, want only the four callers lists", a.Env) + t.Errorf("tools env = %v, want only the three callers lists", a.Env) } for tool, list := range callers { if got := a.Env[views.CallersEnv(tool)]; got != list { @@ -241,10 +270,10 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build } } // ADR 0604 §2, ADR 0636 §4: it holds the seat mesh-catalog, serving its four verbs and seat-definitions from its tools - // bundle (which stays without a request subject). + // bundle (which stays without a request subject); ADR 0663 §1: and revision, in the same runtime of its own. if len(m.Claims) != 1 || m.Claims[0].Name != index.CatalogSeat || m.Claims[0].Scope != catalogue.ScopeMesh || - !slices.Equal(m.Claims[0].Serves, []string{"modules", "module", "provides", "dependents", index.SeatDefinitionsVerb}) { - t.Errorf("claims = %+v, want mesh-catalog with its five verbs", m.Claims) + !slices.Equal(m.Claims[0].Serves, []string{"modules", "module", "provides", "dependents", index.SeatDefinitionsVerb, "revision"}) { + t.Errorf("claims = %+v, want mesh-catalog with its six verbs", m.Claims) } for _, e := range []string{index.SeatDefinitionChangedEvent, index.SeatDefinitionRemovedEvent} { if slices.Contains(m.Emits, e) { diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 0392d211..28251b25 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -12,7 +12,8 @@ "module", "provides", "dependents", - "seat-definitions" + "seat-definitions", + "revision" ] } ], @@ -22,7 +23,6 @@ "indexer_verdicts", "indexer_judge", "manifests", - "modules", "verdicts", "graph", "indexer_read_manifests", @@ -56,7 +56,12 @@ "indexer_drop_try_candidate", "indexer_recover_tries", "indexer_take_in_merge", - "indexer_fingerprints_again" + "indexer_fingerprints_again", + "indexer_catalog_revision", + "indexer_catalog_revisions", + "indexer_candidate_graphs", + "indexer_recompute", + "indexer_drop_candidate_graph" ], "consumes": [ "go-package-registry.published", @@ -77,7 +82,6 @@ { "name": "candidates", "history": 10 }, { "name": "heard", "history": 64 }, { "name": "manifests", "history": 1 }, - { "name": "modules", "history": 20 }, { "name": "predefined", "history": 5 }, { "name": "config-candidates", "history": 5 }, { "name": "taken", "history": 1 }, @@ -87,7 +91,6 @@ { "name": "try-recovery", "history": 1 }, { "name": "own-build", "history": 1 }, { "name": "packages", "history": 5 }, - { "name": "graph", "history": 5 }, { "name": "view-marks", "history": 1 }, { "name": "findings", "history": 5 }, { "name": "first-views", "history": 1 }, @@ -99,6 +102,9 @@ { "name": "pending", "history": 1 }, { "name": "unjudged", "history": 1 } ], + "objects": [ + { "name": "catalog-revisions", "max-mib": 512, "form": 1, "catch-up": "indexer_take_in" } + ], "emits": [ "take-in.pending", "take-in.taken", @@ -194,7 +200,6 @@ ], "env": { "MESH_VIEW_CALLERS_MANIFESTS": "mesh-grants,node-tools,mesh-delivery-assigner", - "MESH_VIEW_CALLERS_MODULES": "mesh-grants,node-tools,mesh-delivery-assigner", "MESH_VIEW_CALLERS_VERDICTS": "mesh-grants", "MESH_VIEW_CALLERS_GRAPH": "mesh-delivery-assigner" }