diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index a0860700..f469f775 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -1099,7 +1099,7 @@ func TestIndexerJudgesAnUnreadableVerdictAgainAtStart(t *testing.T) { // 1923/AC Manifest: digest, None2: "the manifest cannot be read: claims node-build-go, which is a seat in the mesh's own namespace that it does not define", Inputs2: &meshcatalog.ModulesByModuleValueInputs2{Tree: "tree-avahi", Locks: map[string]string{"go.sum": goSum}}} rev, objects, err := index.CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk"}, - map[string]meshcatalog.CatalogEntry{"avahi": index.CatalogEntryOf("avahi", stale, catalogue.Manifest{}, graph.Node{None: stale.None2})}) + map[string]meshcatalog.CatalogEntry{"avahi": index.CatalogEntryOf("avahi", stale, catalogue.Manifest{}, "", graph.Node{None: stale.None2})}) if err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_graph.go b/modules/mesh-delivery-indexer/internal/index/catalog_graph.go index d1e6f001..ba471139 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_graph.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_graph.go @@ -28,9 +28,9 @@ const CatalogEntryLayout = 1 // its manifest's digest and verdict, the main package its process runs (MainOf), the package versions it builds against // and what the indexer computes its next revision from (Inputs: the lead's ruling on milestone 47/G finding 1), so that // ModuleValueOf gives the module's value back from the entry alone. The acting rules are the revision's to add. -func CatalogEntryOf(module string, cur meshcatalog.ModulesByModuleValue, m catalogue.Manifest, n graph.Node) meshcatalog.CatalogEntry { +func CatalogEntryOf(module string, cur meshcatalog.ModulesByModuleValue, m catalogue.Manifest, raw string, n graph.Node) meshcatalog.CatalogEntry { e := meshcatalog.CatalogEntry{V: CatalogEntryLayout, Module: module, Repository: cur.Repository, Path: cur.Path, - Commit: cur.Commit, Manifest: cur.Manifest, Verdict: cur.Verdict, Fp: n.FP, None: n.None, Bases: n.Bases, + Commit: cur.Commit, Manifest: manifestWhole(raw), ManifestDigest: cur.Manifest, Verdict: cur.Verdict, Fp: n.FP, None: n.None, Bases: n.Bases, Packages: maps.Clone(cur.Packages), Provides: n.Provides, Requires: n.Requires, Awaits: n.Awaits, Invokes: n.Invokes, Serves: n.Serves, LeftOut: n.LeftOut, Main: MainOf(m)} if n.FP == "" && n.None == "" { @@ -60,7 +60,7 @@ func CatalogEntryOf(module string, cur meshcatalog.ModulesByModuleValue, m catal // ruling on milestone 47/G finding 1; ADR 0656 §6: one form of the fact). The old line's fields (the old-form // fingerprint, the reader needs) are not kept. func ModuleValueOf(e meshcatalog.CatalogEntry) meshcatalog.ModulesByModuleValue { - v := meshcatalog.ModulesByModuleValue{V: keys.Version, Repository: e.Repository, Path: e.Path, Commit: e.Commit, Manifest: e.Manifest, + v := meshcatalog.ModulesByModuleValue{V: keys.Version, Repository: e.Repository, Path: e.Path, Commit: e.Commit, Manifest: e.ManifestDigest, Verdict: e.Verdict, Fp2: e.Fp, None2: e.None, Packages: maps.Clone(e.Packages)} if in := e.Inputs; in != nil { v.Toolchains, v.Paths, v.Problems, v.Warnings, v.Unknown = maps.Clone(in.Toolchains), maps.Clone(in.Paths), in.Problems, in.Warnings, in.Unknown @@ -139,3 +139,16 @@ func MainOf(m catalogue.Manifest) string { } return "" } + +// manifestWhole is the validated manifest the entry carries whole (design 47 G §443): the bytes the forge answered, +// read as the object they are; an entry of a module whose manifest is not kept carries an empty object. +func manifestWhole(raw string) map[string]any { + whole := map[string]any{} + if raw == "" { + return whole + } + if err := json.Unmarshal([]byte(raw), &whole); err != nil { + return map[string]any{} + } + return whole +} diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_graph_test.go b/modules/mesh-delivery-indexer/internal/index/catalog_graph_test.go new file mode 100644 index 00000000..5b8f76e3 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/catalog_graph_test.go @@ -0,0 +1,25 @@ +package index + +import ( + "testing" + + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/sdk/graph" + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// Design 47 G §506, §443: an entry carries the manifest whole and the digest of the bytes the forge answered, set at take-in +// and never computed again from the manifest whole; the module's value read back from the entry holds that digest. +func TestAnEntryCarriesItsManifestWholeAndTheDigestOfItsBytes(t *testing.T) { + raw := `{ "module":"avahi", "version":"1" }` // its encoding is not the one a map gives back + digest := meshcatalogclient.ManifestDigest([]byte(raw)) + cur := meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Path: "modules/avahi", Commit: "c", Manifest: digest} + e := CatalogEntryOf("avahi", cur, catalogue.Manifest{}, raw, graph.Node{}) + if e.ManifestDigest != digest || e.Manifest["module"] != "avahi" { + t.Fatalf("entry digest %q manifest %v, want the digest %s of the bytes and the manifest whole", e.ManifestDigest, e.Manifest, digest) + } + if back := ModuleValueOf(e); back.Manifest != digest { + t.Errorf("the module's value read back holds %q, want the entry's digest %s", back.Manifest, digest) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index b044b4bc..efe8ebbc 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -189,11 +189,11 @@ func (ix *Indexer) keepTrunkRevision(ctx context.Context, edges map[string]graph // Every module the view holds is an entry, one the recompute computed no edges for (a manifest it cannot read) too. entries := make(map[string]meshcatalog.CatalogEntry, len(held)) for name, cur := range held { - entries[name] = CatalogEntryOf(name, cur, edges[name].manifest, whole[name]) + entries[name] = CatalogEntryOf(name, cur, edges[name].manifest, ix.manifestBytes(ctx, cur.Manifest), whole[name]) } for name, e := range edges { if _, ok := held[name]; !ok { - entries[name] = CatalogEntryOf(name, e.cur, e.manifest, whole[name]) + entries[name] = CatalogEntryOf(name, e.cur, e.manifest, ix.manifestBytes(ctx, e.cur.Manifest), whole[name]) } } return ix.keepEntries(ctx, entries) @@ -229,12 +229,13 @@ func (ix *Indexer) KeepView(ctx context.Context) error { entries := make(map[string]meshcatalog.CatalogEntry, len(held)) for name, cur := range held { var m catalogue.Manifest + var raw string if cur.Manifest != "" { - if _, read, err := ix.manifestRaw(ctx, cur.Manifest); err == nil { - m = read + if bytes, read, err := ix.manifestRaw(ctx, cur.Manifest); err == nil { + m, raw = read, bytes } } - entries[name] = CatalogEntryOf(name, cur, m, nodes[name]) + entries[name] = CatalogEntryOf(name, cur, m, raw, nodes[name]) } return ix.keepEntries(ctx, entries) } @@ -481,3 +482,15 @@ func (m markSource) Get(state string) (*views.Entry, bool, error) { } return &views.Entry{Value: data, Revision: uint64(at)}, true, nil } + +// manifestBytes is the manifest the module's value names by its digest, as the forge answered it; empty when it is not kept. +func (ix *Indexer) manifestBytes(ctx context.Context, digest string) string { + if digest == "" || ix.Manifests == nil { + return "" + } + raw, _, err := ix.manifestRaw(ctx, digest) + if err != nil { + return "" + } + return raw +} diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index 17b32ef0..0eea9496 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -1105,6 +1105,12 @@ func (ix *Indexer) takeInWholeBy(ctx context.Context, entries []PendingEntry, ow return applied, fmt.Errorf("taking %s (%s#%d) in: %w", e.Key, e.Forge.Repository, e.Forge.Number, err) } } + // What the judge pass of the take-in carried (the verdicts) is kept with the view too; a store that is full refuses it and + // the take-in holds its merges as it holds them for the revision itself (design 47 G §181). + if err := ix.KeepView(ctx); errors.Is(err, ErrObjectsFull) { + ix.release(ctx, claimed) + return applied, err + } for _, e := range claimed { if err := ix.Pending.Delete(ctx, e.Key, e.Revision); err != nil { ix.logf("take-in: %s.%s is taken in and not removed: %v", PendingState, e.Key, err) diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index 86cdaabf..29da34be 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -866,18 +866,18 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) } list, err := tools[CatalogRevisionsTool].Run(map[string]any{}) l, _ := list.(RevisionsAnswer) - if err != nil || len(l.Revisions) != 2 || l.Revisions[0].Name != "candidate.g2.h2" || l.Revisions[1].Name != "trunk.1" || !l.Revisions[1].Current || + if err != nil || len(l.Revisions) != 3 || l.Revisions[0].Name != "candidate.g2.h2" || l.Revisions[1].Name != "trunk.1" || l.Revisions[2].Name != "trunk.2" || !l.Revisions[2].Current || l.Fill.Cap != 2*size || l.Fill.Bytes != store.used() || len(l.Unlisted) != 0 { - t.Errorf("indexer_catalog_revisions = %+v (%v): want candidate.g2.h2 and the current trunk.1, the fill against its cap", l, err) + t.Errorf("indexer_catalog_revisions = %+v (%v): want candidate.g2.h2, trunk.1 of the view and the current trunk.2 of the merge, the fill against its cap", l, err) } if c, err := tools[CandidateGraphsTool].Run(map[string]any{}); err != nil || len(c.(RevisionsAnswer).Revisions) != 1 { t.Errorf("indexer_candidate_graphs = %+v (%v): want g2's alone", c, err) } - if one, err := tools[CatalogRevisionTool].Run(map[string]any{}); err != nil || one.(map[string]any)["name"] != "trunk.1" { + if one, err := tools[CatalogRevisionTool].Run(map[string]any{}); err != nil || one.(map[string]any)["name"] != "trunk.2" { t.Errorf("indexer_catalog_revision = %v (%v): want the newest trunk revision whole", one, err) } - if got, err := tools[RecomputeTool].Run(map[string]any{"why": "a check"}); err != nil || !strings.Contains(got.(*TakenIn).Answer, "equal to the kept revision 1") { - t.Errorf("indexer_recompute = %+v (%v): want equal to the kept revision 1", got, err) + if got, err := tools[RecomputeTool].Run(map[string]any{"why": "a check"}); err != nil || !strings.Contains(got.(*TakenIn).Answer, "equal to the kept revision 2") { + t.Errorf("indexer_recompute = %+v (%v): want equal to the kept revision 2", got, err) } }) t.Run("T7 a toolchain report evaluates pending again: the revision then names the running build and is taken in", func(t *testing.T) { diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index 5de024cf..4465c9fb 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -296,7 +296,7 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // entries := map[string]meshcatalog.CatalogEntry{} for name := range mods.vals { m, _ := moduleOf(t, mods, name) - entries[name] = CatalogEntryOf(name, m, catalogue.Manifest{}, graphOf(t, ix, name)) + entries[name] = CatalogEntryOf(name, m, catalogue.Manifest{}, "", graphOf(t, ix, name)) } // W3.1's main: the Go main package the module's process runs, read from its manifest (here the indexer's own); a // manifest that runs no process names none. @@ -307,7 +307,7 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // if m, err := catalogue.Decode(own); err != nil || MainOf(m) != "cmd/mesh-delivery-indexer" || MainOf(catalogue.Manifest{}) != "" { t.Errorf("the indexer's main = %q (%v), want cmd/mesh-delivery-indexer", MainOf(m), err) } - if e := entries["avahi"]; e.Fp != want["avahi"] || e.Commit == "" || e.Manifest == "" || !reflect.DeepEqual(e.Bases, []string{"mesh-tools"}) || + if e := entries["avahi"]; e.Fp != want["avahi"] || e.Commit == "" || e.ManifestDigest == "" || !reflect.DeepEqual(e.Bases, []string{"mesh-tools"}) || e.Toolchains["go"] != (meshcatalog.BuildEdge{Module: "mesh-build-go", Version: "go1.27.1", Recipe: "1"}) { t.Errorf("the catalog entry of avahi = %+v", e) } @@ -373,7 +373,7 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // gk, _ := graph.Key(name) rawNode, _, found, _ := nodes.Get(context.Background(), gk) if back, err := graph.Decode(rawNode); !found || err != nil || !reflect.DeepEqual(NodeOf(e), back) || back.FP != e.Fp || - !reflect.DeepEqual(back.Bases, e.Bases) || !reflect.DeepEqual(CatalogEntryOf(name, ModuleValueOf(e), catalogue.Manifest{}, back).Bases, e.Bases) { + !reflect.DeepEqual(back.Bases, e.Bases) || !reflect.DeepEqual(CatalogEntryOf(name, ModuleValueOf(e), catalogue.Manifest{}, "", back).Bases, e.Bases) { t.Errorf("the view's node of %s = %+v (%v), want %+v", name, back, err, NodeOf(e)) } } @@ -483,7 +483,7 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // } return json.Marshal(merged) } - heads := meshdelivery.SourceByMainValue{Heads: map[string]string{catalogueRepo: merged.Merge}} + heads := sourceOf(catalogueRepo, merged.Merge) var firstLog []string var logMu sync.Mutex first.Logf = func(f string, a ...any) { @@ -491,13 +491,13 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // firstLog = append(firstLog, fmt.Sprintf(f, a...)) logMu.Unlock() } - err = first.ApplySource(context.Background(), 5, heads) + err = applyS(first, context.Background(), 5, heads) close(applied) first.catchUpWG.Wait() if err != nil || len(firstSaid) != 0 || !strings.Contains(strings.Join(firstLog, "\n"), "the forge bridge does not answer") { t.Fatalf("a catch-up whose message is not read said %+v (%v), log %q, want no revision and the bridge's silence said", firstSaid, err, firstLog) } - err = first.ApplySource(context.Background(), 6, heads) + err = applyS(first, context.Background(), 6, heads) first.catchUpWG.Wait() if err != nil || len(firstSaid) != 1 || firstSaid[0].Revision != 1 || firstSaid[0].Entries != int64(len(builderModules(t))) || firstSaid[0].Source != 6 { diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 8707e686..c8837992 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -192,7 +192,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build index.PendingTool, index.TakeInPendingTool, index.DropPendingTool, index.TryCandidatesTool, index.DropTryCandidateTool, index.RecoverTriesTool, index.TakeInMergeTool, index.FingerprintsAgainTool, // issue 1966; 1585/T12, T13 (ADR 0633 §5) // milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered. index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool) - if !reflect.DeepEqual(m.Tools, want) || len(want) != 45 || slices.Contains(m.Tools, "modules") { + if !reflect.DeepEqual(m.Tools, want) || len(want) != 44 || slices.Contains(m.Tools, "modules") { t.Errorf("tools = %v, want %v", m.Tools, want) } // Every read tool of the indexer a reader of this repository invokes is one it serves (the checker's seventh first