diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index 09fd60658..d1bf3dcac 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -212,6 +212,7 @@ func main() { _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.TakeInSkipTool, Verb: "take-in-skip"}) return err })...) + tools = append(tools, index.DryRunsTools(keptRead{stdio.State(index.DryRunsState)})...) tools = append(tools, index.RunsTools(keptRead{stdio.State(index.RunsState)}, keptRead{stdio.State(index.CandidatesState)}, asks, time.Now, func(what, why string) error { _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.RunDropTool, Verb: "run-drop"}) return err diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index 737b27c38..c9bd95909 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -363,6 +363,10 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before . if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, index.ManifestsState), History: index.ManifestsHistory}); err != nil { t.Fatal(err) } + // The dry runs' scratch, made by the grants part from the manifest's state entry dry-runs (design 47 H2 §804, §820). + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, index.DryRunsState), History: 1, TTL: time.Hour, MaxBytes: 16 << 20}); err != nil { + t.Fatal(err) + } // The builder's module log, as the runtime raises it (ADR 0297), filled by put before the indexer starts. if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(builds.BuilderModule, builds.BuilderLog), Subjects: []string{buscompose.LogSubject(builds.BuilderModule, builds.BuilderLog) + ".>"}, AllowDirect: true}); err != nil { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index aca38fc31..043bad17d 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -333,7 +333,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro defer stopInput() // The state takeins: hand acts ask there (watched below), and a running take-in keeps what it waits on (H2 §803, H-A-409). takeIns := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.TakeInsState), "no hand act of the indexer is answered", logger.Printf) - ix := &index.Indexer{Module: cred.Module, TakeIns: awaitedStore{b: takeIns}, NamedSets: named, Findings: writer, OldSections: awaitedStore{b: oldSections}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: modules, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known, + // The state dry-runs, a dry run's scratch of the packages it writes (H2 §804): every key it holds goes at this start, before + // any dry run, so no dry run outlives a start. + dryRuns := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.DryRunsState), "no dry run is computed: a branch's take-in, a run's candidate and a branch's re-pin wait for it", logger.Printf) + dryRuns.Ready(func(kv jetstream.KeyValue) { + if err := index.ClearDryRuns(ctx, kvStore{kv}); err != nil { + logger.Printf("the state %s is not cleared at this start: %v", index.DryRunsState, err) + } + }) + ix := &index.Indexer{Module: cred.Module, TakeIns: awaitedStore{b: takeIns}, DryRuns: awaitedStore{b: dryRuns}, NamedSets: named, Findings: writer, OldSections: awaitedStore{b: oldSections}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: modules, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known, Predefined: awaitedStore{b: predefined}, Logf: logger.Printf, Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), BranchHead: branchHeadVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call), Packages: awaitedStore{b: packages, js: js, name: index.Bucket(cred.Module, index.PackagesState)}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call), diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go index e2f463232..9976e707b 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go @@ -5,7 +5,9 @@ package main import ( "context" "encoding/json" + "errors" "os" + "reflect" "slices" "strconv" "strings" @@ -62,14 +64,17 @@ func untilModule(t *testing.T, ctx context.Context, js jetstream.JetStream, name // go.sum line names it, ADR 0573 §2; design 47 H-A-157), and plain, which requires // nothing of it, are taken in with mesh-sdk as main's first source revision. A source revision moving mesh-sdk to a commit // changing its buildstep package is taken in as one revision whose movers are mesh-sdk and importer, of another repository, -// and not plain; the packages key of mesh-sdk/go names that commit with a digest per package. +// and not plain; the packages key of mesh-sdk/go names that commit with a digest per package. Design 47 H2 §804: a key a dry +// run left in the state dry-runs is gone at the start, and the take-in of a branch moving mesh-sdk again, a dry run, leaves +// the trunk's packages state byte for byte as before and no key in dry-runs once it answered. func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) { const ( sdkGo = "git.novox.be/novox/mesh-sdk/go" - m1, m2 = "1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222" + m1, m2, m3 = "1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222", "4444444444444444444444444444444444444444" catalogMerge = "3333333333333333333333333333333333333333" goModContent = "module importer\n\nrequire git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000\n" sdkRoot1, sdk2 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + sdk3 = "cccccccccccccccccccccccccccccccccccccccc" ) avahi, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json") if err != nil { @@ -92,7 +97,7 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) { "package.json": "3333333333333333333333333333333333333333", "package-lock.json": "4444444444444444444444444444444444444444", "tsconfig.json": "5555555555555555555555555555555555555555"}}} } - var packages jetstream.KeyValue + var packages, dryRuns jetstream.KeyValue var heard *takeIns js, ctx, _, _ := startIndexer(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) { var err error @@ -118,6 +123,14 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) { forge := serveCommits(t, js.Conn()) forge.put(m1, sdkAt(sdkRoot1, "f5", "6666666666666666666666666666666666666666")) forge.put(m2, sdkAt(sdk2, "f6", "7777777777777777777777777777777777777777")) + forge.put(m3, sdkAt(sdk3, "f7", "8888888888888888888888888888888888888888")) + // A dry run's key left by an indexer that stopped mid-run (§804). + if dryRuns, err = js.KeyValue(ctx, index.Bucket(testIndexer, index.DryRunsState)); err != nil { + t.Fatal(err) + } + if _, err := dryRuns.Put(ctx, index.DryRunPrefix("beta", "0123")+index.PackagesState+".left", []byte(`{"value":"e30="}`)); err != nil { + t.Fatal(err) + } forge.put(catalogMerge, repoAt{trees: map[string]string{"modules/importer": "t1", "modules/plain": "t2"}, files: map[string]string{"modules/importer/module.json": named("importer"), "modules/plain/module.json": named("plain"), "modules/importer/go.mod": goModContent, "modules/importer/go.sum": "golang.org/x/sys v0.1.0 h1:x=\n", @@ -125,7 +138,7 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) { regs := registerAvahi(t, ctx, js, "main") for _, r := range []meshcatalog.Registration{{V: 1, Module: "importer", Origin: "novox/mesh-catalog", Path: "modules/importer", Branches: []string{"main"}}, {V: 1, Module: "plain", Origin: "novox/mesh-catalog", Path: "modules/plain", Branches: []string{"main"}}, - {V: 1, Module: "mesh-sdk", Origin: "novox/mesh-sdk", Path: ".", Branches: []string{"main"}}} { + {V: 1, Module: "mesh-sdk", Origin: "novox/mesh-sdk", Path: ".", Branches: []string{"main", "beta"}}} { raw, _ := json.Marshal(r) if _, err := regs.Put(ctx, index.RegistrationKey(r.Module), raw); err != nil { t.Fatal(err) @@ -134,16 +147,40 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) { seedBuildModules(t, ctx, js, nil) heard = hearTakeIns(t, js.Conn()) }) - say := func(source int64, start bool, heads ...meshdelivery.SourceMovedEventHeadsItem) { + sayOn := func(branch string, source int64, start bool, heads ...meshdelivery.SourceMovedEventHeadsItem) { t.Helper() - raw, _ := json.Marshal(meshdelivery.SourceMovedEvent{V: 1, Branch: "main", Source: source, Call: "0123456789abcdef", Start: start, Heads: heads}) - if _, err := js.Publish(ctx, meshdelivery.SubjectSourceMoved("main", strconv.FormatInt(source, 10)), raw); err != nil { + raw, _ := json.Marshal(meshdelivery.SourceMovedEvent{V: 1, Branch: branch, Source: source, Call: "0123456789abcdef", Start: start, Heads: heads}) + if _, err := js.Publish(ctx, meshdelivery.SubjectSourceMoved(branch, strconv.FormatInt(source, 10)), raw); err != nil { t.Fatal(err) } } + say := func(source int64, start bool, heads ...meshdelivery.SourceMovedEventHeadsItem) { + t.Helper() + sayOn("main", source, start, heads...) + } + // every is each key of kv with its value. + every := func(kv jetstream.KeyValue) map[string]string { + t.Helper() + out := map[string]string{} + ks, err := kv.Keys(ctx) + if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) { + t.Fatal(err) + } + for _, k := range ks { + e, err := kv.Get(ctx, k) + if err != nil { + t.Fatal(err) + } + out[k] = string(e.Value()) + } + return out + } say(10, true, meshdelivery.SourceMovedEventHeadsItem{Repository: "novox/mesh-sdk", Commit: m1}, meshdelivery.SourceMovedEventHeadsItem{Repository: "novox/mesh-catalog", Commit: catalogMerge}) heard.next(t, "main", 10) heard.acked(t, "main") + if left := every(dryRuns); len(left) != 0 { + t.Errorf("dry-runs holds %v after the start, want every key gone (§804)", left) + } // mesh-sdk has released no version: importer keeps the package for its kind-1 edge, with no pin (H-A-161). untilModule(t, ctx, js, "importer", func(m meshcatalog.ModulesByModuleValue) bool { pin, kept := m.Packages[sdkGo] @@ -169,4 +206,18 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) { pub, err := index.DecodePublishedPackages(v) return v != nil && err == nil && pub.Commit == m2 && len(pub.Packages) == 3 }) + + // A branch's take-in moving mesh-sdk to m3 is a dry run: its packages stay in dry-runs while it runs, the trunk's are + // untouched, and its keys go once it answered (§804). + heard.acked(t, "main") + before := every(packages) + sayOn("beta", 1, true, meshdelivery.SourceMovedEventHeadsItem{Repository: "novox/mesh-catalog", Commit: catalogMerge}, meshdelivery.SourceMovedEventHeadsItem{Repository: "novox/mesh-sdk", Commit: m3}) + heard.next(t, "beta", 1) + heard.acked(t, "beta") + if after := every(packages); !reflect.DeepEqual(after, before) { + t.Errorf("the branch's take-in changed the trunk's packages state:\n%v\nwant\n%v", after, before) + } + if left := every(dryRuns); len(left) != 0 { + t.Errorf("dry-runs holds %v once the branch's take-in answered, want none", left) + } } diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 90dbbde3c..4f4df5137 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -132,6 +132,9 @@ type Indexer struct { Bound func(context.Context, time.Duration) (context.Context, context.CancelFunc) // TakeIns is the state takeins, where a running take-in keeps what it waits on (design 47 H2 §803, H-A-409); nil keeps none. TakeIns Store + // DryRuns is the state dry-runs, a dry run's scratch of the packages it writes (dryrun.go; design 47 H2 §804); nil keeps + // a dry run's packages in memory. + DryRuns Store Builds *Builds // Named asks the forge for the git object of each named path of the module directories given, at a commit // (the seat git's trees with paths; novox/hq ADR 0499 clauses 2 and 3): the indexer is the one reader of a @@ -161,7 +164,7 @@ type Indexer struct { filling atomic.Bool // a FillTaken runs in the background (KickFill) fillAgain atomic.Bool // a kick arrived while it ran underSeats bool // a dry run inside judge.WithSeats: its judge passes do not hold the seat list (verdicts.go) - isDry bool // a dry run (dryrun.go): it shares the live packages state, so it drops no kept package key (packages.go) + isDry bool // a dry run (dryrun.go): its packages are a working copy over the live state, and it drops no kept package key (packages.go) judgeMu sync.Mutex // one judge pass at a time (verdicts.go) keepMu sync.Mutex // one catalog graph revision kept at a time (keepEntries) // Pins is the state pins (pins.go); nil keeps none. PinsPending says how many messages the indexer's durable consumer of the diff --git a/modules/mesh-delivery-indexer/internal/index/dryrun.go b/modules/mesh-delivery-indexer/internal/index/dryrun.go index 2fe12d418..51b3cb61d 100644 --- a/modules/mesh-delivery-indexer/internal/index/dryrun.go +++ b/modules/mesh-delivery-indexer/internal/index/dryrun.go @@ -1,12 +1,20 @@ package index import ( + "context" + "encoding/json" + "errors" "fmt" "maps" "reflect" + "slices" + "strings" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/keys" + "git.novox.be/novox/mesh-sdk/go/views" ) type dryRun struct { @@ -14,12 +22,20 @@ type dryRun struct { modules, graph *overlayStore candidates, taken, predefined *overlayStore oldSections *overlayStore + // scratch is the state dry-runs this dry run's packages are written to, under prefix (§804); nil when the indexer keeps + // none, and the packages then stay in memory. + scratch Store + prefix string + logf func(string, ...any) } -// dry is an indexer over overlays of this one's states (overlay.go), for a run on request's candidate (runcandidates.go): what -// it computes stays in memory and nothing reaches the bus but the manifests and the packages, which clause 1 writes all the -// same. It says no event, kicks nothing in the background, and knows what this one knows of the source and the toolchains. Every field of Indexer is named here (TestDryNamesEveryField holds it so). -func (ix *Indexer) dry() *dryRun { +// dry is an indexer over overlays of this one's states (overlay.go), for a run on request's candidate (runcandidates.go), a +// branch's take-in or re-pin: what it computes stays in memory but the packages it writes, kept in the state dry-runs under +// the branch and call given (scratchStore; design 47 H2 §804, H-A-413), so the trunk's packages state is left untouched; its +// manifests are written create-once by digest as every take-in's (H-A-413's addendum). The caller ends it (end) once it +// answered. It says no event, kicks nothing in the background, and knows what this one knows of the source and the +// toolchains. Every field of Indexer is named here. +func (ix *Indexer) dry(branch, call string) *dryRun { over := func(st Store) (Store, *overlayStore) { if st == nil { return nil, nil @@ -27,8 +43,14 @@ func (ix *Indexer) dry() *dryRun { o := newOverlay(st) return o, o } - d := &dryRun{} - x := &Indexer{Module: ix.Module, Manifests: ix.Manifests, Packages: ix.Packages, JudgeID: ix.JudgeID, Files: ix.Files, GoSum: ix.GoSum, + d := &dryRun{scratch: ix.DryRuns, prefix: DryRunPrefix(branch, call), logf: ix.logf} + packages := ix.Packages + if packages != nil && d.scratch != nil { + packages = &scratchStore{under: ix.Packages, scratch: d.scratch, prefix: d.prefix + PackagesState + "."} + } else if packages != nil { + packages = newOverlay(ix.Packages) + } + x := &Indexer{Module: ix.Module, Manifests: ix.Manifests, Packages: packages, DryRuns: nil, JudgeID: ix.JudgeID, Files: ix.Files, GoSum: ix.GoSum, GoMod: ix.GoMod, Now: ix.Now, Bound: ix.Bound, // A dry run keeps no waiting-on: the take-in it computes for says its waits through the live indexer (H-A-409). TakeIns: nil, Builds: ix.Builds, Named: ix.Named, Trunk: ix.Trunk, BranchHead: ix.BranchHead, Read: ix.Read, Contains: ix.Contains, @@ -85,3 +107,236 @@ func (ix *Indexer) refuseDisagreement(c meshcatalog.CandidatesByGroupValue) erro } return nil } + +// DryRunsState is the indexer's state dry-runs (design 47 H2 §804, §820; H-A-413): a dry run's scratch, the overlay of the +// packages state a dry run writes, keyed ... (DryRunPrefix, the kind the overlaid state's name and +// the name its key), so a dry run leaves the trunk's packages state untouched. The indexer deletes a dry run's keys when it +// answers, and every key at its start; each lives an hour at most (the state's TTL). indexer_dry_runs reads it. +const ( + DryRunsState = "dry-runs" + DryRunsTool = "indexer_dry_runs" +) + +// DryRunPrefix is the prefix of a dry run's keys in the state dry-runs: its branch, escaped as a key's name is, and its call. +func DryRunPrefix(branch, call string) string { + return keys.EscapeName(branch) + "." + keys.EscapeName(call) + "." +} + +// scratchRevision lifts a revision of the state dry-runs above every revision of the state it overlays, as overlayStore does. +const scratchRevision = 1 << 62 + +// scratched is a value of the state dry-runs: the value a dry run wrote over its key, or that it deleted it. +type scratched struct { + Value []byte `json:"value,omitempty"` + Deleted bool `json:"deleted,omitempty"` +} + +// scratchStore is a store whose writes go to the state dry-runs under its prefix, over the store it reads through: a key +// it wrote reads as written, a key it deleted as absent, every other as the store under it holds it. Its compare-and-set +// holds against what it reads, a key it wrote carrying the dry-runs revision lifted by scratchRevision. +type scratchStore struct { + under, scratch Store + prefix string +} + +func (s *scratchStore) held(ctx context.Context, key string) (scratched, uint64, bool, error) { + raw, rev, found, err := s.scratch.Get(ctx, s.prefix+key) + if err != nil || !found { + return scratched{}, 0, false, err + } + var v scratched + if err := json.Unmarshal(raw, &v); err != nil { + return v, 0, false, fmt.Errorf("%s.%s%s: %w", DryRunsState, s.prefix, key, err) + } + return v, rev, true, nil +} + +func (s *scratchStore) Get(ctx context.Context, key string) ([]byte, uint64, bool, error) { + v, rev, found, err := s.held(ctx, key) + if err != nil { + return nil, 0, false, err + } + if found { + if v.Deleted { + return nil, rev + scratchRevision, false, nil + } + return v.Value, rev + scratchRevision, true, nil + } + if s.under == nil { + return nil, 0, false, nil + } + return s.under.Get(ctx, key) +} + +// put checks the revision read against what the key reads as now and writes it into the state dry-runs. +func (s *scratchStore) put(ctx context.Context, key string, value []byte, revision uint64, create, del bool) error { + _, own, mine, err := s.held(ctx, key) + if err != nil { + return err + } + _, rev, found, err := s.Get(ctx, key) + if err != nil { + return err + } + if create && found || !create && (!found || rev != revision) { + return ErrConflict + } + data, err := json.Marshal(scratched{Value: value, Deleted: del}) + if err != nil { + return err + } + if mine { + return s.scratch.Update(ctx, s.prefix+key, data, own) + } + return s.scratch.Create(ctx, s.prefix+key, data) +} + +func (s *scratchStore) Create(ctx context.Context, key string, value []byte) error { + return s.put(ctx, key, value, 0, true, false) +} + +func (s *scratchStore) Update(ctx context.Context, key string, value []byte, revision uint64) error { + return s.put(ctx, key, value, revision, false, false) +} + +func (s *scratchStore) Delete(ctx context.Context, key string, revision uint64) error { + return s.put(ctx, key, nil, revision, false, true) +} + +func (s *scratchStore) Keys(ctx context.Context) ([]string, error) { + var out []string + if s.under != nil { + var err error + if out, err = s.under.Keys(ctx); err != nil { + return nil, err + } + } + all, err := s.scratch.Keys(ctx) + if err != nil { + return nil, err + } + mine := map[string]bool{} + for _, k := range all { + if !strings.HasPrefix(k, s.prefix) { + continue + } + key := strings.TrimPrefix(k, s.prefix) + v, _, found, err := s.held(ctx, key) + if err != nil { + return nil, err + } + if found { + mine[key] = !v.Deleted + } + } + out = slices.DeleteFunc(out, func(k string) bool { _, ok := mine[k]; return ok }) + for k, live := range mine { + if live { + out = append(out, k) + } + } + slices.Sort(out) + return out, nil +} + +// end deletes the dry run's keys from the state dry-runs, once it answered (§804); a key it cannot delete is said, and goes +// with the state's TTL or at the indexer's next start. +func (d *dryRun) end(ctx context.Context) { + if d == nil || d.scratch == nil { + return + } + if err := dropPrefixed(ctx, d.scratch, d.prefix); err != nil { + d.logf("the dry run's keys %s* are not all deleted from %s: %v", d.prefix, DryRunsState, err) + } +} + +// ClearDryRuns deletes every key of the state dry-runs, at the indexer's start (§804): no dry run outlives a start. +func ClearDryRuns(ctx context.Context, st Store) error { return dropPrefixed(ctx, st, "") } + +func dropPrefixed(ctx context.Context, st Store, prefix string) error { + all, err := st.Keys(ctx) + if err != nil { + return err + } + var errs []error + for _, k := range all { + if !strings.HasPrefix(k, prefix) { + continue + } + _, rev, found, err := st.Get(ctx, k) + if err == nil && found { + err = st.Delete(ctx, k, rev) + } + if err != nil && !errors.Is(err, ErrConflict) { + errs = append(errs, fmt.Errorf("%s: %w", k, err)) + } + } + return errors.Join(errs...) +} + +// DryRunKey is one key of the state dry-runs as indexer_dry_runs answers it: the key, the value the dry run wrote (none when +// it deleted the key) and whether it deleted it. +type DryRunKey struct { + Key string `json:"key"` + Value json.RawMessage `json:"value,omitempty"` + Deleted bool `json:"deleted,omitempty"` +} + +// DryRunsAnswer is indexer_dry_runs' answer: the keys in key order up to views.PageBytes, capped when more are held, and +// how many the state holds in all. +type DryRunsAnswer struct { + Keys []DryRunKey `json:"keys"` + Capped bool `json:"capped,omitempty"` + Total int `json:"total"` +} + +// ReadDryRuns is what the state dry-runs holds now, at most views.PageBytes of values (§804, §811). +func ReadDryRuns(ctx context.Context, st Store) (DryRunsAnswer, error) { + out := DryRunsAnswer{Keys: []DryRunKey{}} + if st == nil { + return out, nil + } + all, err := st.Keys(ctx) + if err != nil { + return out, err + } + slices.Sort(all) + out.Total = len(all) + size := 0 + for _, k := range all { + raw, _, found, err := st.Get(ctx, k) + if err != nil { + return out, err + } + if !found { + continue + } + var v scratched + if err := json.Unmarshal(raw, &v); err != nil { + return out, fmt.Errorf("%s.%s: %w", DryRunsState, k, err) + } + if size += len(k) + len(v.Value); size > views.PageBytes && len(out.Keys) > 0 { + out.Capped = true + break + } + e := DryRunKey{Key: k, Deleted: v.Deleted} + if !v.Deleted { + e.Value = v.Value + } + out.Keys = append(out.Keys, e) + } + return out, nil +} + +// DryRunsTools are the dry runs' tools: indexer_dry_runs reads the state dry-runs (§804). +func DryRunsTools(st Store) []stdio.Tool { + return []stdio.Tool{{Name: DryRunsTool, + Description: "The dry runs' scratch state dry-runs (design 47 H2 §804): the packages a branch's take-in, a run's candidate " + + "or a branch's re-pin computes, kept as ..packages. while it runs, so the trunk's packages state " + + "is left untouched; a dry run's keys go when it answers and every key at the indexer's start. Each key with the " + + "value it wrote or deleted, up to 768 KB (capped) and the total held.", + Input: map[string]any{"type": "object", "properties": map[string]any{}}, + Run: func(map[string]any) (any, error) { + return ReadDryRuns(context.Background(), st) + }}} +} diff --git a/modules/mesh-delivery-indexer/internal/index/packages_gone_test.go b/modules/mesh-delivery-indexer/internal/index/packages_gone_test.go index 56ac91e81..4c3b23c0d 100644 --- a/modules/mesh-delivery-indexer/internal/index/packages_gone_test.go +++ b/modules/mesh-delivery-indexer/internal/index/packages_gone_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "reflect" "strings" "testing" "time" @@ -115,18 +116,26 @@ func TestARepositorysPackagesAreThoseInTheMergesTree(t *testing.T) { }) for _, row := range []struct { - name string - dry bool - branch string + name string + dry, scratch bool + branch string }{ - {"a branch's take-in keeps the key: only the trunk's merge says a directory is gone", false, "feat/x"}, - {"a dry run keeps the key: it shares the live packages state", true, TrunkBranch}, + {"a branch's take-in keeps the key: only the trunk's merge says a directory is gone", false, false, "feat/x"}, + {"a dry run keeps the key: its packages are a working copy over the live state", true, false, TrunkBranch}, + // Design 47 H2 §804: over the state dry-runs, a dry run's packages are kept there alone, under its branch and call, + // read by indexer_dry_runs and gone at its end; the live packages state is left as it was. + {"a dry run over the state dry-runs writes its packages there alone and drops them at its end", true, true, TrunkBranch}, } { t.Run(row.name, func(t *testing.T) { s := newSetup(t, live, gone) - ix := s.ix + if row.scratch { + s.ix.DryRuns = newMem() + } + before := s.ix.publishedAll(ctx) + ix, d := s.ix, (*dryRun)(nil) if row.dry { - ix = s.ix.dry().ix + d = s.ix.dry("beta", "t") + ix = d.ix } if err := ix.publish(ctx, catalogueRepo, merge, reaching(live), false, row.branch); err != nil { t.Fatal(err) @@ -134,6 +143,24 @@ func TestARepositorysPackagesAreThoseInTheMergesTree(t *testing.T) { if _, kept := s.ix.publishedAll(ctx)[gone]; !kept { t.Errorf("the kept key of %s was dropped", gone) } + if !row.scratch { + return + } + if got := s.ix.publishedAll(ctx); !reflect.DeepEqual(got, before) { + t.Errorf("the live packages state is %+v after the dry run, want %+v as before", got, before) + } + if mine := ix.publishedAll(ctx); mine[live].Commit != merge { + t.Errorf("the dry run reads %s at %q, want its own write at %s", live, mine[live].Commit, merge) + } + read, err := DryRunsTools(s.ix.DryRuns)[0].Run(map[string]any{}) + a, _ := read.(DryRunsAnswer) + if err != nil || a.Total == 0 || len(a.Keys) != a.Total || !strings.HasPrefix(a.Keys[0].Key, DryRunPrefix("beta", "t")+PackagesState+".") { + t.Errorf("indexer_dry_runs = %+v (%v), want the dry run's packages under beta.t.packages.", read, err) + } + d.end(ctx) + if a, err := ReadDryRuns(ctx, s.ix.DryRuns); err != nil || a.Total != 0 { + t.Errorf("dry-runs holds %+v (%v) after the dry run's end, want none", a, err) + } }) } diff --git a/modules/mesh-delivery-indexer/internal/index/runcandidates.go b/modules/mesh-delivery-indexer/internal/index/runcandidates.go index bfcfead88..9353b42b1 100644 --- a/modules/mesh-delivery-indexer/internal/index/runcandidates.go +++ b/modules/mesh-delivery-indexer/internal/index/runcandidates.go @@ -211,6 +211,11 @@ func (ix *Indexer) writeRunCandidate(ctx context.Context, group string, members pulls := make([]meshcatalog.CandidatesByGroupValuePullsItem, len(placed)) // One take-in per base branch: the members on one base are taken in together, so one member sees another's. bases := map[string]*dryRun{} + defer func() { + for _, d := range bases { + d.end(ctx) // the run's scratch goes once it answered (design 47 H2 §804) + } + }() for i, m := range placed { d, ok := bases[m.base] if !ok { @@ -218,7 +223,7 @@ func (ix *Indexer) writeRunCandidate(ctx context.Context, group string, members if err != nil { return fmt.Errorf("the run %s: the revision view of %s: %w", group, m.base, err) } - d = ix.dry() + d = ix.dry(m.base, group) d.ix.Modules, d.ix.Graph, d.ix.Taken = view, nodes, newOverlay(nil) d.ix.Logf = func(format string, args ...any) { ix.logf("the run "+group+", computed and not written: "+format, args...) diff --git a/modules/mesh-delivery-indexer/internal/index/takein_source.go b/modules/mesh-delivery-indexer/internal/index/takein_source.go index ecb392210..4d108e3a5 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein_source.go +++ b/modules/mesh-delivery-indexer/internal/index/takein_source.go @@ -79,7 +79,8 @@ func (ix *Indexer) takeInSource(ctx context.Context, e meshdelivery.SourceMovedE if err != nil { return taken, err } - d := ix.dry() + d := ix.dry(e.Branch, e.Call) + defer d.end(ctx) d.ix.Modules, d.ix.Graph, d.ix.Taken = view, nodes, newOverlay(nil) d.ix.Logf = func(format string, args ...any) { ix.logf("take-in of "+e.Branch+": "+format, args...) } x = d.ix diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index 0a4553916..8300429f3 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -636,38 +636,43 @@ func (ix *Indexer) Repin(ctx context.Context, module, version string) (int, erro } named := 0 for _, branch := range branches { - x := ix - if !isTrunkBranch(branch) { - view, nodes, _, err := BranchView(ctx, ix.Revisions, branch) - if err != nil { - return named, err - } - d := ix.dry() - d.ix.Modules, d.ix.Graph, d.ix.Taken = view, nodes, newOverlay(nil) - d.ix.Logf = func(format string, args ...any) { ix.logf("re-pin of "+branch+": "+format, args...) } - x = d.ix - } - only, err := x.repinned(ctx, module) + n, err := ix.repinBranch(ctx, branch, module, version) + named += n if err != nil { return named, err } - if len(only) == 0 { - continue - } - named += len(only) - x.recompute.Lock() - err = x.recomputeLocked(ctx, nil, nil, "", only) - x.recompute.Unlock() - if err != nil { - return named, err - } - if err := ix.keepRepinned(ctx, x, branch, releaseCall(module, version), only); err != nil { - return named, err - } } return named, nil } +// repinBranch re-pins branch's importers of module at version: the trunk in place, another branch as a dry run over its +// revision view, whose scratch goes once it is kept (design 47 H2 §804). It answers how many modules it named. +func (ix *Indexer) repinBranch(ctx context.Context, branch, module, version string) (int, error) { + x, call := ix, releaseCall(module, version) + if !isTrunkBranch(branch) { + view, nodes, _, err := BranchView(ctx, ix.Revisions, branch) + if err != nil { + return 0, err + } + d := ix.dry(branch, call.Call) + defer d.end(ctx) + d.ix.Modules, d.ix.Graph, d.ix.Taken = view, nodes, newOverlay(nil) + d.ix.Logf = func(format string, args ...any) { ix.logf("re-pin of "+branch+": "+format, args...) } + x = d.ix + } + only, err := x.repinned(ctx, module) + if err != nil || len(only) == 0 { + return 0, err + } + x.recompute.Lock() + err = x.recomputeLocked(ctx, nil, nil, "", only) + x.recompute.Unlock() + if err != nil { + return len(only), err + } + return len(only), ix.keepRepinned(ctx, x, branch, call, only) +} + // repinned are the modules the indexer's modules state holds whose packages name a package of module: by their pins, and // by their kept requirements, which name a package they hold no pin of yet (design 47 H-A-204, H-A-165, H-A-167). func (ix *Indexer) repinned(ctx context.Context, module string) (map[string]bool, error) { diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 81ef37ba3..99f0974c0 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -31,6 +31,7 @@ type state struct { History int `json:"history,omitempty"` KeyTTL bool `json:"key-ttl,omitempty"` MaxMiB int `json:"max-mib,omitempty"` + TTL int `json:"ttl-seconds,omitempty"` } type artifact struct { @@ -138,7 +139,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build {Name: index.RequirementsState, History: 5}, {Name: index.SeatDefinitionsState, History: 5}, {Name: index.EntryIndexState, History: 1, MaxMiB: 8}, {Name: index.NamedState, History: 1, MaxMiB: 4}, // design 47 G §695 {Name: index.RegistrationsState, History: 1}, {Name: index.ProposalsState, History: 1, MaxMiB: 4}, - {Name: index.VersionsState, History: 1, MaxMiB: 64}, {Name: index.BuildEdgesState, History: 1}, {Name: index.InputsState, History: 1}, {Name: index.PinsState, History: 1, MaxMiB: index.PinsCapMiB}, {Name: index.HeldState, History: 1}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17 + {Name: index.VersionsState, History: 1, MaxMiB: 64}, {Name: index.BuildEdgesState, History: 1}, {Name: index.InputsState, History: 1}, {Name: index.PinsState, History: 1, MaxMiB: index.PinsCapMiB}, {Name: index.HeldState, History: 1}, + {Name: index.DryRunsState, History: 1, TTL: 3600, MaxMiB: 16}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17; design 47 H2 §804, §820 t.Errorf("state = %+v, want %+v", m.State, want) } // Milestone 47/G W4.2 (design 47, Two graphs; ADR 0619 §2): the catalog graph revisions are objects in the old section, @@ -206,8 +208,9 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered. index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool, index.EntryIndexTool, index.EntryPinsTool, index.NamedDropTool, // design 47 G §675, H-A-204 - index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.PinDropTool, index.PendingTool, index.TakeInPendingTool, index.TakeInAgainTool, index.TakeInSkipTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194) - if !reflect.DeepEqual(m.Tools, want) || len(want) != 44 || !slices.Contains(m.Tools, "modules") { + index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.PinDropTool, index.PendingTool, index.TakeInPendingTool, index.TakeInAgainTool, index.TakeInSkipTool, index.RunsTool, index.RunDropTool, // design 47 G §667, §8, §579, §64 (A-194) + index.DryRunsTool) // design 47 H2 §804 + if !reflect.DeepEqual(m.Tools, want) || len(want) != 45 || !slices.Contains(m.Tools, "modules") { t.Errorf("tools = %v, want %v", m.Tools, want) } // Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section. diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index bc75c22c2..8926fa796 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -70,7 +70,8 @@ "indexer_take_in_again", "indexer_take_in_skip", "indexer_runs", - "indexer_run_drop" + "indexer_run_drop", + "indexer_dry_runs" ], "consumes": [ "go-package-registry.published", @@ -119,7 +120,8 @@ { "name": "build-edges", "history": 1 }, { "name": "inputs", "history": 1 }, { "name": "pins", "history": 1, "max-mib": 4 }, - { "name": "held", "history": 1 } + { "name": "held", "history": 1 }, + { "name": "dry-runs", "history": 1, "ttl-seconds": 3600, "max-mib": 16 } ], "streams": [ { "name": "registration-history", "events": ["registration.*"], "retention": "history", "max-mib": 64 },