I3: a dry run keeps its packages in its own scratch state

A branch's take-in, a run's candidate and a branch's re-pin compute as dry
runs whose packages went to the trunk's packages state. They now write them
to the state dry-runs under <branch>.<call>.packages.<name>, read through to
the live state, deleted when the dry run answers and every key at the
indexer's start; the manifests stay create-once by digest as every take-in's.
indexer_dry_runs reads the state; the manifest asks for it, 16 MiB, one
hour's TTL.

Builds: design 47 H2 §804, §820; H-A-413 and its addendum
This commit is contained in:
2026-11-22 01:08:34 +01:00
parent 8bd66112f0
commit d719a69b7d
12 changed files with 419 additions and 54 deletions
@@ -212,6 +212,7 @@ func main() {
_, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.TakeInSkipTool, Verb: "take-in-skip"})
return err
})...)
tools = append(tools, index.DryRunsTools(keptRead{stdio.State(index.DryRunsState)})...)
tools = append(tools, index.RunsTools(keptRead{stdio.State(index.RunsState)}, keptRead{stdio.State(index.CandidatesState)}, asks, time.Now, func(what, why string) error {
_, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.RunDropTool, Verb: "run-drop"})
return err
@@ -363,6 +363,10 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before .
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, index.ManifestsState), History: index.ManifestsHistory}); err != nil {
t.Fatal(err)
}
// The dry runs' scratch, made by the grants part from the manifest's state entry dry-runs (design 47 H2 §804, §820).
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, index.DryRunsState), History: 1, TTL: time.Hour, MaxBytes: 16 << 20}); err != nil {
t.Fatal(err)
}
// The builder's module log, as the runtime raises it (ADR 0297), filled by put before the indexer starts.
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(builds.BuilderModule, builds.BuilderLog),
Subjects: []string{buscompose.LogSubject(builds.BuilderModule, builds.BuilderLog) + ".>"}, AllowDirect: true}); err != nil {
@@ -333,7 +333,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
defer stopInput()
// The state takeins: hand acts ask there (watched below), and a running take-in keeps what it waits on (H2 §803, H-A-409).
takeIns := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.TakeInsState), "no hand act of the indexer is answered", logger.Printf)
ix := &index.Indexer{Module: cred.Module, TakeIns: awaitedStore{b: takeIns}, NamedSets: named, Findings: writer, OldSections: awaitedStore{b: oldSections}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: modules, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known,
// The state dry-runs, a dry run's scratch of the packages it writes (H2 §804): every key it holds goes at this start, before
// any dry run, so no dry run outlives a start.
dryRuns := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.DryRunsState), "no dry run is computed: a branch's take-in, a run's candidate and a branch's re-pin wait for it", logger.Printf)
dryRuns.Ready(func(kv jetstream.KeyValue) {
if err := index.ClearDryRuns(ctx, kvStore{kv}); err != nil {
logger.Printf("the state %s is not cleared at this start: %v", index.DryRunsState, err)
}
})
ix := &index.Indexer{Module: cred.Module, TakeIns: awaitedStore{b: takeIns}, DryRuns: awaitedStore{b: dryRuns}, NamedSets: named, Findings: writer, OldSections: awaitedStore{b: oldSections}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: modules, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known,
Predefined: awaitedStore{b: predefined}, Logf: logger.Printf,
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), BranchHead: branchHeadVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call),
Packages: awaitedStore{b: packages, js: js, name: index.Bucket(cred.Module, index.PackagesState)}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call),
@@ -5,7 +5,9 @@ package main
import (
"context"
"encoding/json"
"errors"
"os"
"reflect"
"slices"
"strconv"
"strings"
@@ -62,14 +64,17 @@ func untilModule(t *testing.T, ctx context.Context, js jetstream.JetStream, name
// go.sum line names it, ADR 0573 §2; design 47 H-A-157), and plain, which requires
// nothing of it, are taken in with mesh-sdk as main's first source revision. A source revision moving mesh-sdk to a commit
// changing its buildstep package is taken in as one revision whose movers are mesh-sdk and importer, of another repository,
// and not plain; the packages key of mesh-sdk/go names that commit with a digest per package.
// and not plain; the packages key of mesh-sdk/go names that commit with a digest per package. Design 47 H2 §804: a key a dry
// run left in the state dry-runs is gone at the start, and the take-in of a branch moving mesh-sdk again, a dry run, leaves
// the trunk's packages state byte for byte as before and no key in dry-runs once it answered.
func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) {
const (
sdkGo = "git.novox.be/novox/mesh-sdk/go"
m1, m2 = "1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222"
m1, m2, m3 = "1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222", "4444444444444444444444444444444444444444"
catalogMerge = "3333333333333333333333333333333333333333"
goModContent = "module importer\n\nrequire git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000\n"
sdkRoot1, sdk2 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
sdk3 = "cccccccccccccccccccccccccccccccccccccccc"
)
avahi, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json")
if err != nil {
@@ -92,7 +97,7 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) {
"package.json": "3333333333333333333333333333333333333333", "package-lock.json": "4444444444444444444444444444444444444444",
"tsconfig.json": "5555555555555555555555555555555555555555"}}}
}
var packages jetstream.KeyValue
var packages, dryRuns jetstream.KeyValue
var heard *takeIns
js, ctx, _, _ := startIndexer(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) {
var err error
@@ -118,6 +123,14 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) {
forge := serveCommits(t, js.Conn())
forge.put(m1, sdkAt(sdkRoot1, "f5", "6666666666666666666666666666666666666666"))
forge.put(m2, sdkAt(sdk2, "f6", "7777777777777777777777777777777777777777"))
forge.put(m3, sdkAt(sdk3, "f7", "8888888888888888888888888888888888888888"))
// A dry run's key left by an indexer that stopped mid-run (§804).
if dryRuns, err = js.KeyValue(ctx, index.Bucket(testIndexer, index.DryRunsState)); err != nil {
t.Fatal(err)
}
if _, err := dryRuns.Put(ctx, index.DryRunPrefix("beta", "0123")+index.PackagesState+".left", []byte(`{"value":"e30="}`)); err != nil {
t.Fatal(err)
}
forge.put(catalogMerge, repoAt{trees: map[string]string{"modules/importer": "t1", "modules/plain": "t2"},
files: map[string]string{"modules/importer/module.json": named("importer"), "modules/plain/module.json": named("plain"),
"modules/importer/go.mod": goModContent, "modules/importer/go.sum": "golang.org/x/sys v0.1.0 h1:x=\n",
@@ -125,7 +138,7 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) {
regs := registerAvahi(t, ctx, js, "main")
for _, r := range []meshcatalog.Registration{{V: 1, Module: "importer", Origin: "novox/mesh-catalog", Path: "modules/importer", Branches: []string{"main"}},
{V: 1, Module: "plain", Origin: "novox/mesh-catalog", Path: "modules/plain", Branches: []string{"main"}},
{V: 1, Module: "mesh-sdk", Origin: "novox/mesh-sdk", Path: ".", Branches: []string{"main"}}} {
{V: 1, Module: "mesh-sdk", Origin: "novox/mesh-sdk", Path: ".", Branches: []string{"main", "beta"}}} {
raw, _ := json.Marshal(r)
if _, err := regs.Put(ctx, index.RegistrationKey(r.Module), raw); err != nil {
t.Fatal(err)
@@ -134,16 +147,40 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) {
seedBuildModules(t, ctx, js, nil)
heard = hearTakeIns(t, js.Conn())
})
say := func(source int64, start bool, heads ...meshdelivery.SourceMovedEventHeadsItem) {
sayOn := func(branch string, source int64, start bool, heads ...meshdelivery.SourceMovedEventHeadsItem) {
t.Helper()
raw, _ := json.Marshal(meshdelivery.SourceMovedEvent{V: 1, Branch: "main", Source: source, Call: "0123456789abcdef", Start: start, Heads: heads})
if _, err := js.Publish(ctx, meshdelivery.SubjectSourceMoved("main", strconv.FormatInt(source, 10)), raw); err != nil {
raw, _ := json.Marshal(meshdelivery.SourceMovedEvent{V: 1, Branch: branch, Source: source, Call: "0123456789abcdef", Start: start, Heads: heads})
if _, err := js.Publish(ctx, meshdelivery.SubjectSourceMoved(branch, strconv.FormatInt(source, 10)), raw); err != nil {
t.Fatal(err)
}
}
say := func(source int64, start bool, heads ...meshdelivery.SourceMovedEventHeadsItem) {
t.Helper()
sayOn("main", source, start, heads...)
}
// every is each key of kv with its value.
every := func(kv jetstream.KeyValue) map[string]string {
t.Helper()
out := map[string]string{}
ks, err := kv.Keys(ctx)
if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
t.Fatal(err)
}
for _, k := range ks {
e, err := kv.Get(ctx, k)
if err != nil {
t.Fatal(err)
}
out[k] = string(e.Value())
}
return out
}
say(10, true, meshdelivery.SourceMovedEventHeadsItem{Repository: "novox/mesh-sdk", Commit: m1}, meshdelivery.SourceMovedEventHeadsItem{Repository: "novox/mesh-catalog", Commit: catalogMerge})
heard.next(t, "main", 10)
heard.acked(t, "main")
if left := every(dryRuns); len(left) != 0 {
t.Errorf("dry-runs holds %v after the start, want every key gone (§804)", left)
}
// mesh-sdk has released no version: importer keeps the package for its kind-1 edge, with no pin (H-A-161).
untilModule(t, ctx, js, "importer", func(m meshcatalog.ModulesByModuleValue) bool {
pin, kept := m.Packages[sdkGo]
@@ -169,4 +206,18 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) {
pub, err := index.DecodePublishedPackages(v)
return v != nil && err == nil && pub.Commit == m2 && len(pub.Packages) == 3
})
// A branch's take-in moving mesh-sdk to m3 is a dry run: its packages stay in dry-runs while it runs, the trunk's are
// untouched, and its keys go once it answered (§804).
heard.acked(t, "main")
before := every(packages)
sayOn("beta", 1, true, meshdelivery.SourceMovedEventHeadsItem{Repository: "novox/mesh-catalog", Commit: catalogMerge}, meshdelivery.SourceMovedEventHeadsItem{Repository: "novox/mesh-sdk", Commit: m3})
heard.next(t, "beta", 1)
heard.acked(t, "beta")
if after := every(packages); !reflect.DeepEqual(after, before) {
t.Errorf("the branch's take-in changed the trunk's packages state:\n%v\nwant\n%v", after, before)
}
if left := every(dryRuns); len(left) != 0 {
t.Errorf("dry-runs holds %v once the branch's take-in answered, want none", left)
}
}
@@ -132,6 +132,9 @@ type Indexer struct {
Bound func(context.Context, time.Duration) (context.Context, context.CancelFunc)
// TakeIns is the state takeins, where a running take-in keeps what it waits on (design 47 H2 §803, H-A-409); nil keeps none.
TakeIns Store
// DryRuns is the state dry-runs, a dry run's scratch of the packages it writes (dryrun.go; design 47 H2 §804); nil keeps
// a dry run's packages in memory.
DryRuns Store
Builds *Builds
// Named asks the forge for the git object of each named path of the module directories given, at a commit
// (the seat git's trees with paths; novox/hq ADR 0499 clauses 2 and 3): the indexer is the one reader of a
@@ -161,7 +164,7 @@ type Indexer struct {
filling atomic.Bool // a FillTaken runs in the background (KickFill)
fillAgain atomic.Bool // a kick arrived while it ran
underSeats bool // a dry run inside judge.WithSeats: its judge passes do not hold the seat list (verdicts.go)
isDry bool // a dry run (dryrun.go): it shares the live packages state, so it drops no kept package key (packages.go)
isDry bool // a dry run (dryrun.go): its packages are a working copy over the live state, and it drops no kept package key (packages.go)
judgeMu sync.Mutex // one judge pass at a time (verdicts.go)
keepMu sync.Mutex // one catalog graph revision kept at a time (keepEntries)
// Pins is the state pins (pins.go); nil keeps none. PinsPending says how many messages the indexer's durable consumer of the
@@ -1,12 +1,20 @@
package index
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"reflect"
"slices"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/views"
)
type dryRun struct {
@@ -14,12 +22,20 @@ type dryRun struct {
modules, graph *overlayStore
candidates, taken, predefined *overlayStore
oldSections *overlayStore
// scratch is the state dry-runs this dry run's packages are written to, under prefix (§804); nil when the indexer keeps
// none, and the packages then stay in memory.
scratch Store
prefix string
logf func(string, ...any)
}
// dry is an indexer over overlays of this one's states (overlay.go), for a run on request's candidate (runcandidates.go): what
// it computes stays in memory and nothing reaches the bus but the manifests and the packages, which clause 1 writes all the
// same. It says no event, kicks nothing in the background, and knows what this one knows of the source and the toolchains. Every field of Indexer is named here (TestDryNamesEveryField holds it so).
func (ix *Indexer) dry() *dryRun {
// dry is an indexer over overlays of this one's states (overlay.go), for a run on request's candidate (runcandidates.go), a
// branch's take-in or re-pin: what it computes stays in memory but the packages it writes, kept in the state dry-runs under
// the branch and call given (scratchStore; design 47 H2 §804, H-A-413), so the trunk's packages state is left untouched; its
// manifests are written create-once by digest as every take-in's (H-A-413's addendum). The caller ends it (end) once it
// answered. It says no event, kicks nothing in the background, and knows what this one knows of the source and the
// toolchains. Every field of Indexer is named here.
func (ix *Indexer) dry(branch, call string) *dryRun {
over := func(st Store) (Store, *overlayStore) {
if st == nil {
return nil, nil
@@ -27,8 +43,14 @@ func (ix *Indexer) dry() *dryRun {
o := newOverlay(st)
return o, o
}
d := &dryRun{}
x := &Indexer{Module: ix.Module, Manifests: ix.Manifests, Packages: ix.Packages, JudgeID: ix.JudgeID, Files: ix.Files, GoSum: ix.GoSum,
d := &dryRun{scratch: ix.DryRuns, prefix: DryRunPrefix(branch, call), logf: ix.logf}
packages := ix.Packages
if packages != nil && d.scratch != nil {
packages = &scratchStore{under: ix.Packages, scratch: d.scratch, prefix: d.prefix + PackagesState + "."}
} else if packages != nil {
packages = newOverlay(ix.Packages)
}
x := &Indexer{Module: ix.Module, Manifests: ix.Manifests, Packages: packages, DryRuns: nil, JudgeID: ix.JudgeID, Files: ix.Files, GoSum: ix.GoSum,
GoMod: ix.GoMod, Now: ix.Now, Bound: ix.Bound,
// A dry run keeps no waiting-on: the take-in it computes for says its waits through the live indexer (H-A-409).
TakeIns: nil, Builds: ix.Builds, Named: ix.Named, Trunk: ix.Trunk, BranchHead: ix.BranchHead, Read: ix.Read, Contains: ix.Contains,
@@ -85,3 +107,236 @@ func (ix *Indexer) refuseDisagreement(c meshcatalog.CandidatesByGroupValue) erro
}
return nil
}
// DryRunsState is the indexer's state dry-runs (design 47 H2 §804, §820; H-A-413): a dry run's scratch, the overlay of the
// packages state a dry run writes, keyed <branch>.<call>.<kind>.<name> (DryRunPrefix, the kind the overlaid state's name and
// the name its key), so a dry run leaves the trunk's packages state untouched. The indexer deletes a dry run's keys when it
// answers, and every key at its start; each lives an hour at most (the state's TTL). indexer_dry_runs reads it.
const (
DryRunsState = "dry-runs"
DryRunsTool = "indexer_dry_runs"
)
// DryRunPrefix is the prefix of a dry run's keys in the state dry-runs: its branch, escaped as a key's name is, and its call.
func DryRunPrefix(branch, call string) string {
return keys.EscapeName(branch) + "." + keys.EscapeName(call) + "."
}
// scratchRevision lifts a revision of the state dry-runs above every revision of the state it overlays, as overlayStore does.
const scratchRevision = 1 << 62
// scratched is a value of the state dry-runs: the value a dry run wrote over its key, or that it deleted it.
type scratched struct {
Value []byte `json:"value,omitempty"`
Deleted bool `json:"deleted,omitempty"`
}
// scratchStore is a store whose writes go to the state dry-runs under its prefix, over the store it reads through: a key
// it wrote reads as written, a key it deleted as absent, every other as the store under it holds it. Its compare-and-set
// holds against what it reads, a key it wrote carrying the dry-runs revision lifted by scratchRevision.
type scratchStore struct {
under, scratch Store
prefix string
}
func (s *scratchStore) held(ctx context.Context, key string) (scratched, uint64, bool, error) {
raw, rev, found, err := s.scratch.Get(ctx, s.prefix+key)
if err != nil || !found {
return scratched{}, 0, false, err
}
var v scratched
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, false, fmt.Errorf("%s.%s%s: %w", DryRunsState, s.prefix, key, err)
}
return v, rev, true, nil
}
func (s *scratchStore) Get(ctx context.Context, key string) ([]byte, uint64, bool, error) {
v, rev, found, err := s.held(ctx, key)
if err != nil {
return nil, 0, false, err
}
if found {
if v.Deleted {
return nil, rev + scratchRevision, false, nil
}
return v.Value, rev + scratchRevision, true, nil
}
if s.under == nil {
return nil, 0, false, nil
}
return s.under.Get(ctx, key)
}
// put checks the revision read against what the key reads as now and writes it into the state dry-runs.
func (s *scratchStore) put(ctx context.Context, key string, value []byte, revision uint64, create, del bool) error {
_, own, mine, err := s.held(ctx, key)
if err != nil {
return err
}
_, rev, found, err := s.Get(ctx, key)
if err != nil {
return err
}
if create && found || !create && (!found || rev != revision) {
return ErrConflict
}
data, err := json.Marshal(scratched{Value: value, Deleted: del})
if err != nil {
return err
}
if mine {
return s.scratch.Update(ctx, s.prefix+key, data, own)
}
return s.scratch.Create(ctx, s.prefix+key, data)
}
func (s *scratchStore) Create(ctx context.Context, key string, value []byte) error {
return s.put(ctx, key, value, 0, true, false)
}
func (s *scratchStore) Update(ctx context.Context, key string, value []byte, revision uint64) error {
return s.put(ctx, key, value, revision, false, false)
}
func (s *scratchStore) Delete(ctx context.Context, key string, revision uint64) error {
return s.put(ctx, key, nil, revision, false, true)
}
func (s *scratchStore) Keys(ctx context.Context) ([]string, error) {
var out []string
if s.under != nil {
var err error
if out, err = s.under.Keys(ctx); err != nil {
return nil, err
}
}
all, err := s.scratch.Keys(ctx)
if err != nil {
return nil, err
}
mine := map[string]bool{}
for _, k := range all {
if !strings.HasPrefix(k, s.prefix) {
continue
}
key := strings.TrimPrefix(k, s.prefix)
v, _, found, err := s.held(ctx, key)
if err != nil {
return nil, err
}
if found {
mine[key] = !v.Deleted
}
}
out = slices.DeleteFunc(out, func(k string) bool { _, ok := mine[k]; return ok })
for k, live := range mine {
if live {
out = append(out, k)
}
}
slices.Sort(out)
return out, nil
}
// end deletes the dry run's keys from the state dry-runs, once it answered (§804); a key it cannot delete is said, and goes
// with the state's TTL or at the indexer's next start.
func (d *dryRun) end(ctx context.Context) {
if d == nil || d.scratch == nil {
return
}
if err := dropPrefixed(ctx, d.scratch, d.prefix); err != nil {
d.logf("the dry run's keys %s* are not all deleted from %s: %v", d.prefix, DryRunsState, err)
}
}
// ClearDryRuns deletes every key of the state dry-runs, at the indexer's start (§804): no dry run outlives a start.
func ClearDryRuns(ctx context.Context, st Store) error { return dropPrefixed(ctx, st, "") }
func dropPrefixed(ctx context.Context, st Store, prefix string) error {
all, err := st.Keys(ctx)
if err != nil {
return err
}
var errs []error
for _, k := range all {
if !strings.HasPrefix(k, prefix) {
continue
}
_, rev, found, err := st.Get(ctx, k)
if err == nil && found {
err = st.Delete(ctx, k, rev)
}
if err != nil && !errors.Is(err, ErrConflict) {
errs = append(errs, fmt.Errorf("%s: %w", k, err))
}
}
return errors.Join(errs...)
}
// DryRunKey is one key of the state dry-runs as indexer_dry_runs answers it: the key, the value the dry run wrote (none when
// it deleted the key) and whether it deleted it.
type DryRunKey struct {
Key string `json:"key"`
Value json.RawMessage `json:"value,omitempty"`
Deleted bool `json:"deleted,omitempty"`
}
// DryRunsAnswer is indexer_dry_runs' answer: the keys in key order up to views.PageBytes, capped when more are held, and
// how many the state holds in all.
type DryRunsAnswer struct {
Keys []DryRunKey `json:"keys"`
Capped bool `json:"capped,omitempty"`
Total int `json:"total"`
}
// ReadDryRuns is what the state dry-runs holds now, at most views.PageBytes of values (§804, §811).
func ReadDryRuns(ctx context.Context, st Store) (DryRunsAnswer, error) {
out := DryRunsAnswer{Keys: []DryRunKey{}}
if st == nil {
return out, nil
}
all, err := st.Keys(ctx)
if err != nil {
return out, err
}
slices.Sort(all)
out.Total = len(all)
size := 0
for _, k := range all {
raw, _, found, err := st.Get(ctx, k)
if err != nil {
return out, err
}
if !found {
continue
}
var v scratched
if err := json.Unmarshal(raw, &v); err != nil {
return out, fmt.Errorf("%s.%s: %w", DryRunsState, k, err)
}
if size += len(k) + len(v.Value); size > views.PageBytes && len(out.Keys) > 0 {
out.Capped = true
break
}
e := DryRunKey{Key: k, Deleted: v.Deleted}
if !v.Deleted {
e.Value = v.Value
}
out.Keys = append(out.Keys, e)
}
return out, nil
}
// DryRunsTools are the dry runs' tools: indexer_dry_runs reads the state dry-runs (§804).
func DryRunsTools(st Store) []stdio.Tool {
return []stdio.Tool{{Name: DryRunsTool,
Description: "The dry runs' scratch state dry-runs (design 47 H2 §804): the packages a branch's take-in, a run's candidate " +
"or a branch's re-pin computes, kept as <branch>.<call>.packages.<name> while it runs, so the trunk's packages state " +
"is left untouched; a dry run's keys go when it answers and every key at the indexer's start. Each key with the " +
"value it wrote or deleted, up to 768 KB (capped) and the total held.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) {
return ReadDryRuns(context.Background(), st)
}}}
}
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"reflect"
"strings"
"testing"
"time"
@@ -115,18 +116,26 @@ func TestARepositorysPackagesAreThoseInTheMergesTree(t *testing.T) {
})
for _, row := range []struct {
name string
dry bool
branch string
name string
dry, scratch bool
branch string
}{
{"a branch's take-in keeps the key: only the trunk's merge says a directory is gone", false, "feat/x"},
{"a dry run keeps the key: it shares the live packages state", true, TrunkBranch},
{"a branch's take-in keeps the key: only the trunk's merge says a directory is gone", false, false, "feat/x"},
{"a dry run keeps the key: its packages are a working copy over the live state", true, false, TrunkBranch},
// Design 47 H2 §804: over the state dry-runs, a dry run's packages are kept there alone, under its branch and call,
// read by indexer_dry_runs and gone at its end; the live packages state is left as it was.
{"a dry run over the state dry-runs writes its packages there alone and drops them at its end", true, true, TrunkBranch},
} {
t.Run(row.name, func(t *testing.T) {
s := newSetup(t, live, gone)
ix := s.ix
if row.scratch {
s.ix.DryRuns = newMem()
}
before := s.ix.publishedAll(ctx)
ix, d := s.ix, (*dryRun)(nil)
if row.dry {
ix = s.ix.dry().ix
d = s.ix.dry("beta", "t")
ix = d.ix
}
if err := ix.publish(ctx, catalogueRepo, merge, reaching(live), false, row.branch); err != nil {
t.Fatal(err)
@@ -134,6 +143,24 @@ func TestARepositorysPackagesAreThoseInTheMergesTree(t *testing.T) {
if _, kept := s.ix.publishedAll(ctx)[gone]; !kept {
t.Errorf("the kept key of %s was dropped", gone)
}
if !row.scratch {
return
}
if got := s.ix.publishedAll(ctx); !reflect.DeepEqual(got, before) {
t.Errorf("the live packages state is %+v after the dry run, want %+v as before", got, before)
}
if mine := ix.publishedAll(ctx); mine[live].Commit != merge {
t.Errorf("the dry run reads %s at %q, want its own write at %s", live, mine[live].Commit, merge)
}
read, err := DryRunsTools(s.ix.DryRuns)[0].Run(map[string]any{})
a, _ := read.(DryRunsAnswer)
if err != nil || a.Total == 0 || len(a.Keys) != a.Total || !strings.HasPrefix(a.Keys[0].Key, DryRunPrefix("beta", "t")+PackagesState+".") {
t.Errorf("indexer_dry_runs = %+v (%v), want the dry run's packages under beta.t.packages.", read, err)
}
d.end(ctx)
if a, err := ReadDryRuns(ctx, s.ix.DryRuns); err != nil || a.Total != 0 {
t.Errorf("dry-runs holds %+v (%v) after the dry run's end, want none", a, err)
}
})
}
@@ -211,6 +211,11 @@ func (ix *Indexer) writeRunCandidate(ctx context.Context, group string, members
pulls := make([]meshcatalog.CandidatesByGroupValuePullsItem, len(placed))
// One take-in per base branch: the members on one base are taken in together, so one member sees another's.
bases := map[string]*dryRun{}
defer func() {
for _, d := range bases {
d.end(ctx) // the run's scratch goes once it answered (design 47 H2 §804)
}
}()
for i, m := range placed {
d, ok := bases[m.base]
if !ok {
@@ -218,7 +223,7 @@ func (ix *Indexer) writeRunCandidate(ctx context.Context, group string, members
if err != nil {
return fmt.Errorf("the run %s: the revision view of %s: %w", group, m.base, err)
}
d = ix.dry()
d = ix.dry(m.base, group)
d.ix.Modules, d.ix.Graph, d.ix.Taken = view, nodes, newOverlay(nil)
d.ix.Logf = func(format string, args ...any) {
ix.logf("the run "+group+", computed and not written: "+format, args...)
@@ -79,7 +79,8 @@ func (ix *Indexer) takeInSource(ctx context.Context, e meshdelivery.SourceMovedE
if err != nil {
return taken, err
}
d := ix.dry()
d := ix.dry(e.Branch, e.Call)
defer d.end(ctx)
d.ix.Modules, d.ix.Graph, d.ix.Taken = view, nodes, newOverlay(nil)
d.ix.Logf = func(format string, args ...any) { ix.logf("take-in of "+e.Branch+": "+format, args...) }
x = d.ix
@@ -636,38 +636,43 @@ func (ix *Indexer) Repin(ctx context.Context, module, version string) (int, erro
}
named := 0
for _, branch := range branches {
x := ix
if !isTrunkBranch(branch) {
view, nodes, _, err := BranchView(ctx, ix.Revisions, branch)
if err != nil {
return named, err
}
d := ix.dry()
d.ix.Modules, d.ix.Graph, d.ix.Taken = view, nodes, newOverlay(nil)
d.ix.Logf = func(format string, args ...any) { ix.logf("re-pin of "+branch+": "+format, args...) }
x = d.ix
}
only, err := x.repinned(ctx, module)
n, err := ix.repinBranch(ctx, branch, module, version)
named += n
if err != nil {
return named, err
}
if len(only) == 0 {
continue
}
named += len(only)
x.recompute.Lock()
err = x.recomputeLocked(ctx, nil, nil, "", only)
x.recompute.Unlock()
if err != nil {
return named, err
}
if err := ix.keepRepinned(ctx, x, branch, releaseCall(module, version), only); err != nil {
return named, err
}
}
return named, nil
}
// repinBranch re-pins branch's importers of module at version: the trunk in place, another branch as a dry run over its
// revision view, whose scratch goes once it is kept (design 47 H2 §804). It answers how many modules it named.
func (ix *Indexer) repinBranch(ctx context.Context, branch, module, version string) (int, error) {
x, call := ix, releaseCall(module, version)
if !isTrunkBranch(branch) {
view, nodes, _, err := BranchView(ctx, ix.Revisions, branch)
if err != nil {
return 0, err
}
d := ix.dry(branch, call.Call)
defer d.end(ctx)
d.ix.Modules, d.ix.Graph, d.ix.Taken = view, nodes, newOverlay(nil)
d.ix.Logf = func(format string, args ...any) { ix.logf("re-pin of "+branch+": "+format, args...) }
x = d.ix
}
only, err := x.repinned(ctx, module)
if err != nil || len(only) == 0 {
return 0, err
}
x.recompute.Lock()
err = x.recomputeLocked(ctx, nil, nil, "", only)
x.recompute.Unlock()
if err != nil {
return len(only), err
}
return len(only), ix.keepRepinned(ctx, x, branch, call, only)
}
// repinned are the modules the indexer's modules state holds whose packages name a package of module: by their pins, and
// by their kept requirements, which name a package they hold no pin of yet (design 47 H-A-204, H-A-165, H-A-167).
func (ix *Indexer) repinned(ctx context.Context, module string) (map[string]bool, error) {
@@ -31,6 +31,7 @@ type state struct {
History int `json:"history,omitempty"`
KeyTTL bool `json:"key-ttl,omitempty"`
MaxMiB int `json:"max-mib,omitempty"`
TTL int `json:"ttl-seconds,omitempty"`
}
type artifact struct {
@@ -138,7 +139,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
{Name: index.RequirementsState, History: 5}, {Name: index.SeatDefinitionsState, History: 5},
{Name: index.EntryIndexState, History: 1, MaxMiB: 8}, {Name: index.NamedState, History: 1, MaxMiB: 4}, // design 47 G §695
{Name: index.RegistrationsState, History: 1}, {Name: index.ProposalsState, History: 1, MaxMiB: 4},
{Name: index.VersionsState, History: 1, MaxMiB: 64}, {Name: index.BuildEdgesState, History: 1}, {Name: index.InputsState, History: 1}, {Name: index.PinsState, History: 1, MaxMiB: index.PinsCapMiB}, {Name: index.HeldState, History: 1}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17
{Name: index.VersionsState, History: 1, MaxMiB: 64}, {Name: index.BuildEdgesState, History: 1}, {Name: index.InputsState, History: 1}, {Name: index.PinsState, History: 1, MaxMiB: index.PinsCapMiB}, {Name: index.HeldState, History: 1},
{Name: index.DryRunsState, History: 1, TTL: 3600, MaxMiB: 16}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17; design 47 H2 §804, §820
t.Errorf("state = %+v, want %+v", m.State, want)
}
// Milestone 47/G W4.2 (design 47, Two graphs; ADR 0619 §2): the catalog graph revisions are objects in the old section,
@@ -206,8 +208,9 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
// milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered.
index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool,
index.EntryIndexTool, index.EntryPinsTool, index.NamedDropTool, // design 47 G §675, H-A-204
index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.PinDropTool, index.PendingTool, index.TakeInPendingTool, index.TakeInAgainTool, index.TakeInSkipTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194)
if !reflect.DeepEqual(m.Tools, want) || len(want) != 44 || !slices.Contains(m.Tools, "modules") {
index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.PinDropTool, index.PendingTool, index.TakeInPendingTool, index.TakeInAgainTool, index.TakeInSkipTool, index.RunsTool, index.RunDropTool, // design 47 G §667, §8, §579, §64 (A-194)
index.DryRunsTool) // design 47 H2 §804
if !reflect.DeepEqual(m.Tools, want) || len(want) != 45 || !slices.Contains(m.Tools, "modules") {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
// Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section.
+4 -2
View File
@@ -70,7 +70,8 @@
"indexer_take_in_again",
"indexer_take_in_skip",
"indexer_runs",
"indexer_run_drop"
"indexer_run_drop",
"indexer_dry_runs"
],
"consumes": [
"go-package-registry.published",
@@ -119,7 +120,8 @@
{ "name": "build-edges", "history": 1 },
{ "name": "inputs", "history": 1 },
{ "name": "pins", "history": 1, "max-mib": 4 },
{ "name": "held", "history": 1 }
{ "name": "held", "history": 1 },
{ "name": "dry-runs", "history": 1, "ttl-seconds": 3600, "max-mib": 16 }
],
"streams": [
{ "name": "registration-history", "events": ["registration.*"], "retention": "history", "max-mib": 64 },