Names N3 (indexer): the indexer names its own buckets from its credential's module

Its 26 `<Module>_<state>` bucket constants (candidates, manifests, verdicts, packages, taken, the
control records, pins, pending and the rest), its consumers, findings and first views were named
from the constant Module; they are now index.Bucket(cred.Module, <state>), and its words name
the state where they named the bucket.

Builds: design 47 G A-296

Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
This commit is contained in:
2026-11-12 19:18:56 +01:00
parent 0de534c692
commit cf93d30150
30 changed files with 125 additions and 167 deletions
@@ -225,7 +225,7 @@ func askTakeIn(s stdio.KeptState, bound time.Duration) asker {
}
r, err := s.Put(key, json.RawMessage(value))
if err != nil {
return index.TakeInAsk{}, fmt.Errorf("the ask could not be written to %s: %w", index.TakeInsBucket, err)
return index.TakeInAsk{}, fmt.Errorf("the ask could not be written to %s: %w", index.TakeInsState, err)
}
mu.Lock()
rev = r
@@ -240,7 +240,7 @@ func askTakeIn(s stdio.KeptState, bound time.Duration) asker {
case a := <-answered:
return a, nil
case <-wait.Done():
return index.TakeInAsk{}, fmt.Errorf("the indexer did not answer within %s: the take-in is still running (a forge read can take 30 s); its answer will be in %s.%s", bound, index.TakeInsBucket, key)
return index.TakeInAsk{}, fmt.Errorf("the indexer did not answer within %s: the take-in is still running (a forge read can take 30 s); its answer will be in %s.%s", bound, index.TakeInsState, key)
}
}
}
@@ -250,10 +250,10 @@ func heldCounts(s stdio.KeptState) func() (index.OldSectionsCount, error) {
return func() (index.OldSectionsCount, error) {
e, err := s.Get(index.OldSectionsKey)
if err != nil {
return index.OldSectionsCount{}, fmt.Errorf("reading %s.%s: %w", index.OldSectionsBucket, index.OldSectionsKey, err)
return index.OldSectionsCount{}, fmt.Errorf("reading %s.%s: %w", index.OldSectionsState, index.OldSectionsKey, err)
}
if e == nil {
return index.OldSectionsCount{}, fmt.Errorf("the indexer has not counted the old sections yet: %s holds no %s", index.OldSectionsBucket, index.OldSectionsKey)
return index.OldSectionsCount{}, fmt.Errorf("the indexer has not counted the old sections yet: %s holds no %s", index.OldSectionsState, index.OldSectionsKey)
}
return index.DecodeOldSections(e.Value)
}
@@ -36,12 +36,12 @@ func TestIndexerAnswersARunsInputOverOneMiBInChunks(t *testing.T) {
sum := sha256.Sum256(body)
digest := hex.EncodeToString(sum[:])
js, _, _, _ := startIndexer(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) {
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.InputsBucket, History: 1})
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(index.Module, index.InputsState), History: 1})
if err != nil {
t.Fatal(err)
}
// The indexer's own seat list (A-186), its state seat-definitions: empty, so the bootstrapped mesh-catalog stands.
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.SeatDefinitionsBucket, History: 5}); err != nil {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(index.Module, index.SeatDefinitionsState), History: 5}); err != nil {
t.Fatal(err)
}
objs, err := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: catalogue.BusName(index.Module, catalogue.StateObjectsName(index.InputsState))})
@@ -82,6 +82,13 @@ accounts {
// named by indexer_pending with the build it waits for. A group's first member is gathered meanwhile. Started again as
// that build, the indexer takes the revision in and leaves the group gathered; the group's second member and its end
// then make one take-in, said take-in.taken under the group's key.
// The beds' indexer's own findings and first views, named from the module its credential names (design 47 G A-296).
var (
findingsBucket = index.Bucket(index.Module, index.FindingsState)
firstViewsBucket = index.Bucket(index.Module, views.StatusState)
)
func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
var running atomic.Pointer[string]
first := "fp-the-indexer-that-runs"
@@ -99,9 +106,9 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
builds.end(t, "novox/mesh-catalog", "c", "modules/c")
}, func(ctx context.Context, js jetstream.JetStream) {
// The states and the stream the controller makes from the indexer's manifest (old sections, ADR 0595 §11).
for bucket, history := range map[string]uint8{index.PendingBucket: 1, index.VerdictsBucket: index.VerdictsHistory,
index.TakenBucket: 1, index.PackagesBucket: 1,
index.SeatDefinitionsBucket: 1, index.RequirementsBucket: 1, findingsBucket: 1, index.UnjudgedBucket: 1} {
for bucket, history := range map[string]uint8{index.Bucket(index.Module, index.PendingState): 1, index.Bucket(index.Module, index.VerdictsState): index.VerdictsHistory,
index.Bucket(index.Module, index.TakenState): 1, index.Bucket(index.Module, index.PackagesState): 1,
index.Bucket(index.Module, index.SeatDefinitionsState): 1, index.Bucket(index.Module, index.RequirementsState): 1, findingsBucket: 1, index.Bucket(index.Module, index.UnjudgedState): 1} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucket, History: history}); err != nil {
t.Fatal(err)
}
@@ -195,7 +202,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
if err != nil || views0.Revision > cur.Revision() || views0.Deleted || len(views0.Value) == 0 || !views0.Created.Equal(bucketCreated(t, ctx, candidates)) {
t.Errorf("views event = %+v, bucket at %d (%v)", views0, cur.Revision(), err)
}
marks, _ := js.KeyValue(ctx, index.MarksBucket)
marks, _ := js.KeyValue(ctx, index.Bucket(index.Module, views.MarksState))
f := &views.Follower{Owner: index.Module, Tool: "indexer_read_candidates", State: "candidates",
Ask: func(context.Context, string) (views.Answer, error) {
return views.FirstView(index.Module, "candidates", jsSource{ctx, candidates}, jsSource{ctx, marks}, "", time.Now())
@@ -310,17 +317,17 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
if !ok || err != nil || own.Fp2 != named || own.Commit != m1.Merge {
t.Fatalf("modules.%s = %+v (%v), want the named build %s at the merge", ix, own, err, named)
}
if _, ok := get(index.ManifestsBucket, own.Manifest); !ok {
if _, ok := get(index.Bucket(index.Module, index.ManifestsState), own.Manifest); !ok {
t.Errorf("manifests.%s is not written", own.Manifest)
}
if _, ok := get(index.VerdictsBucket, own.Manifest); !ok {
if _, ok := get(index.Bucket(index.Module, index.VerdictsState), own.Manifest); !ok {
t.Errorf("the verdict on the indexer's manifest %s is not written", own.Manifest)
}
repoKey, _ := trunk.RepositoryKey(m1.Repository)
if _, ok := get(index.ModulesState, "a"); ok {
t.Errorf("modules.a is written while the revision is pending")
}
if _, ok := get(index.TakenBucket, repoKey); ok {
if _, ok := get(index.Bucket(index.Module, index.TakenState), repoKey); ok {
t.Errorf("taken.%s is written while the revision is pending", repoKey)
}
if e, err := candidates.Get(ctx, "feat/x"); err != nil {
@@ -343,7 +350,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
t.Fatalf("take-in.pending.%s = %+v (said %v) within 20 s, want the group with its first member", groupKey, gathered, ok)
}
// indexer_pending, the tool over the same state, names both waits.
pendingKV, err := js.KeyValue(ctx, index.PendingBucket)
pendingKV, err := js.KeyValue(ctx, index.Bucket(index.Module, index.PendingState))
if err != nil {
t.Fatal(err)
}
@@ -380,7 +387,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
if raw, ok := get(index.ModulesState, "a"); !ok || !strings.Contains(string(raw), m1.Merge) {
t.Errorf("modules.a = %s after the take-in at the start, want it at the merge %s", raw, m1.Merge)
}
if _, ok := get(index.TakenBucket, repoKey); !ok {
if _, ok := get(index.Bucket(index.Module, index.TakenState), repoKey); !ok {
t.Errorf("taken.%s is not written by the take-in at the start", repoKey)
}
if len(w) != 1 || w[0].Key != groupKey || len(w[0].Keys) != 1 || w[0].Keys[0] != g1Key {
@@ -448,7 +455,7 @@ func trunkIndexerEntry(t *testing.T, ctx context.Context, js jetstream.JetStream
t.Fatal(err)
}
digest := meshcatalogclient.ManifestDigest(raw)
manifests, err := js.KeyValue(ctx, index.ManifestsBucket)
manifests, err := js.KeyValue(ctx, index.Bucket(index.Module, index.ManifestsState))
if err != nil {
t.Fatal(err)
}
@@ -492,7 +499,7 @@ func standInBuildSeat(t *testing.T, ctx context.Context, js jetstream.JetStream,
t.Fatal(err)
}
// The indexer's state inputs and its objects, where the validation's request is kept for the build seat's holder (§354).
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.InputsBucket, History: 1}); err != nil {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(index.Module, index.InputsState), History: 1}); err != nil {
t.Fatal(err)
}
if _, err := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: catalogue.BusName(index.Module, catalogue.StateObjectsName(index.InputsState))}); err != nil {
@@ -729,12 +736,12 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before .
t.Fatal(err)
}
}
for _, bucket := range []string{index.PinsBucket, index.Module + "_" + process.ConsumersState} {
for _, bucket := range []string{index.Bucket(index.Module, index.PinsState), index.Module + "_" + process.ConsumersState} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucket, History: 1}); err != nil {
t.Fatal(err)
}
}
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.NamedBucket, History: 1}); err != nil {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(index.Module, index.NamedState), History: 1}); err != nil {
t.Fatal(err)
}
// The build modules' streams for toolchain.reported (declared by them, heard by the indexer) and its durable consumers.
@@ -746,7 +753,7 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before .
t.Fatal(err)
}
}
candidates, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.CandidatesBucket, History: index.CandidatesHistory})
candidates, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(index.Module, index.CandidatesState), History: index.CandidatesHistory})
if err != nil {
t.Fatal(err)
}
@@ -756,7 +763,7 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before .
}
modules := revisionView{js: js}
// The bucket of the manifests, made by the controller from the manifest's state entry manifests (history 1, issue 1449).
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.ManifestsBucket, History: index.ManifestsHistory}); err != nil {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(index.Module, index.ManifestsState), History: index.ManifestsHistory}); err != nil {
t.Fatal(err)
}
// The builder's module log, as the runtime raises it (ADR 0297), filled by put before the indexer starts.
@@ -766,7 +773,7 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before .
}
// The indexer's marks (ADR 0615 §5), and a waiter on manifest-accepted, which the views family never wakes (its events
// are ordinary events on the shared stream, ADR 0635 §5).
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.MarksBucket, History: 1}); err != nil {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(index.Module, views.MarksState), History: 1}); err != nil {
t.Fatal(err)
}
acceptedHeard.Store(0)
@@ -838,14 +845,14 @@ const buildSeatVerbs = `{"name":"current","input":{"type":"object"},"answer":{"t
// revision (revision 1) holding both holders beside others, the entries a test's own fixture holds.
func seedBuildModules(t *testing.T, ctx context.Context, js jetstream.JetStream, others map[string]meshcatalog.CatalogEntry) {
t.Helper()
defs, err := js.KeyValue(ctx, index.SeatDefinitionsBucket)
defs, err := js.KeyValue(ctx, index.Bucket(index.Module, index.SeatDefinitionsState))
if errors.Is(err, jetstream.ErrBucketNotFound) {
defs, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.SeatDefinitionsBucket, History: 1})
defs, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(index.Module, index.SeatDefinitionsState), History: 1})
}
if err != nil {
t.Fatal(err)
}
manifests, err := js.KeyValue(ctx, index.ManifestsBucket)
manifests, err := js.KeyValue(ctx, index.Bucket(index.Module, index.ManifestsState))
if err != nil {
t.Fatal(err)
}
@@ -879,7 +886,7 @@ func seedBuildModules(t *testing.T, ctx context.Context, js jetstream.JetStream,
t.Fatal(err)
}
// A holder the mesh took in has its accepted verdict: the indexer judges and says nothing of it at its start.
if verdicts, err := js.KeyValue(ctx, index.VerdictsBucket); err == nil {
if verdicts, err := js.KeyValue(ctx, index.Bucket(index.Module, index.VerdictsState)); err == nil {
vkey, err := meshcatalogclient.VerdictsKey(digest)
if err != nil {
t.Fatal(err)
@@ -912,7 +919,7 @@ func seedBuildModules(t *testing.T, ctx context.Context, js jetstream.JetStream,
// §699): the indexer consumes it in the seat's form, and its own manifest is judged against the seat list it holds.
func seedAssignerSeat(t *testing.T, ctx context.Context, js jetstream.JetStream) {
t.Helper()
defs, err := js.KeyValue(ctx, index.SeatDefinitionsBucket)
defs, err := js.KeyValue(ctx, index.Bucket(index.Module, index.SeatDefinitionsState))
if err != nil {
t.Fatal(err)
}
@@ -962,7 +969,7 @@ func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9; I1, I3: the m
builds.end(t, "novox/mesh-controller", "build-agent", "modules/build-agent")
}, func(ctx context.Context, js jetstream.JetStream) {
var err error
if asks, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.TakeInsBucket, History: 1}); err != nil {
if asks, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(index.Module, index.TakeInsState), History: 1}); err != nil {
t.Fatal(err)
}
})
@@ -1069,7 +1076,7 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4, 15
}, func(ctx context.Context, js jetstream.JetStream) {
// The indexer's state unjudged, as the controller makes it from its manifest; the verdicts state is made only once
// the merge is taken in (issue 1966 note 154: a take-in at the start, before the verdicts state's bind).
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.UnjudgedBucket, History: 1}); err != nil {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(index.Module, index.UnjudgedState), History: 1}); err != nil {
t.Fatal(err)
}
})
@@ -1101,7 +1108,7 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4, 15
m.Inputs2.Locks["tools"] != goSum || m.Path != "modules/avahi" { // 1606/AC2
t.Errorf("module = %+v within 20 s, want its path modules/avahi, its manifest digest %s, its inputs and none2 'toolchain not yet reported'", m, digest)
}
kv, err := js.KeyValue(ctx, index.ManifestsBucket)
kv, err := js.KeyValue(ctx, index.Bucket(index.Module, index.ManifestsState))
if err != nil {
t.Fatal(err)
}
@@ -1116,7 +1123,7 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4, 15
// 1966 (ADR 0668 §4, note 154): the take-in wrote its four manifests with no verdicts state on the bus, so it keeps
// them in its state unjudged; when the verdicts state binds, each is judged as new (bad refused in full, never as a
// warning), every accepted one is said as manifest-accepted, and the state is left empty.
unjudged, err := js.KeyValue(ctx, index.UnjudgedBucket)
unjudged, err := js.KeyValue(ctx, index.Bucket(index.Module, index.UnjudgedState))
if err != nil {
t.Fatal(err)
}
@@ -1126,7 +1133,7 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4, 15
t.Errorf("the take-in's new manifest %s is not kept unjudged: the state holds %v", meshcatalogclient.ManifestDigest([]byte(raw)), kept)
}
}
verdicts, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.VerdictsBucket, History: index.VerdictsHistory})
verdicts, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(index.Module, index.VerdictsState), History: index.VerdictsHistory})
if err != nil {
t.Fatal(err)
}
@@ -1281,10 +1288,10 @@ func TestIndexerBuildsARunsCandidateAndDropsItWhenEveryMembersPipelineEnded(t *t
}
}
var err error
if runs, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.RunsBucket, History: 1}); err != nil {
if runs, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(index.Module, index.RunsState), History: 1}); err != nil {
t.Fatal(err)
}
regs, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.RegistrationsBucket, History: 1})
regs, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(index.Module, index.RegistrationsState), History: 1})
if err != nil {
t.Fatal(err)
}
@@ -1371,7 +1378,7 @@ func TestIndexerJudgesAnUnreadableVerdictAgainAtStart(t *testing.T) { // 1923/AC
if err != nil {
t.Fatal(err)
}
manifests, err := js.KeyValue(ctx, index.ManifestsBucket)
manifests, err := js.KeyValue(ctx, index.Bucket(index.Module, index.ManifestsState))
if err != nil {
t.Fatal(err)
}
@@ -1622,7 +1629,7 @@ func TestIndexerKeepsWhatAWantAndARunningBuildNameAndLetsAGoneOneLeave(t *testin
}
}
})
named, err := js.KeyValue(ctx, index.NamedBucket)
named, err := js.KeyValue(ctx, index.Bucket(index.Module, index.NamedState))
if err != nil {
t.Fatal(err)
}
@@ -1663,7 +1670,7 @@ func TestIndexerAppliesThePinsEventsThroughItsDurableConsumer(t *testing.T) {
}
say(meshdelivery.SubjectPinned, meshdelivery.Pins{Record: "r1", Milestone: "47/G", Catalog: []meshdelivery.PinsCatalogItem{{Branch: "main", Revision: 9}}})
say(meshdelivery.SubjectPinsStated, meshdelivery.PinsStatedEvent{At: time.Now().UTC().Add(time.Minute)})
pins, err := js.KeyValue(ctx, index.PinsBucket)
pins, err := js.KeyValue(ctx, index.Bucket(index.Module, index.PinsState))
if err != nil {
t.Fatal(err)
}
@@ -349,7 +349,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return err
}
defer conn.Close()
candidates, err := process.EssentialBucket(ctx, js, "indexer", index.CandidatesBucket, "write a candidate's fingerprints")
candidates, err := process.EssentialBucket(ctx, js, "indexer", index.Bucket(cred.Module, index.CandidatesState), "write a candidate's fingerprints")
if err != nil {
return err
}
@@ -369,16 +369,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
default:
logger.Printf("the view of the trunk read from trunk revision %d", at)
}
manifests, err := process.EssentialBucket(ctx, js, "indexer", index.ManifestsBucket, "keep a module's manifest")
manifests, err := process.EssentialBucket(ctx, js, "indexer", index.Bucket(cred.Module, index.ManifestsState), "keep a module's manifest")
if err != nil {
return err
}
// Its taken state (ADR 0556 clause 1) is bound when it appears, as every state a delivery adds (issue 1790): until then
// a merge's taken key is said and not written, and the rest of the indexer runs.
taken := process.AwaitBucket(ctx, js, conn, index.TakenBucket, "no repository's taken key is written", logger.Printf)
taken := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.TakenState), "no repository's taken key is written", logger.Printf)
// The control records (ADR 0565 §2), new with this delivery and bound when they appear: until then nothing is written to them.
predefined := process.AwaitBucket(ctx, js, conn, index.PredefinedBucket, "no predefined list is written", logger.Printf)
configCandidates := process.AwaitBucket(ctx, js, conn, index.ConfigCandidatesBucket, "no pull request's config declarations are written", logger.Printf)
predefined := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.PredefinedState), "no predefined list is written", logger.Printf)
configCandidates := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.ConfigCandidatesState), "no pull request's config declarations are written", logger.Printf)
// The builder's log (ADR 0571 clause 1), read on the indexer's own credential (the calls connection) through its declared read: every entry it
// keeps at the start, then each new one as the builder appends it (no timer). It names the directory of a module whose key
// names none (fillPaths) and the places a registered module was built from; no build moves a fingerprint (ADR 0530 §1).
@@ -393,27 +393,27 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
defer calls.Close()
// The verdicts on manifests (ADR 0578 §1): a state this delivery brings, bound when it appears.
verdicts := process.AwaitBucket(ctx, js, conn, index.VerdictsBucket, "no verdict on a manifest is written", logger.Printf)
verdicts := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.VerdictsState), "no verdict on a manifest is written", logger.Printf)
// The published packages (ADR 0593 §2): a state this delivery brings, bound when it appears; until then every
// fingerprint takes the version line.
packages := process.AwaitBucket(ctx, js, conn, index.PackagesBucket, "no package is published, so every fingerprint takes the version line", logger.Printf)
packages := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.PackagesState), "no package is published, so every fingerprint takes the version line", logger.Printf)
// How many modules use each old section (issue 1966 D4): a state this delivery brings, bound when it appears; until then
// the judge refuses no use of an old section.
oldSections := process.AwaitBucket(ctx, js, conn, index.OldSectionsBucket, "the count of the old sections is not kept, so the judge refuses no use of one", logger.Printf)
oldSections := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.OldSectionsState), "the count of the old sections is not kept, so the judge refuses no use of one", logger.Printf)
// What each consumer's go.sum requires of each mesh Go module (DLV-I7): a state this delivery brings, bound when it
// appears; until then indexer_requirements answers none, and the builder's floor waits on its answer.
requirements := process.AwaitBucket(ctx, js, conn, index.RequirementsBucket, "what consumers require is not kept", logger.Printf)
requirements := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.RequirementsState), "what consumers require is not kept", logger.Printf)
// The module graph (ADR 0603 §2) is held in memory, read with the view: each revision's entries carry its nodes.
// The state heard (A-80): the forge messages handled (key forge) and what the indexer heard and compares later (the time a call is
// unanswered since, the newest toolchain-agreed and version-delivered, the artifacts published so far of a version).
heardStore := awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.HeardBucket, "what the indexer heard is not kept", logger.Printf)}
ownBuildState := process.AwaitBucket(ctx, js, conn, index.OwnBuildBucket, "which build this indexer runs is not kept", logger.Printf)
heardStore := awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.HeardState), "what the indexer heard is not kept", logger.Printf)}
ownBuildState := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.OwnBuildStateName), "which build this indexer runs is not kept", logger.Printf)
// The manifests a take-in wrote and no judge pass judged yet (issue 1966): a state this delivery brings, bound when it
// appears; until then a take-in's new manifests are kept for their judging in its memory alone.
unjudged := process.AwaitBucket(ctx, js, conn, index.UnjudgedBucket, "a take-in's new manifests are kept for their judging in its memory alone", logger.Printf)
unjudged := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.UnjudgedState), "a take-in's new manifests are kept for their judging in its memory alone", logger.Printf)
// The runs on request the indexer holds a candidate for (A-174): a state this delivery brings, bound when it appears.
runs := process.AwaitBucket(ctx, js, conn, index.RunsBucket, "no run on request's candidate is written or dropped", logger.Printf)
findings := process.AwaitBucket(ctx, js, conn, findingsBucket, "no finding of the indexer is written", logger.Printf)
runs := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.RunsState), "no run on request's candidate is written or dropped", logger.Printf)
findings := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.FindingsState), "no finding of the indexer is written", logger.Printf)
// The indexer's findings are the seat mesh-conditions' generated finding, written through mesh-sdk's held finding writer
// (design 47 G §689, A-29): it holds what is written before the state is bound, and starts once the state is.
writer, err := views.NewHeldWriter(views.WriterConfig[mcseat.Finding, mcseat.FindingCleared, mcseat.FindingsStated, mcseat.FindingsAskedEvent]{
@@ -430,8 +430,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
})
// The seat definitions (design 26, DLV-I17): a state this delivery brings, bound when it appears; until then none is kept.
seatDefs := process.AwaitBucket(ctx, js, conn, index.SeatDefinitionsBucket, "no seat definition is kept", logger.Printf)
marks := process.AwaitBucket(ctx, js, conn, index.MarksBucket, "no change is said on the views family, so every reader waits on its first view", logger.Printf)
seatDefs := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.SeatDefinitionsState), "no seat definition is kept", logger.Printf)
marks := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, views.MarksState), "no change is said on the views family, so every reader waits on its first view", logger.Printf)
seatBus := &seatBus{defs: awaitedSource{ctx, seatDefs}, marks: awaitedSource{ctx, marks}}
// The build modules are found again at every change of the seat list (buildmodules.go), once the indexer is made.
var buildFollow atomic.Pointer[buildModulesFollow]
@@ -443,13 +443,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
readWhenBound(seatReader, seatDefs, marks)
var fillFollow atomic.Pointer[versionsFill]
// Kept while named (design 47 G §675): the want and the running build that name a fingerprint, heard below.
named := index.NewNamed(awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.NamedBucket, "no entry is kept for a want or a running build that names it", logger.Printf)}, logger.Printf)
pinsBucket := process.AwaitBucket(ctx, js, conn, index.PinsBucket, "no record's pin on a catalog graph revision is kept, so no revision is removed", logger.Printf)
named := index.NewNamed(awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.NamedState), "no entry is kept for a want or a running build that names it", logger.Printf)}, logger.Printf)
pinsBucket := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.PinsState), "no record's pin on a catalog graph revision is kept, so no revision is removed", logger.Printf)
pinsStore := awaitedStore{b: pinsBucket}
consumers := process.AwaitBucket(ctx, js, conn, index.Module+"_"+process.ConsumersState, "indexer_consumer reads no consumer", logger.Printf)
consumers := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, process.ConsumersState), "indexer_consumer reads no consumer", logger.Printf)
// The state inputs and the object store beside it; the seat mesh-catalog's verb input is answered here in chunks of the core
// connection, of any size (design 47 G §712, A-237).
inputsStore := awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.InputsBucket, "no input of a run ask is kept, so the verb input answers none", logger.Printf)}
inputsStore := awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.InputsState), "no input of a run ask is kept, so the verb input answers none", logger.Printf)}
inObjects := &inputObjects{js: js, module: cred.Module}
stopInput, err := calls.AnswerSeat(meshcatalog.Seat, seatlist.Definition(seatReader.Reader, meshcatalog.Seat), map[string]process.SeatHandler{
meshcatalog.VerbInput: index.InputVerb(inputsStore, inObjects, conn, buscalls.SeatAckWithin, logger.Printf)})
@@ -461,23 +461,23 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
Verdicts: awaitedStore{b: verdicts}, SayRefused: sayRefused(conn, cred, logger.Printf), SayAccepted: sayAccepted(conn, cred),
Predefined: awaitedStore{b: predefined}, ConfigCandidates: awaitedStore{b: configCandidates}, Logf: logger.Printf,
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call),
ReadOpen: openVia(calls.Call), Packages: awaitedStore{b: packages, js: js, name: index.PackagesBucket}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call),
ReadOpen: openVia(calls.Call), Packages: awaitedStore{b: packages, js: js, name: index.Bucket(cred.Module, index.PackagesState)}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call),
OwnBuild: ownBuildVia(calls.Call, cred.Node), OwnBuildState: awaitedStore{b: ownBuildState}, Heard: heardStore, Placement: placementVia(calls.CallKept), RunningWhole: runningWholeVia(js), Runs: awaitedStore{b: runs}, Graph: graph,
SeatDefinitions: awaitedStore{b: seatDefs}, ReadFile: readFileVia(calls.Call), SaySeatDefinition: saySeats,
Revisions: revisions, EntryIndex: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.EntryIndexBucket, "no entry is found by its fingerprint", logger.Printf)}, // Message a kept call: a read, called again through a transport fault (ADR 0668 §1)
Revisions: revisions, EntryIndex: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.EntryIndexState), "no entry is found by its fingerprint", logger.Printf)}, // Message a kept call: a read, called again through a transport fault (ADR 0668 §1)
// Each catalog revision may bring another holder of a build seat: the build modules are found again (buildmodules.go).
SayCatalogRevision: afterRevision(refreshingBuildModules(sayCatalogRevision(conn, cred), &buildFollow), &fillFollow),
SayFill: sayFill(conn, cred),
// The catalog's released versions (design 47 G §349): the state versions, bound when it appears, and the event version-released.
Versions: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.VersionsBucket, "no released version is kept, so no package-published makes a version", logger.Printf)},
BuildEdges: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.BuildEdgesBucket, "no build of another watched branch keeps its edges", logger.Printf)},
Versions: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.VersionsState), "no released version is kept, so no package-published makes a version", logger.Printf)},
BuildEdges: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.BuildEdgesState), "no build of another watched branch keeps its edges", logger.Printf)},
Pins: pinsStore, PinsPending: pendingOf(js, cred),
Inputs: inputsStore,
InputObjects: inObjects,
SayVersionReleased: sayVersionReleased(conn, cred), SayVersionsFilled: sayVersionsFilled(conn, cred),
// The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear;
// until then every merge is taken in at once, as before.
Pending: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.PendingBucket, "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)},
Pending: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.PendingState), "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)},
SayTakeIn: sayTakeIn(js, cred),
// The validation's run ask (design 47 G §713, A-247, A-248), put under the indexer's own asker.
ValidationAsks: buildrun.Asker{Name: index.ValidationAsker, Publisher: buildrun.JetStreamAsks{JS: js}},
@@ -486,8 +486,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// The registrations and the changes that wait for their warrant (design 47 G §2, §667): their states are bound when they
// appear; a warrant is heard on the router's decided event for this module, below.
registry := newRegistry(js, conn, cred, calls.Call,
awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.RegistrationsBucket, "no registration is kept or written", logger.Printf)},
awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.ProposalsBucket, "no change of a registration waits for its warrant", logger.Printf)},
awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.RegistrationsState), "no registration is kept or written", logger.Printf)},
awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.ProposalsState), "no change of a registration waits for its warrant", logger.Printf)},
routerRecordOf(cred.Module, calls.Call, calls.Ask), ix, logger.Printf)
ix.Message = messageVia(calls.CallKeptWhileOpen, ix.TakeInPending, logger.Printf) // a kept call, called again through a transport fault while the commit's take-in is pending (ADR 0668 §1, A-190)
registry.Pruned = ix.PruneBranchBuilds
@@ -560,7 +560,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
hub := process.NewEventHub(logger.Printf)
reader := &views.ProcessReader{Module: index.Module, Call: calls.Call, Logf: logger.Printf, Hub: hub,
Findings: func() views.WireWriter { return writer }}
followAssignments(ctx, reader, calls.Call, process.AwaitBucket(ctx, js, conn, firstViewsBucket, "indexer_first_views reads nothing and indexer_first_view_again is not heard", logger.Printf), cred, js, gate, func(ctx context.Context) {
followAssignments(ctx, reader, calls.Call, process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, views.StatusState), "indexer_first_views reads nothing and indexer_first_view_again is not heard", logger.Printf), cred, js, gate, func(ctx context.Context) {
if err := ix.Recompute(ctx); err != nil {
logger.Printf("writing the modules again after the assignments of a build module changed: %v", err)
}
@@ -615,10 +615,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// as every other input. A module the forge could not read is also read again by itself on the next source revision and
// merged message (index.RetryUnread).
// A state not on the bus yet at this start is watched once it appears (issue 1790).
process.AwaitBucket(ctx, js, conn, index.TakeInsBucket, "indexer_take_in is not answered; a module the forge could not read is still read again by itself", logger.Printf).Ready(func(kv jetstream.KeyValue) {
process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.TakeInsState), "indexer_take_in is not answered; a module the forge could not read is still read again by itself", logger.Printf).Ready(func(kv jetstream.KeyValue) {
eachTakeIn := func(e jetstream.KeyValueEntry) { takenIn(ctx, ix, kv, e, logger.Printf) }
if tw, err := settle(ctx, kv, eachTakeIn); err != nil {
process.NotWatched(logger.Printf, index.TakeInsBucket, err, "indexer_take_in is not answered until the indexer starts again")
process.NotWatched(logger.Printf, index.Bucket(cred.Module, index.TakeInsState), err, "indexer_take_in is not answered until the indexer starts again")
} else {
go func() { errs <- follow(ctx, tw, eachTakeIn) }()
}
@@ -633,7 +633,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// Through the seat git (milestone 47/G W4.15, design 47 *A part that cannot read the forge keeps what it has*): a seat that
// does not answer is waited for at most index.GitWaitBound before the judging below runs on the kept definitions, said in
// indexer_first_views and as view-waits past five minutes, and read again on the next forge message heard.
defs := gitWaits(ctx, ix, writer, cred.Node, process.AwaitBucket(ctx, js, conn, firstViewsBucket, "indexer_first_views does not say the wait for the seat git", logger.Printf), logger.Printf)
defs := gitWaits(ctx, ix, writer, cred.Node, process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, views.StatusState), "indexer_first_views does not say the wait for the seat git", logger.Printf), logger.Printf)
seatDefs.Ready(func(jetstream.KeyValue) { defs.Start(ctx) })
// Every manifest judged at the start, so a judge that changed with this program judges every one again (ADR 0578
// §1), once the verdicts state is there; each merge judges them again (index.ApplyMessage).
@@ -65,10 +65,10 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T)
var packages, seatDefs, findings, takeIns jetstream.KeyValue
js, ctx, _, modules := startIndexer(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) {
var err error
if packages, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.PackagesBucket, History: index.PackagesHistory}); err != nil {
if packages, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(index.Module, index.PackagesState), History: index.PackagesHistory}); err != nil {
t.Fatal(err)
}
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.TakenBucket, History: 1}); err != nil {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(index.Module, index.TakenState), History: 1}); err != nil {
t.Fatal(err)
}
// Both languages' toolchains, so mesh-sdk (a TypeScript package) gets a fingerprint that moves with its source.
@@ -85,7 +85,7 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T)
}
serveForge(t, js.Conn(), listing, goSumContent)
// The seat definitions' state (DLV-I17), as the grants part makes it from the manifest.
if seatDefs, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.SeatDefinitionsBucket, History: 5}); err != nil {
if seatDefs, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(index.Module, index.SeatDefinitionsState), History: 5}); err != nil {
t.Fatal(err)
}
// DLV-I19, I20: the indexer's findings and takeins states, and the checker's stream asks with the indexer's durable
@@ -93,7 +93,7 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T)
if findings, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: findingsBucket, History: 1}); err != nil {
t.Fatal(err)
}
if takeIns, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.TakeInsBucket, History: 1}); err != nil {
if takeIns, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(index.Module, index.TakeInsState), History: 1}); err != nil {
t.Fatal(err)
}
seedBuildModules(t, ctx, js, nil)
@@ -26,10 +26,6 @@ import (
"mesh-delivery-indexer/internal/index"
)
// findingsBucket is the indexer's own findings, which the condition keeper raises and clears (ADR 0564 §2): a value too
// large for its views event, and its first view of the assigner's assignments unanswered past its bound.
var findingsBucket = index.FindingsBucket
// readyOf binds a state's bucket: now for one bound at the start, once it appears for one awaited.
type readyOf func(func(jetstream.KeyValue))
@@ -108,10 +104,6 @@ func clearHeld(ctx context.Context, w index.FindingWriter, scope, id, kind, why
// graphState is the module graph's state, named apart from run's bucket of the same name.
const graphState = graph.State
// firstViewsBucket is the indexer's own state of its first views (views.StatusState): what indexer_first_views reads and
// where indexer_first_view_again asks.
var firstViewsBucket = index.Module + "_" + views.StatusState
// followAssignments reads the seat mesh-module-assigner's assignments through its holder's first view and then its views family,
// never its bucket (ADR 0615 §1, DLV-F1r), into gate: the holder is the module the seat list names for the seat, asked through
// mesh-sdk's seatlist.HolderOf (design 47 G A-119, A-126, A-131), so no module is named and a new holder is followed with no change
@@ -19,8 +19,7 @@ import (
// edge entries in the state build-edges, under <module>.<branch>, and what the branch's newest build of the module
// published as heard, branch-build.<module>.<branch> in the state heard (§374, A-83).
const (
BuildEdgesState = "build-edges"
BuildEdgesBucket = Module + "_" + BuildEdgesState
BuildEdgesState = "build-edges"
// BranchBuildPrefix opens branch-build.<module>.<branch>, each part escaped as a name (A-80).
BranchBuildPrefix = "branch-build."
)
@@ -88,7 +88,7 @@ type Indexer struct {
// Findings is the indexer's finding writer (design 47 G A-29), where a manifest-accepted whose publish failed twice is kept
// until said (accepted.go, DLV-I9b); nil writes none.
Findings FindingWriter
// Packages is the packages state (PackagesBucket, packages.go; novox/hq ADR 0593 §2); nil publishes none,
// Packages is the packages state (PackagesState, packages.go; novox/hq ADR 0593 §2); nil publishes none,
// and every fingerprint takes the version line.
Packages Store
// SeatDefinitions is the state seat-definitions (seatdefs.go; design 26, DLV-I17); nil keeps none.
@@ -22,19 +22,13 @@ import (
// against the trunk's manifests the indexer holds. mesh-config reads both in place of the controller's mirror and its
// `pull-declares`; a key's write is its event, nothing polls.
// The two states' buckets.
var (
PredefinedBucket = Module + "_" + PredefinedState
ConfigCandidatesBucket = Module + "_" + ConfigCandidatesState
)
// writePredefined keeps the predefined list a merged message carries, unless the key names a later merge: a merge older
// than the one it names never replaces it (ADR 0348 §7). The same list at the same merge writes nothing (ADR 0289 §1).
func (ix *Indexer) writePredefined(ctx context.Context, f forgemsg.Forge) error {
if ix.Predefined == nil || len(f.Predefined) == 0 {
return nil
}
return ix.notYet(PredefinedBucket, write(ctx, ix.Predefined, PredefinedListKey, func(value []byte, found bool) ([]byte, bool, error) {
return ix.notYet(PredefinedState, write(ctx, ix.Predefined, PredefinedListKey, func(value []byte, found bool) ([]byte, bool, error) {
if found {
var cur struct { // the list is compared as it was kept, byte for byte
Commit string `json:"commit"`
@@ -61,9 +55,9 @@ func (ix *Indexer) writePredefined(ctx context.Context, f forgemsg.Forge) error
// notYet says a state not on the bus yet (it comes with this delivery, bound when it appears) and lets the message go:
// what it would have written is written at the next message that carries it.
func (ix *Indexer) notYet(bucket string, err error) error {
func (ix *Indexer) notYet(state string, err error) error {
if errors.Is(err, process.ErrNotYet) {
ix.logf("the state %s is not on the bus yet: nothing is written to it for this message", bucket)
ix.logf("the state %s is not on the bus yet: nothing is written to it for this message", state)
return nil
}
return err
@@ -170,7 +164,7 @@ func (ix *Indexer) writeConfigCandidate(ctx context.Context, repository string,
if err != nil {
return err
}
return ix.notYet(ConfigCandidatesBucket, write(ctx, ix.ConfigCandidates, key, func(value []byte, found bool) ([]byte, bool, error) {
return ix.notYet(ConfigCandidatesState, write(ctx, ix.ConfigCandidates, key, func(value []byte, found bool) ([]byte, bool, error) {
if found {
var cur meshcatalog.ConfigCandidatesByPullValue
if json.Unmarshal(value, &cur) == nil && cur.Head == head && reflect.DeepEqual(cur.Declares, declares) {
@@ -191,7 +185,7 @@ func (ix *Indexer) dropConfigCandidate(ctx context.Context, repository string, n
if err != nil {
return nil
}
return ix.notYet(ConfigCandidatesBucket, write(ctx, ix.ConfigCandidates, key, func(_ []byte, found bool) ([]byte, bool, error) { return nil, found, nil }))
return ix.notYet(ConfigCandidatesState, write(ctx, ix.ConfigCandidates, key, func(_ []byte, found bool) ([]byte, bool, error) { return nil, found, nil }))
}
// configCandidateOfPull is a pull message's part in config-candidates: a head whose manifests the message carries is written,
@@ -19,8 +19,6 @@ import (
const (
// EntryIndexState is the old state section of the entries' digests by fingerprint.
EntryIndexState = "entry-index"
// EntryIndexBucket is the bucket of the state.
EntryIndexBucket = Module + "_" + EntryIndexState
// EntryIndexTool is the tool that reads a fingerprint's key and its object's digest (class read).
EntryIndexTool = "indexer_entries"
)
@@ -7,10 +7,9 @@ import (
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
)
// FindingsState is the indexer's state of its findings, and FindingsBucket its bucket (design 47 G §383, §687).
// FindingsState is the indexer's state of its findings (design 47 G §383, §687).
const (
FindingsState = "findings"
FindingsBucket = Module + "_" + FindingsState
FindingsState = "findings"
)
// FindingWriter is the indexer's finding writer, mesh-sdk's held writer of the seat mesh-conditions' generated finding (design
@@ -12,9 +12,8 @@ import (
// server delivered it, when it was published and handled, what it names and what the indexer did with it. The state keeps
// the last 64 values (its history); mesh-delivery-view's indexer_heard reads them, newest first, with their times.
const (
HeardState = "heard"
HeardBucket = Module + "_" + HeardState
HeardKey = "forge"
HeardState = "heard"
HeardKey = "forge"
// What the indexer did with a message.
HeardApplied = "applied"
HeardLeft = "left for redelivery"
@@ -24,8 +24,6 @@ import (
const (
// NamedState is the old state section of the sets of members by fingerprint.
NamedState = "named"
// NamedBucket is the bucket of the state.
NamedBucket = Module + "_" + NamedState
// NamedDropTool is the tool that removes one member from one fingerprint's set by hand (class act).
NamedDropTool = "indexer_named_drop"
// NamedWanted and NamedRunning name the two streams the sets are read from, for Named.ReadDone.
@@ -9,21 +9,17 @@ const (
// The indexer's own states, with the revisions each keeps per key (module.json `state`).
CandidatesState = "candidates"
CandidatesBucket = Module + "_" + CandidatesState
CandidatesHistory = 10
ModulesState = "modules"
ManifestsState = "manifests"
ManifestsBucket = Module + "_" + ManifestsState
ManifestsHistory = 1
// VerdictsState and VerdictsBucket are in verdicts.go.
// VerdictsState is in verdicts.go.
VerdictsHistory = 5
PredefinedState = "predefined"
ConfigCandidatesState = "config-candidates"
PackagesState = "packages"
PackagesBucket = Module + "_" + PackagesState
PackagesHistory = 5
TakenState = "taken"
TakenBucket = Module + "_" + TakenState
PredefinedListKey = "list"
ForgeBridgeModule = "mesh-delivery-forge-bridge"
GateModule = "mesh-delivery-gate"
@@ -35,3 +31,7 @@ const (
// ToolchainsBucket is the toolchains state of a build module (reads: mesh-build-go.toolchains, mesh-build-node.toolchains).
func ToolchainsBucket(module string) string { return module + "_" + toolchainsState }
// Bucket is the bus's bucket of the indexer's own state, `<module>_<state>`, module its credential's, never a constant
// (design 47 G A-296).
func Bucket(module, state string) string { return module + "_" + state }
@@ -16,8 +16,7 @@ const (
// OldSectionsTool is the tool that reads the count.
OldSectionsTool = "indexer_old_sections"
// OldSectionsState is the indexer's state of the count: the bucket mesh-delivery-indexer_oldsections.
OldSectionsState = "oldsections"
OldSectionsBucket = Module + "_" + OldSectionsState
OldSectionsState = "oldsections"
// OldSectionsKey is the one key it holds.
OldSectionsKey = "counts"
)
@@ -22,7 +22,7 @@ import (
// The published packages (novox/hq ADR 0593 §2): the indexer alone reads, from the forge's listing, the content digest
// of each package of every mesh Go module a held module reaches, and writes the module's latest published version and
// digests in its state `packages` (PackagesBucket), one key per Go module. The builder, the checker and the
// digests in its state `packages`, one key per Go module. The builder, the checker and the
// build modules read it there; nobody else computes a digest or a version.
//
// A Go module is reached when a held module's go.sum names it (KeptInputs2.Packages) or holds an import a build
@@ -33,8 +33,7 @@ const (
// PendingState is the indexer's state of held merges: the bucket mesh-delivery-indexer_pending, keyed
// takein.PendingKey, so the key order is the merge order. An old `state` entry (the indexer is foundation, ADR
// 0595 §11), history 1, reached by the indexer alone.
PendingState = "pending"
PendingBucket = Module + "_" + PendingState
PendingState = "pending"
// PendingTool reads it (ADR 0633 §5); TakeInPendingTool and DropPendingTool steer it.
PendingTool = "indexer_pending"
TakeInPendingTool = "indexer_take_in_pending"
@@ -471,7 +470,7 @@ func (ix *Indexer) Arrive(ctx context.Context, seq uint64, f forgemsg.Forge) (Ap
switch err := ix.Pending.Create(ctx, key, data); {
case errors.Is(err, process.ErrNotYet):
// The state this delivery brings is not on the bus yet (issue 1790): the merge is taken in at once, as before.
ix.logf("take-in: the state %s is not on the bus yet, so %s#%d is taken in at once, as before ADR 0633", PendingBucket, f.Repository, f.Number)
ix.logf("take-in: the state %s is not on the bus yet, so %s#%d is taken in at once, as before ADR 0633", PendingState, f.Repository, f.Number)
a, err := ix.ApplyMessage(ctx, f)
return a, "", err
case err != nil && !errors.Is(err, ErrConflict):
@@ -1083,7 +1082,7 @@ func (ix *Indexer) takeInWholeBy(ctx context.Context, entries []PendingEntry, ow
// Every manifest a take-in writes is judged in it and its verdict said before take-in.taken (issue 1966 note 154,
// ADR 0668 §4): with the verdicts state not on the bus yet, nothing is taken in; its bind evaluates pending again.
ix.logf("take-in: %d merges wait, oldest %s: the state %s is not on the bus yet, so nothing is taken in until it is, and no manifest is written without its verdict",
len(entries), entries[0].Key, VerdictsBucket)
len(entries), entries[0].Key, VerdictsState)
return Applied{}, errVerdictsNotYet
}
claimed, ok := ix.claim(ctx, entries, own, by)
@@ -25,9 +25,8 @@ import (
// keys, `made` (written once, create-only, when the state is first used) and `pins-stated` (the `at` of the newest applied event of
// that name). Pins count only once a pins-stated later than made is applied.
const (
// PinsState and PinsBucket are the state's name and its bucket; PinsCapMiB is its cap in the manifest (about 1,800 keys of about 0.2 KiB).
// PinsState is the state's name; PinsCapMiB is its cap in the manifest (about 1,800 keys of about 0.2 KiB).
PinsState = "pins"
PinsBucket = Module + "_" + PinsState
PinsHistory = 1
PinsCapMiB = 4
// PinsMadeKey and PinsStatedKey are the two keys of the state that are no revision's.
@@ -26,12 +26,8 @@ import (
const (
// RegistrationsState is the old state section of the registrations, one key per module (§2).
RegistrationsState = "registrations"
// RegistrationsBucket is the bucket of the state.
RegistrationsBucket = Module + "_" + RegistrationsState
// ProposalsState is the old state section of the changes that wait for their warrant (§667), capped at 4 MiB.
ProposalsState = "proposals"
// ProposalsBucket is the bucket of the state.
ProposalsBucket = Module + "_" + ProposalsState
// RegistrationHistoryStream is the old streams section that keeps every version of every registration (§2, §9).
RegistrationHistoryStream = "registration-history"
// RegistrationHistoryEvent is the event of the stream, one subject per module.
@@ -18,9 +18,8 @@ import (
// The builder's floor asks it through indexer_requirements, so a version is never numbered below what a consumer
// already requires.
const (
RequirementsState = "requirements"
RequirementsBucket = Module + "_" + RequirementsState
RequirementsTool = "indexer_requirements"
RequirementsState = "requirements"
RequirementsTool = "indexer_requirements"
)
// Requirement is the value of one package's key in the requirements state: By maps each consumer, "<repository>:<dir>",
@@ -27,9 +27,8 @@ import (
// callers are the holders of the build seats) and by the tool indexer_inputs. A key older than a day is deleted at each new
// ask, judged over at most the 20 oldest keys.
const (
InputsState = "inputs"
InputsBucket = Module + "_" + InputsState
InputsTool = "indexer_inputs"
InputsState = "inputs"
InputsTool = "indexer_inputs"
// InputsKeptFor is how long an input is kept at the longest.
InputsKeptFor = 24 * time.Hour
// InputsSwept is how many of the oldest keys each new ask judges.
@@ -32,8 +32,7 @@ import (
// RunsState is the indexer's state of the runs on request it holds a candidate for (A-174): the bucket
// mesh-delivery-indexer_runs, one key per run, run-<id>.
const (
RunsState = "runs"
RunsBucket = Module + "_" + RunsState
RunsState = "runs"
)
// Run is one value of the state runs: the run's members as run-started named them, and the modules whose pipeline of the
@@ -41,8 +41,7 @@ import (
const (
// SeatDefinitionsState is the indexer's state of the seat definitions (design 26), one key per seat and source.
SeatDefinitionsState = "seat-definitions"
SeatDefinitionsBucket = Module + "_" + SeatDefinitionsState
SeatDefinitionsState = "seat-definitions"
// The seat mesh-catalog's verb and events (ADR 0636 §4): the indexer, which holds the seat, serves the verb and says
// the events on the seat's own subject, for the trunks' definitions only, changed when a digest moved and removed when
// a trunk's definition went.
@@ -20,8 +20,7 @@ import (
const (
TakeInTool = "indexer_take_in"
// TakeInsState is the indexer's state of take-in asks: the bucket mesh-delivery-indexer_takeins, keyed as the modules state.
TakeInsState = "takeins"
TakeInsBucket = Module + "_" + TakeInsState
TakeInsState = "takeins"
)
// TakeInAsk is a value of the takeins state: which module a person asked to be taken in, why, who and when, and, once the
@@ -21,8 +21,7 @@ import (
const (
// UnjudgedState is the indexer's state of the digests a take-in wrote and no judge pass judged yet: the bucket
// mesh-delivery-indexer_unjudged, one key per manifest digest.
UnjudgedState = "unjudged"
UnjudgedBucket = Module + "_" + UnjudgedState
UnjudgedState = "unjudged"
)
// UnjudgedValue is one key of the unjudged state: the module whose new manifest the digest is, and the merge that wrote it.
@@ -55,7 +54,7 @@ func (ix *Indexer) keepUnjudged(ctx context.Context, digest, module string, w *m
return data, false, nil
})
if errors.Is(err, process.ErrNotYet) {
ix.logf("judge: the state %s is not on the bus yet, so the new manifest %s of %s is kept for its judging only in this take-in", UnjudgedBucket, digest, module)
ix.logf("judge: the state %s is not on the bus yet, so the new manifest %s of %s is kept for its judging only in this take-in", UnjudgedState, digest, module)
return nil
}
if err != nil {
@@ -73,7 +72,7 @@ func (ix *Indexer) withUnjudged(ctx context.Context, fresh map[string]bool) map[
keys, err := ix.Unjudged.Keys(ctx)
if err != nil {
if !errors.Is(err, process.ErrNotYet) {
ix.logf("judge: the state %s cannot be listed (%v), so only this take-in's new manifests are judged as new", UnjudgedBucket, err)
ix.logf("judge: the state %s cannot be listed (%v), so only this take-in's new manifests are judged as new", UnjudgedState, err)
}
return fresh
}
@@ -19,15 +19,13 @@ import (
)
// The indexer judges every manifest it holds, once, with its own judge (novox/hq ADR 0578 §1): it writes each verdict in
// its state verdicts (VerdictsBucket), one key per manifest digest, only when the verdict differs from the one
// its state verdicts (VerdictsState), one key per manifest digest, only when the verdict differs from the one
// held; a module's value in the modules state carries a refusal (Verdict, Problems), so a reader of modules takes it
// without a second read; a new refusal is said as the event manifest-refused; a manifest with a field the judge does not know waits instead (ownBuildIsNewest). It judges at its start, so a judge that
// changed (a new indexer) judges every manifest again, and after every merge it takes in, since a rule between manifests
// can turn on any of them. Nothing polls.
// VerdictsBucket is the indexer's state of verdicts; VerdictsState is its local name, as the tools bundle reads it.
const VerdictsBucket = Module + "_" + VerdictsState
// VerdictsState is the indexer's state of verdicts, its local name, as the tools bundle reads it.
const VerdictsState = "verdicts"
// Judged is what one pass of the judge did: the manifests judged, the verdicts written, and the modules refused.
@@ -229,7 +227,7 @@ func (ix *Indexer) judgeOverHeld(ctx context.Context, fresh map[string]bool, ove
}
wrote, revision, err := ix.writeVerdict(ctx, digests[name], v)
if errors.Is(err, process.ErrNotYet) {
return out, fmt.Errorf("the state %s is not on the bus yet: %w", VerdictsBucket, err)
return out, fmt.Errorf("the state %s is not on the bus yet: %w", VerdictsState, err)
}
if err != nil {
ix.logf("judge: the verdict on %s (digest %s) is not written: %v", name, digests[name], err)
@@ -512,7 +510,6 @@ func (ix *Indexer) writeVerdict(ctx context.Context, digest string, v meshcatalo
// does not, since when and why.
const (
OwnBuildStateName = "own-build"
OwnBuildBucket = Module + "_" + OwnBuildStateName
OwnBuildKey = "state"
)
@@ -21,8 +21,7 @@ import (
// the source revision built and its edge entries. It hears every artifact's package-published and records a version as
// released once every artifact the module's entry names is published at it.
const (
VersionsState = "versions"
VersionsBucket = Module + "_" + VersionsState
VersionsState = "versions"
// VersionsKept is how many versions of a module the state keeps (§349): at each release the module's oldest beyond them
// go, never its highest of any MAJOR.MINOR.
VersionsKept = 100
@@ -2,12 +2,8 @@ package index
import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
"git.novox.be/novox/mesh-sdk/go/views"
)
// MarksBucket is the indexer's state of what its views family said each state up to.
const MarksBucket = Module + "_" + views.MarksState
// FirstViewTools are the indexer's first-view tools (ADR 0621 §1, ADR 0622 §1, ADR 0631 §1): the tool, the state it answers, and
// the words of what that state holds. Each answers only its callers list (CallersEnv), which the manifest writes. The fourth
// first view, of which modules exist, is the seat mesh-catalog's verb revision (ADR 0663 §1, RevisionTool), granted by
@@ -38,17 +34,8 @@ type ViewedTool struct {
Tool, State, Words string
}
// ViewedStates are the states the indexer's views family says, each by its bucket: the ones its readers read.
var ViewedStates = map[string]string{
"manifests": ManifestsBucket,
VerdictsState: VerdictsBucket,
PredefinedState: PredefinedBucket,
ConfigCandidatesState: ConfigCandidatesBucket,
"candidates": CandidatesBucket,
PackagesState: PackagesBucket,
HeardState: HeardBucket,
"taken": TakenBucket,
FindingsState: FindingsBucket,
SeatDefinitionsState: SeatDefinitionsBucket,
UnjudgedState: UnjudgedBucket,
// ViewedStates are the states the indexer's views family says: the ones its readers read.
var ViewedStates = map[string]bool{
ManifestsState: true, VerdictsState: true, PredefinedState: true, ConfigCandidatesState: true, CandidatesState: true,
PackagesState: true, HeardState: true, TakenState: true, FindingsState: true, SeatDefinitionsState: true, UnjudgedState: true,
}
+1 -1
View File
@@ -1,6 +1,6 @@
// Package judge is the mesh's one judge of module manifests (novox/hq ADR 0578 §1, §2): strict decoding, the seat and
// namespace rules, the bus and stream limits, and the rules between the manifests of one catalogue. Only the indexer,
// which judges every manifest it takes in and writes the verdicts (index.VerdictsBucket), and the checker, which
// which judges every manifest it takes in and writes the verdicts (its state verdicts), and the checker, which
// judges every manifest a change touches by running this judge built at the trunk or a group's head (cmd/mesh-judge,
// as the composer is run), use it. Every other part decodes a manifest leniently (catalogue.Decode) and takes the
// verdict; it never judges (ADR 0578 §3).
@@ -147,9 +147,11 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
if index.Module != m.Module {
t.Errorf("the indexer names itself %q, its manifest's module is %q", index.Module, m.Module)
}
for bucket, history := range map[string]int{index.CandidatesBucket: index.CandidatesHistory, index.ManifestsBucket: index.ManifestsHistory,
index.VerdictsBucket: index.VerdictsHistory, index.PackagesBucket: index.PackagesHistory, index.TakenBucket: 1, index.PredefinedBucket: 5, index.ConfigCandidatesBucket: 5} {
if !slices.ContainsFunc(m.State, func(st state) bool { return m.Module+"_"+st.Name == bucket && st.History == history }) {
// A-296: its own buckets are named from its credential's module (index.Bucket), each a state its manifest keeps.
own := func(state string) string { return index.Bucket("a-indexer", state) }
for bucket, history := range map[string]int{own(index.CandidatesState): index.CandidatesHistory, own(index.ManifestsState): index.ManifestsHistory,
own(index.VerdictsState): index.VerdictsHistory, own(index.PackagesState): index.PackagesHistory, own(index.TakenState): 1, own(index.PredefinedState): 5, own(index.ConfigCandidatesState): 5} {
if !slices.ContainsFunc(m.State, func(st state) bool { return "a-indexer_"+st.Name == bucket && st.History == history }) {
t.Errorf("the indexer names its bucket %s with %d revisions, its manifest's states are %+v", bucket, history, m.State)
}
}