Merge branch 'ix16-wip' into feat/47-G-build-indexer

This commit is contained in:
2026-11-11 13:55:20 +01:00
32 changed files with 2277 additions and 122 deletions
@@ -126,7 +126,8 @@ func TestManifest(t *testing.T) {
if !slices.Equal(m.Consumes, []string{"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-delivery-indexer.views.graph", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", buckets.TellerModule + "." + tellerevents.AskEnded, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-engine.manifest-reads-changed"}) {
t.Errorf("consumes = %v, want the indexer's views of graph, manifests and modules, the teller's ask-ended and the engines' reads changed", m.Consumes)
}
if want := []string{"mesh-delivery-indexer.graph", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules",
if want := []string{"mesh-delivery-indexer.graph", "mesh-delivery-indexer.manifests",
"seat:mesh-catalog.revision", // ADR 0663 §2: which modules exist
"seat:mesh-catalog.seat-definitions", // ADR 0636 §4: the seat list
"seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", // ADR 0565 §9, §11: the seat alone
"seat:node-backup.backed-up", "seat:node-backup.now",
+1 -1
View File
@@ -52,7 +52,7 @@
"invokes": [
"mesh-delivery-indexer.graph",
"mesh-delivery-indexer.manifests",
"mesh-delivery-indexer.modules",
"seat:mesh-catalog.revision",
"seat:mesh-catalog.seat-definitions",
"seat:mesh-hand-acts.hand-act",
"seat:mesh-hand-acts.hand-acts",
@@ -13,8 +13,9 @@ import (
"mesh-delivery-indexer/internal/index"
)
// catalogRead reads the indexer's modules, manifests and graph states for the seat's verbs.
func catalogRead(modules, manifests, nodes stdio.KeptState, seatDefinitions, marks views.Source) index.CatalogRead {
// catalogRead reads the indexer's view of the trunk (its modules and module graph, from the newest catalog graph
// revision) and its manifests state for the seat's verbs.
func catalogRead(modules views.Source, manifests stdio.KeptState, nodes views.Source, seatDefinitions, marks views.Source) index.CatalogRead {
return index.CatalogRead{
SeatDefinitions: seatDefinitions, Marks: marks,
Modules: func() (map[string]meshcatalog.ModulesByModuleValue, error) {
@@ -24,11 +25,11 @@ func catalogRead(modules, manifests, nodes stdio.KeptState, seatDefinitions, mar
}
out := map[string]meshcatalog.ModulesByModuleValue{}
for _, k := range keys {
e, err := modules.Get(k)
e, ok, err := modules.Get(k)
if err != nil {
return nil, fmt.Errorf("modules.%s: %w", k, err)
}
if e == nil {
if !ok {
continue
}
if m, err := meshcatalogclient.DecodeModule(e.Value); err == nil {
@@ -59,11 +60,11 @@ func catalogRead(modules, manifests, nodes stdio.KeptState, seatDefinitions, mar
}
g := graph.Graph{}
for _, k := range keys {
e, err := nodes.Get(k)
e, ok, err := nodes.Get(k)
if err != nil {
return nil, fmt.Errorf("graph.%s: %w", k, err)
}
if e == nil {
if !ok {
continue
}
if n, err := graph.Decode(e.Value); err == nil {
@@ -76,7 +77,7 @@ func catalogRead(modules, manifests, nodes stdio.KeptState, seatDefinitions, mar
}
// catalogTools are the seat's verbs over the indexer's own states, served beside the indexer's own tools, since the indexer's manifest claims the seat (ADR 0604 §2).
func catalogTools() []stdio.Tool {
return index.CatalogTools(catalogRead(stdio.State("modules"), stdio.State("manifests"), stdio.State(graph.State),
func catalogTools(state func(string) views.Source) []stdio.Tool {
return index.CatalogTools(catalogRead(state("modules"), stdio.State("manifests"), state(graph.State),
keptSource(index.SeatDefinitionsState), keptSource(views.MarksState)))
}
@@ -17,6 +17,7 @@ import (
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/busstate"
"mesh-delivery-indexer/internal/index"
)
@@ -56,7 +57,10 @@ func Tools(ask asker, now func() time.Time, read stdio.KeptState, counts func()
"source names for that repository, for a merge whose merged message the indexer never heard (the forge bridge has no " +
"tool): a head whose trunk holds no module.json is taken in as a merge that leaves none (every module of the repository " +
"is dropped, one another repository's trunk holds handed over from there first, listed in handed and dropped); a head " +
"that holds modules is answered \"take each module in by name\".",
"that holds modules is answered \"take each module in by name\". With neither module nor repository it is the indexer's " +
"catch-up tool, named in its manifest beside the catalog graph revisions: it takes in the head the gate's source names for " +
"every repository, as merges, and keeps the view as the next revision when an entry moved; the node agent calls it so " +
"after restoring them. Answers the repositories taken in and, in unread, those whose merged message was not read.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "the module's name, as the catalogue registers it"},
"repository": map[string]any{"type": "string", "description": "instead of module: the repository (owner/name) whose head is taken in"},
@@ -65,9 +69,12 @@ func Tools(ask asker, now func() time.Time, read stdio.KeptState, counts func()
var key string
var value []byte
var err error
if str(args, "repository") != "" && str(args, "module") == "" {
switch {
case str(args, "repository") == "" && str(args, "module") == "":
key, value, err = index.NewCatchUp(str(args, "why"), stdio.CallerOf(args), now())
case str(args, "repository") != "" && str(args, "module") == "":
key, value, err = index.NewTakeInRepository(str(args, "repository"), str(args, "why"), stdio.CallerOf(args), now())
} else {
default:
key, value, err = index.NewTakeIn(str(args, "module"), str(args, "why"), stdio.CallerOf(args), now())
}
if err != nil {
@@ -275,16 +282,26 @@ func main() {
asks := askTakeIn(stdio.State(index.TakeInsState), 25*time.Second)
acts := meshhandactsclient.Recorder{Ask: stdio.Ask}
tools := Tools(asks, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState)), acts)
tools = append(tools, catalogTools()...)
tools = append(tools, viewTools(os.Getenv, keptSource, time.Now)...)
// The modules and the module graph are the newest catalog graph revision's (milestone 47/G): revision.go.
revisions := bundleObjects{busstate.ObjectsOf(index.CatalogRevisionsObjects)}
trunk := index.TrunkStates{Objects: revisions, Under: keptSource}
tools = append(tools, catalogTools(trunk.State)...)
// The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly
// the parts whose entry declares it (novox/hq ADR 0663).
tools = append(tools, index.RevisionTool(revisions, index.Module))
tools = append(tools, index.CatalogRevisionTools(revisions, asks, time.Now, func(what, why string) error {
_, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.DropCandidateGraphTool, Verb: "drop-candidate-graph"})
return err
})...)
tools = append(tools, viewTools(os.Getenv, trunk.State, time.Now)...)
tools = append(tools, index.PendingTools(keptRead{stdio.State(index.PendingState)}, keptRead{stdio.State(index.OwnBuildStateName)}, asks, time.Now)...)
tools = append(tools, index.FingerprintsTools(asks, time.Now, func(what, why string) error {
_, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.FingerprintsAgainTool, Verb: "fingerprints-again"})
return err
}, 10*time.Second)...)
tools = append(tools, index.TryCandidateTools(keptRead{stdio.State("candidates")}, keptRead{stdio.State(index.TryRecoveryState)}, asks, time.Now)...)
tools = append(tools, configReadersTool(stdio.NewWatchedStates()), busStateTool(keptSource), requirementsTool(keptSource),
packageUsersTool(keptSource), seatDefinitionTool(keptSource), outOfStepTool(keptSource),
tools = append(tools, configReadersTool(stdio.NewWatchedStates(), trunk.State("modules")), busStateTool(trunk.State), requirementsTool(keptSource),
packageUsersTool(trunk.State), seatDefinitionTool(keptSource), outOfStepTool(keptSource),
holdsTool(stdio.State(index.HoldsState), asks, time.Now, acts), engineReadsTool(time.Now))
if err := stdio.Serve("", tools); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err)
@@ -0,0 +1,50 @@
package main
import (
"context"
"git.novox.be/novox/mesh-sdk/go/busstate"
"mesh-delivery-indexer/internal/index"
)
// bundleObjects is the objects catalog-revisions as the bundle reaches them through its runtime (the old objects section
// of a foundation part, reached by the module's own name).
type bundleObjects struct{ o busstate.Objects }
func (b bundleObjects) Put(_ context.Context, name string, data []byte) error {
_, err := b.o.Put(name, data, "")
return err
}
func (b bundleObjects) Get(_ context.Context, name string) ([]byte, error) {
d, err := b.o.Get(name)
if err != nil {
return nil, err
}
if d.Info.Name == "" || d.Info.Deleted {
return nil, index.ErrNoObject
}
return d.Data, nil
}
func (b bundleObjects) Delete(_ context.Context, name string) error { return b.o.Delete(name) }
func (b bundleObjects) Names(context.Context) ([]string, error) {
infos, err := b.o.List()
if err != nil {
return nil, err
}
names := []string{}
for _, i := range infos {
if !i.Deleted {
names = append(names, i.Name)
}
}
return names, nil
}
func (b bundleObjects) Fill(context.Context) (int64, int64, error) {
f, err := b.o.Fill()
return int64(f.Bytes), int64(f.MaxBytes), err
}
@@ -37,9 +37,9 @@ func viewTools(getenv func(string) string, state func(string) views.Source, now
func keptSource(name string) views.Source { return views.KeptSource{State: stdio.State(name)} }
// configReadersTool is indexer_config_readers (design 32 *A value no longer read*; DLV-I10): read from one watched copy
// of the indexer's modules, manifests and config-candidates states, so an ask on each manifest-accepted reads no state
// of the indexer's manifests and config-candidates states and its view of the trunk (the newest catalog graph revision), so an ask on each manifest-accepted reads no state
// whole again.
func configReadersTool(watched *stdio.WatchedStates) stdio.Tool {
func configReadersTool(watched *stdio.WatchedStates, modules views.Source) stdio.Tool {
return stdio.Tool{Name: index.ConfigReadersTool,
Description: "Who reads a module's config keys now (design 32): every value key and own secret any manifest of the module " +
"the indexer keeps ever declared, each with its readers — the module itself where its trunk's manifest declares it, and " +
@@ -53,13 +53,15 @@ func configReadersTool(watched *stdio.WatchedStates) stdio.Tool {
return nil, fmt.Errorf("%s names the module", index.ConfigReadersTool)
}
trunk := map[string]meshcatalog.ModulesByModuleValue{}
mods, err := watched.Values("modules")
keys, err := modules.Keys()
if err != nil {
return nil, fmt.Errorf("the modules state cannot be read: %w", err)
return nil, fmt.Errorf("the view of the trunk cannot be read: %w", err)
}
for k, v := range mods {
if m, err := meshcatalogclient.DecodeModule(v); err == nil {
trunk[k] = m
for _, k := range keys {
if e, ok, err := modules.Get(k); err == nil && ok {
if m, err := meshcatalogclient.DecodeModule(e.Value); err == nil {
trunk[k] = m
}
}
}
kept := map[string]meshcatalog.ManifestsByDigestValue{}
@@ -39,6 +39,7 @@ import (
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-delivery/sdk/graph"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-indexer/internal/index"
@@ -86,7 +87,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
}, func(ctx context.Context, js jetstream.JetStream) {
// The states and the stream the controller makes from the indexer's manifest (old sections, ADR 0595 §11).
for bucket, history := range map[string]uint8{index.PendingBucket: 1, index.VerdictsBucket: index.VerdictsHistory,
index.GraphBucket: 1, index.TakenBucket: 1, index.PackagesBucket: 1,
index.TakenBucket: 1, index.PackagesBucket: 1,
index.SeatDefinitionsBucket: 1, index.RequirementsBucket: 1, findingsBucket: 1, index.HoldsBucket: 1, index.UnjudgedBucket: 1} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucket, History: history}); err != nil {
t.Fatal(err)
@@ -262,6 +263,13 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
named := held.WaitsFor
get := func(bucket, key string) ([]byte, bool) {
t.Helper()
if bucket == index.ModulesState { // the view of the trunk, from the newest catalog graph revision
e, err := revisionView{js: js}.Get(ctx, key)
if err != nil {
return nil, false
}
return e.Value(), true
}
kv, err := js.KeyValue(ctx, bucket)
if err != nil {
t.Fatal(err)
@@ -273,7 +281,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
return e.Value(), true
}
// Only what building and moving the named indexer needs is written.
raw, ok := get(index.ModulesBucket, ix)
raw, ok := get(index.ModulesState, ix)
own, err := meshcatalogclient.DecodeModule(raw)
if !ok || err != nil || own.Fp2 != named || own.Commit != m1.Merge {
t.Fatalf("modules.%s = %+v (%v), want the named build %s at the merge", ix, own, err, named)
@@ -284,11 +292,8 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
if _, ok := get(index.VerdictsBucket, own.Manifest); !ok {
t.Errorf("the verdict on the indexer's manifest %s is not written", own.Manifest)
}
if gk, _ := graph.Key(ix); !func() bool { _, ok := get(index.GraphBucket, gk); return ok }() {
t.Errorf("the indexer's key %s of the module graph is not written", gk)
}
repoKey, _ := trunk.RepositoryKey(m1.Repository)
if _, ok := get(index.ModulesBucket, "a"); ok {
if _, ok := get(index.ModulesState, "a"); ok {
t.Errorf("modules.a is written while the revision is pending")
}
if _, ok := get(index.TakenBucket, repoKey); ok {
@@ -348,7 +353,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
break
}
}
if raw, ok := get(index.ModulesBucket, "a"); !ok || !strings.Contains(string(raw), m1.Merge) {
if raw, ok := get(index.ModulesState, "a"); !ok || !strings.Contains(string(raw), m1.Merge) {
t.Errorf("modules.a = %s after the take-in at the start, want it at the merge %s", raw, m1.Merge)
}
if _, ok := get(index.TakenBucket, repoKey); !ok {
@@ -357,7 +362,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
if len(w) != 1 || w[0].Key != groupKey || len(w[0].Keys) != 1 || w[0].Keys[0] != g1Key {
t.Fatalf("indexer_pending after the start = %+v, want only the group gathered, its first member %s", w, g1Key)
}
if _, ok := get(index.ModulesBucket, "c"); ok {
if _, ok := get(index.ModulesState, "c"); ok {
t.Errorf("modules.c is written while its group is gathered")
}
@@ -376,8 +381,8 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
}
var hasC, hasK bool
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && !(hasC && hasK && len(w) == 0); time.Sleep(50 * time.Millisecond) {
_, hasC = get(index.ModulesBucket, "c")
_, hasK = get(index.ModulesBucket, "k")
_, hasC = get(index.ModulesState, "c")
_, hasK = get(index.ModulesState, "k")
w = waits()
}
if !hasC || !hasK || len(w) != 0 {
@@ -471,7 +476,75 @@ var acceptedHeard atomic.Int64
// startIndexerRun starts the bus, the stream FORGE with the indexer's durable consumer, the buckets, lets put fill
// mesh-controller_builds, and runs the program; restart starts it again. mesh-delivery-gate_source is never created.
func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, jetstream.KeyValue, func()) {
// revisionView reads the indexer's view of the trunk as the indexer itself does at a restart (milestone 47/G): the newest
// trunk revision its objects catalog-revisions keep, a module by its key.
type revisionView struct{ js jetstream.JetStream }
func (v revisionView) Get(ctx context.Context, key string) (jetstream.KeyValueEntry, error) {
view, _, _, err := index.TrunkView(ctx, newBusObjects(v.js, testAccount, index.CatalogRevisionsObjects))
if err != nil {
return nil, err
}
value, rev, found, err := view.Get(ctx, key)
if err != nil {
return nil, err
}
if !found {
return nil, jetstream.ErrKeyNotFound
}
return viewEntry{key: key, value: value, revision: rev}, nil
}
// History is the module's value in every trunk revision the objects keep that holds it, oldest first.
func (v revisionView) History(ctx context.Context, key string) ([]jetstream.KeyValueEntry, error) {
o := newBusObjects(v.js, testAccount, index.CatalogRevisionsObjects)
names, err := o.Names(ctx)
if err != nil {
return nil, err
}
var revs []int64
for _, n := range names {
if r, ok := index.TrunkRevisionOf(n); ok {
revs = append(revs, r)
}
}
slices.Sort(revs)
var out []jetstream.KeyValueEntry
for _, r := range revs {
_, entries, err := index.ReadRevision(ctx, o, fmt.Sprintf("trunk.%d", r))
if err != nil {
return nil, err
}
for _, e := range entries {
if e.Module != key {
continue
}
data, err := meshcatalogclient.EncodeModule(index.ModuleValueOf(e))
if err != nil {
return nil, err
}
out = append(out, viewEntry{key: key, value: data, revision: uint64(r)})
}
}
return out, nil
}
// viewEntry is one module's value of the view.
type viewEntry struct {
key string
value []byte
revision uint64
}
func (e viewEntry) Bucket() string { return index.CatalogRevisionsObjects }
func (e viewEntry) Key() string { return e.key }
func (e viewEntry) Value() []byte { return e.value }
func (e viewEntry) Revision() uint64 { return e.revision }
func (e viewEntry) Created() time.Time { return time.Time{} }
func (e viewEntry) Delta() uint64 { return 0 }
func (e viewEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut }
func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, revisionView, func()) {
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
if err != nil {
t.Fatal(err)
@@ -526,10 +599,11 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before .
if err != nil {
t.Fatal(err)
}
modules, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.ModulesBucket, History: index.ModulesHistory})
if err != nil {
// The indexer's objects catalog-revisions, which the grants part creates from its manifest's old objects section.
if _, err := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: newBusObjects(nil, testAccount, index.CatalogRevisionsObjects).bucket()}); err != nil {
t.Fatal(err)
}
modules := revisionView{js: js}
// The bucket of the manifests, made by the controller from the manifest's state entry manifests (history 1, issue 1449).
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.ManifestsBucket, History: index.ManifestsHistory}); err != nil {
t.Fatal(err)
@@ -583,7 +657,7 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before .
}
// startIndexer is startIndexerRun for a test that never starts the program again.
func startIndexer(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, jetstream.KeyValue) {
func startIndexer(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, revisionView) {
js, ctx, candidates, modules, _ := startIndexerRun(t, put, before...)
return js, ctx, candidates, modules
}
@@ -612,8 +686,14 @@ func (l buildLog) end(t *testing.T, repo, module, path string) {
// message on FORGE makes modules.<m> within 20 s, the fingerprint of the merge's tree and the recorded inputs.
func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9; I1, I3: the modules are the merged message's, no build record names them (issue 1851)
// The bucket holds only build-agent's record, which no fingerprint reads since R4g (ADR 0530 §1).
var asks jetstream.KeyValue
js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds buildLog) {
builds.end(t, "novox/mesh-controller", "build-agent", "modules/build-agent")
}, func(ctx context.Context, js jetstream.JetStream) {
var err error
if asks, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.TakeInsBucket, History: 1}); err != nil {
t.Fatal(err)
}
})
const merge = "89abcdef0123456789abcdef0123456789abcdef"
data, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
@@ -636,6 +716,60 @@ func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9; I1, I3: the m
if m.Repository != "novox/mesh-delivery" || m.Commit != merge || m.Path != "modules/a" || m.Manifest == "" || m.Fp != "" || m.None != "" { // 1606/AC2
t.Errorf("module = %+v within 20 s, want path modules/a, its manifest, and no value of today's form", m)
}
// Milestone 47/G W4.13 (design 47, the indexer whole): the source revision taken in is one catalog graph revision kept
// as objects, said as catalog-revision on the seat's subject, every entry it lists stored once by digest.
objects := newBusObjects(js, testAccount, index.CatalogRevisionsObjects)
list, err := index.ListRevisions(ctx, objects, false)
if err != nil || len(list.Revisions) != 1 || list.Revisions[0].Name != "trunk.1" || !list.Revisions[0].Current ||
list.Entries != list.Revisions[0].Entries || len(list.Unlisted) != 0 || list.Fill.Bytes == 0 {
t.Fatalf("revisions kept = %+v (%v): want trunk.1 alone, current, each entry it lists stored once", list, err)
}
events, err := js.Stream(ctx, "EVENTS")
if err != nil {
t.Fatal(err)
}
if msg, err := events.GetLastMsgForSubject(ctx, "mesh.seat.mesh-catalog.event."+meshcatalog.EventCatalogRevision); err != nil ||
!strings.Contains(string(msg.Data), `"revision":1`) {
t.Errorf("catalog-revision said: %v (%v): want revision 1", msg, err)
}
// W4.12 through the process: a recompute by hand moves nothing and answers equal; a candidate's half dropped by hand
// leaves the trunk's revision and its entries.
if err := index.KeepRevision(ctx, objects, meshcatalog.CatalogRevision{Kind: "candidate", Group: "g1", Heads: "h1", Revision: 1,
Entries: []meshcatalog.EntryRef{{Module: "x", Digest: "dx"}}}, map[string][]byte{"dx": []byte(`{"module":"x"}`)}); err != nil {
t.Fatal(err)
}
answer := func(key string, value []byte) index.TakenIn {
t.Helper()
w, err := asks.Watch(ctx, key, jetstream.UpdatesOnly())
if err != nil {
t.Fatal(err)
}
defer w.Stop()
if _, err := asks.Put(ctx, key, value); err != nil {
t.Fatal(err)
}
for e := range w.Updates() {
if e == nil {
continue
}
if a, err := index.DecodeTakeIn(e.Value()); err == nil && a.Done() {
return *a.Result
}
}
t.Fatalf("%s is not answered", key)
return index.TakenIn{}
}
key, value, _ := index.NewRecomputeAsk("a check", "ctl", time.Now())
if r := answer(key, value); !r.Stored || !strings.Contains(r.Answer, "equal to the kept revision 1") {
t.Errorf("indexer_recompute answered %+v: want equal to the kept revision 1", r)
}
key, value, _ = index.NewDropCandidateGraphAsk("g1", "h1", "its gate event never came", "ctl", time.Now())
if r := answer(key, value); !r.Stored || !slices.Equal(r.Dropped, []string{"entry.dx"}) {
t.Errorf("indexer_drop_candidate_graph answered %+v: want candidate.g1.h1 dropped with entry.dx", r)
}
if after, err := index.ListRevisions(ctx, objects, false); err != nil || len(after.Revisions) != 1 || after.Entries != list.Entries {
t.Errorf("revisions after the drop = %+v (%v): want trunk.1 and its entries alone", after, err)
}
}
// TestIndexerKeepsManifestsAtAMerge: a merged message of mesh-catalog carrying the real manifest of the module avahi and the
@@ -960,17 +1094,16 @@ func TestIndexerJudgesAnUnreadableVerdictAgainAtStart(t *testing.T) { // 1923/AC
if _, err := manifests.Put(ctx, mkey, mdata); err != nil {
t.Fatal(err)
}
stale, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "89abcdef0123456789abcdef0123456789abcdef", Path: "modules/avahi",
// The view the indexer starts from is the newest trunk revision its objects keep (milestone 47/G).
stale := meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "89abcdef0123456789abcdef0123456789abcdef", Path: "modules/avahi",
Manifest: digest, None2: "the manifest cannot be read: claims node-build-go, which is a seat in the mesh's own namespace that it does not define",
Inputs2: &meshcatalog.ModulesByModuleValueInputs2{Tree: "tree-avahi", Locks: map[string]string{"go.sum": goSum}}})
Inputs2: &meshcatalog.ModulesByModuleValueInputs2{Tree: "tree-avahi", Locks: map[string]string{"go.sum": goSum}}}
rev, objects, err := index.CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk"},
map[string]meshcatalog.CatalogEntry{"avahi": index.CatalogEntryOf("avahi", stale, catalogue.Manifest{}, graph.Node{None: stale.None2})})
if err != nil {
t.Fatal(err)
}
held, err := js.KeyValue(ctx, index.ModulesBucket)
if err != nil {
t.Fatal(err)
}
if _, err := held.Put(ctx, "avahi", stale); err != nil {
if err := index.KeepRevision(ctx, newBusObjects(js, testAccount, index.CatalogRevisionsObjects), rev, objects); err != nil {
t.Fatal(err)
}
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.ToolchainsBucket("mesh-build-go"), History: 1})
@@ -375,9 +375,21 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil {
return err
}
modules, err := process.EssentialBucket(ctx, js, "indexer", index.ModulesBucket, "write a module's fingerprint")
if err != nil {
return err
// The view of the trunk is the newest trunk revision of the catalog graph the objects keep (milestone 47/G, ADR 0656 §6:
// one form of the fact), read once here and kept as a revision after every write; objects not on the bus yet, or none
// kept, start it empty, and the first start's catch-up takes the trunk in once the gate's source is read (catchUp).
revisions := newBusObjects(js, cred, index.CatalogRevisionsObjects)
modules, graph, at, err := index.TrunkView(ctx, revisions)
switch {
case errors.Is(err, process.ErrNotYet):
logger.Printf("the view of the trunk starts empty: %v; the trunk is taken in once they are there and the gate's source is read", err)
modules, graph = index.MemoryStore(), index.MemoryStore()
case err != nil:
return fmt.Errorf("the view of the trunk cannot be read from the newest catalog graph revision: %w", err)
case at == 0:
logger.Printf("the view of the trunk starts empty: %s keeps no trunk revision yet; the trunk is taken in once the gate's source is read", index.CatalogRevisionsObjects)
default:
logger.Printf("the view of the trunk read from trunk revision %d", at)
}
manifests, err := process.EssentialBucket(ctx, js, "indexer", index.ManifestsBucket, "keep a module's manifest")
if err != nil {
@@ -413,8 +425,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// What each consumer's go.sum requires of each mesh Go module (DLV-I7): a state this delivery brings, bound when it
// appears; until then indexer_requirements answers none, and the builder's floor waits on its answer.
requirements := process.AwaitBucket(ctx, js, conn, index.RequirementsBucket, "what consumers require is not kept", logger.Printf)
// The module graph (ADR 0603 §2): a state this delivery brings, bound when it appears; until then no graph is written.
graph := process.AwaitBucket(ctx, js, conn, index.GraphBucket, "no module graph is written", logger.Printf)
// The module graph (ADR 0603 §2) is held in memory, read with the view: each revision's entries carry its nodes.
ownBuildState := process.AwaitBucket(ctx, js, conn, index.OwnBuildBucket, "which build this indexer runs is not kept", logger.Printf)
// The manifests a take-in wrote and no judge pass judged yet (issue 1966): a state this delivery brings, bound when it
// appears; until then a take-in's new manifests are kept for their judging in its memory alone.
@@ -427,14 +438,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
seatBus := &seatBus{defs: awaitedSource{ctx, seatDefs}, marks: awaitedSource{ctx, marks}}
seatReader, saySeats := followSeatList(seatBus, saySeatDefinition(js, cred), logger.Printf)
readWhenBound(seatReader, seatDefs, marks)
ix := &index.Indexer{Findings: awaitedStore{b: findings}, OldSections: awaitedStore{b: oldSections}, Unjudged: awaitedStore{b: unjudged}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known,
ix := &index.Indexer{Findings: awaitedStore{b: findings}, OldSections: awaitedStore{b: oldSections}, Unjudged: awaitedStore{b: unjudged}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: modules, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known,
Verdicts: awaitedStore{b: verdicts}, SayRefused: sayRefused(conn, cred, logger.Printf), SayAccepted: sayAccepted(conn, cred),
Predefined: awaitedStore{b: predefined}, ConfigCandidates: awaitedStore{b: configCandidates}, Logf: logger.Printf,
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call),
ReadOpen: openVia(calls.Call), Packages: awaitedStore{b: packages, js: js, name: index.PackagesBucket}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call),
OwnBuild: ownBuildVia(calls.Call, cred.Node), OpenTries: openTriesVia(calls.Call), OwnBuildState: awaitedStore{b: ownBuildState}, TryRecovery: awaitedStore{b: tryRecovery}, Graph: awaitedStore{b: graph},
OwnBuild: ownBuildVia(calls.Call, cred.Node), OpenTries: openTriesVia(calls.Call), OwnBuildState: awaitedStore{b: ownBuildState}, TryRecovery: awaitedStore{b: tryRecovery}, Graph: graph,
SeatDefinitions: awaitedStore{b: seatDefs}, ReadFile: readFileVia(calls.Call), SaySeatDefinition: saySeats,
Message: messageVia(calls.CallKept), // a kept call: a read, called again through a transport fault (ADR 0668 §1)
Message: messageVia(calls.CallKept), Revisions: revisions, // Message a kept call: a read, called again through a transport fault (ADR 0668 §1)
SayCatalogRevision: sayCatalogRevision(conn, cred),
SayFill: sayFill(conn, cred),
// The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear;
// until then every merge is taken in at once, as before.
Pending: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.PendingBucket, "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)},
@@ -744,10 +757,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// said up to; a first view (the tools bundle) answers at that mark.
startViews(ctx, js, cred, marks,
findings,
map[string]readyOf{"manifests": boundNow(manifests), "modules": boundNow(modules), "candidates": boundNow(candidates),
map[string]readyOf{"manifests": boundNow(manifests), "candidates": boundNow(candidates),
index.VerdictsState: awaited(verdicts), index.PredefinedState: awaited(predefined),
index.ConfigCandidatesState: awaited(configCandidates), index.PackagesState: awaited(packages),
graphState: awaited(graph), index.HeardState: awaited(heard.b), "taken": awaited(taken), "findings": awaited(findings),
index.HeardState: awaited(heard.b), "taken": awaited(taken), "findings": awaited(findings),
index.SeatDefinitionsState: awaited(seatDefs)}, errs, logger.Printf)
cc, err := process.HearEvents(cons, func(m jetstream.Msg) {
handle(ctx, m, ix, heard, logger)
@@ -79,6 +79,9 @@ func (m *memStore) Delete(_ context.Context, k string, _ uint64) error { delete(
const head = "0123456789abcdef0123456789abcdef01234567"
// testAccount is the bus account the tests run the indexer as.
var testAccount = process.Account{Node: "n1", Module: "mesh-delivery-indexer"}
func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part)
// Review round 2: the indexer's views emitter writes its finding view-unsaid (Emitter.Findings set).
if em := viewsEmitter(context.Background(), nil, process.Account{}, nil, &process.AwaitedBucket{}, nil, t.Logf); em.Findings == nil {
@@ -143,6 +146,10 @@ func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part)
t.Errorf("answered leaves %v, presence asked %v", held.vals, asked)
}
})
// Milestone 47/G: the object buckets are the module's own by the name its account runs as, never a fixed name.
if b := newBusObjects(nil, process.Account{Module: "renamed-indexer"}, index.CatalogRevisionsObjects).bucket(); b != "renamed-indexer_catalog-revisions" {
t.Errorf("the catalog graph revisions open as %s, want the account's module's", b)
}
t.Run("not a forge message: terminated and logged", func(t *testing.T) { // 1238 (never redelivered)
var logs bytes.Buffer
st := &memStore{vals: map[string][]byte{}}
@@ -0,0 +1,166 @@
package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"sync"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-indexer/internal/index"
)
// busObjects is one of the indexer's own object buckets in the old `objects` section (novox/hq ADR 0619 §2), which the
// grants part creates: it is opened at its first use and kept; until the bucket exists every call is an error saying so,
// and the next call opens it again (no timer: the next revision or tool call asks).
type busObjects struct {
js jetstream.JetStream
// module is the module this program runs as, read from its bus account (process.Account.Module), never a fixed name.
module string
local string
mu sync.Mutex
o jetstream.ObjectStore
}
// newBusObjects is the object bucket of this local name of the module the account runs as.
func newBusObjects(js jetstream.JetStream, cred process.Account, local string) *busObjects {
return &busObjects{js: js, module: cred.Module, local: local}
}
// bucket is the bus's name of the object bucket: the module's own name and the local name.
func (b *busObjects) bucket() string { return catalogue.ObjectsBucket(b.module, b.local) }
func (b *busObjects) open(ctx context.Context) (jetstream.ObjectStore, error) {
b.mu.Lock()
defer b.mu.Unlock()
if b.o != nil {
return b.o, nil
}
o, err := b.js.ObjectStore(ctx, b.bucket())
if errors.Is(err, jetstream.ErrBucketNotFound) || errors.Is(err, jetstream.ErrStreamNotFound) {
return nil, fmt.Errorf("the object bucket %s is not on the bus yet (the grants part creates it from the manifest's objects): %w", b.local, process.ErrNotYet)
}
if err != nil {
return nil, fmt.Errorf("the object bucket %s is not open: %w", b.local, err)
}
b.o = o
return o, nil
}
func (b *busObjects) Put(ctx context.Context, name string, data []byte) error {
o, err := b.open(ctx)
if err != nil {
return err
}
_, err = o.Put(ctx, jetstream.ObjectMeta{Name: name}, bytes.NewReader(data))
return err
}
func (b *busObjects) Get(ctx context.Context, name string) ([]byte, error) {
o, err := b.open(ctx)
if err != nil {
return nil, err
}
data, err := o.GetBytes(ctx, name)
if errors.Is(err, jetstream.ErrObjectNotFound) {
return nil, index.ErrNoObject
}
return data, err
}
func (b *busObjects) Delete(ctx context.Context, name string) error {
o, err := b.open(ctx)
if err != nil {
return err
}
if err := o.Delete(ctx, name); err != nil && !errors.Is(err, jetstream.ErrObjectNotFound) {
return err
}
return nil
}
func (b *busObjects) Names(ctx context.Context) ([]string, error) {
o, err := b.open(ctx)
if err != nil {
return nil, err
}
infos, err := o.List(ctx)
if errors.Is(err, jetstream.ErrNoObjectsFound) {
return []string{}, nil
}
if err != nil {
return nil, err
}
names := make([]string, 0, len(infos))
for _, i := range infos {
names = append(names, i.Name)
}
return names, nil
}
func (b *busObjects) Fill(ctx context.Context) (int64, int64, error) {
o, err := b.open(ctx)
if err != nil {
return 0, 0, err
}
st, err := o.Status(ctx)
if err != nil {
return 0, 0, err
}
info, err := b.js.Stream(ctx, "OBJ_"+st.Bucket())
if err != nil {
return int64(st.Size()), 0, err
}
return int64(st.Size()), info.CachedInfo().Config.MaxBytes, nil
}
// sayCatalogRevision publishes the seat mesh-catalog's event catalog-revision under the duplicate id of its kind and
// number, so a revision said twice is heard once (design 47, Two graphs).
func sayCatalogRevision(conn *nats.Conn, cred process.Account) func(context.Context, meshcatalog.CatalogRevisionEvent) error {
return func(_ context.Context, e meshcatalog.CatalogRevisionEvent) error {
data, err := json.Marshal(e)
if err != nil {
return err
}
msg := nats.NewMsg(seatEventSubject(meshcatalog.EventCatalogRevision))
id := fmt.Sprintf("%s-%s-%d%s", meshcatalog.EventCatalogRevision, e.Kind, e.Revision, e.Heads)
msg.Header.Set("x-event-id", id)
msg.Header.Set("x-source", cred.Module)
msg.Header.Set("x-node", cred.Node)
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
msg.Header.Set("content-type", "application/json")
msg.Header.Set(nats.MsgIdHdr, id)
msg.Data = data
return conn.PublishMsg(msg)
}
}
// sayFill publishes the seat mesh-catalog's event filling, full or room of the catalog graph revisions under the duplicate
// id given (milestone 47/G W4.6), so one said twice is heard once.
func sayFill(conn *nats.Conn, cred process.Account) func(context.Context, string, string, meshcatalog.Fill) error {
return func(_ context.Context, event, id string, f meshcatalog.Fill) error {
data, err := json.Marshal(f)
if err != nil {
return err
}
msg := nats.NewMsg(seatEventSubject(event))
msg.Header.Set("x-event-id", id)
msg.Header.Set("x-source", cred.Module)
msg.Header.Set("x-node", cred.Node)
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
msg.Header.Set("content-type", "application/json")
msg.Header.Set(nats.MsgIdHdr, id)
msg.Data = data
return conn.PublishMsg(msg)
}
}
@@ -15,6 +15,7 @@ import (
type takerIn interface {
TakeIn(ctx context.Context, module string) index.TakenIn
TakeInRepository(ctx context.Context, repository string) index.TakenIn
CatchUp(ctx context.Context) index.TakenIn
JudgeAgain(ctx context.Context, module, digest string) index.TakenIn
SayAcceptedAgain(ctx context.Context, module string) index.TakenIn
SeatDefinitionsAgain(ctx context.Context, module string, pull int, head string) index.TakenIn
@@ -25,6 +26,8 @@ type takerIn interface {
DropTryCandidate(ctx context.Context, id, by, why string) index.TakenIn
RecoverTriesByHand(ctx context.Context, by, why string) index.TakenIn
FingerprintsAgain(ctx context.Context, module string) index.TakenIn
RecomputeByHand(ctx context.Context, by, why string) index.TakenIn
DropCandidateGraph(ctx context.Context, group, heads, by, why string) index.TakenIn
}
// takenIn answers one value of the takeins state (the tool indexer_take_in, issue 1945): a value not yet answered takes the
@@ -43,7 +46,13 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream
return
}
asked, r := "take-in", index.TakenIn{}
if a.Fingerprints {
if a.Recompute {
asked, r = "recompute of the catalog graph", ix.RecomputeByHand(ctx, a.By, a.Why)
a.Module = r.Module
} else if a.CandidateGroup != "" && a.Drop {
asked, r = "drop of a candidate's catalog half", ix.DropCandidateGraph(ctx, a.CandidateGroup, a.CandidateHeads, a.By, a.Why)
a.Module = r.Module
} else if a.Fingerprints {
asked, r = "computing of fingerprints again", ix.FingerprintsAgain(ctx, a.Module)
} else if a.RecoverTries {
asked, r = "recovery of the tries", ix.RecoverTriesByHand(ctx, a.By, a.Why)
@@ -60,6 +69,9 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream
} else if a.Pending != "" {
asked, r = "hand take-in of what waits", ix.TakeInPendingByHand(ctx, a.Pending, a.By, a.Why)
a.Module = a.Pending
} else if a.CatchUp {
asked, r = "catch-up of every repository", ix.CatchUp(ctx)
a.Module = r.Module
} else if a.Repository != "" {
asked, r = "take-in of a repository", ix.TakeInRepository(ctx, a.Repository)
a.Module = a.Repository
@@ -97,6 +97,14 @@ type Indexer struct {
ReadFile func(ctx context.Context, repository, sha, path string) ([]byte, error)
// SaySeatDefinition says seat-definition-changed (seatdefs.go); nil says nothing.
SaySeatDefinition func(ctx context.Context, c SeatDefinitionChanged) error
// Revisions is the object bucket catalog-revisions (catalog_revisions.go; design 47, Two graphs); nil keeps no
// catalog graph revision.
Revisions Objects
// SayCatalogRevision says the seat mesh-catalog's event catalog-revision for a revision kept; nil says nothing.
SayCatalogRevision func(ctx context.Context, e meshcatalog.CatalogRevisionEvent) error
// SayFill says the seat mesh-catalog's event filling, full or room of the catalog graph revisions, under the duplicate
// id given (W4.6).
SayFill func(ctx context.Context, event, id string, f meshcatalog.Fill) error
// Graph is the module graph's state (GraphBucket, graph.go; novox/hq ADR 0603 §2); nil writes no graph.
Graph Store
// Files asks the forge for every blob under each directory at a commit, by directory then path relative to it, each
@@ -164,12 +172,15 @@ type Indexer struct {
failedAt map[string]string // the head of a repository whose tree lacks a directory asked, by repository: not asked again until the head changes
// OwnBuild answers the fingerprint of the build this program runs, from the node-engine's seat; nil knows none.
OwnBuild func(ctx context.Context) (string, error)
underSeats bool // a dry run inside judge.WithSeats: its judge passes do not hold the seat list (verdicts.go)
judgeMu sync.Mutex // one judge pass at a time (verdicts.go)
ownMu sync.Mutex // guards ownFP
ownFP string // the fingerprint of the build this program runs, fixed once at the start (FixOwnBuild)
ownFixed bool // FixOwnBuild asked already: it asks once
ownSince time.Time // when the build was asked and not known, with ownWhy
underSeats bool // a dry run inside judge.WithSeats: its judge passes do not hold the seat list (verdicts.go)
judgeMu sync.Mutex // one judge pass at a time (verdicts.go)
ownMu sync.Mutex // guards ownFP
keepMu sync.Mutex // one catalog graph revision kept at a time (keepEntries)
catchingUp atomic.Bool // the first start's catch-up runs: its merges keep no revision until it ends (CatchUp)
catchUpWG sync.WaitGroup // the first start's catch-up in the background (kickCatchUp), for tests to wait on
ownFP string // the fingerprint of the build this program runs, fixed once at the start (FixOwnBuild)
ownFixed bool // FixOwnBuild asked already: it asks once
ownSince time.Time // when the build was asked and not known, with ownWhy
ownWhy string
// OwnBuildState keeps which build this process knows it runs, or since when and why not (FixOwnBuild); nil keeps none.
OwnBuildState Store
@@ -0,0 +1,141 @@
package index
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"maps"
"reflect"
"slices"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/sdk/graph"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// The catalog graph (novox/hq ADR 0656; design 47, *Two graphs, one source of truth*): at each revision one entry per
// module, as its manifest declares it, in the seat mesh-catalog's layout (its definition's CatalogEntry), and the
// revision a list of its entries' digests, each entry kept once as an object named by its digest
// (catalog-revisions: trunk.<revision>, candidate.<group>.<heads>, entry.<digest>).
// CatalogEntryLayout is the layout version an entry and a revision are written in.
const CatalogEntryLayout = 1
// CatalogEntryOf is the catalog graph entry of one module: the module graph's node of it as a recompute computed it
// (its build, provision and verb edges, the verb edges left out on a cycle, ADR 0614), with where the trunk holds it,
// its manifest's digest and verdict, the main package its process runs (MainOf), the package versions it builds against
// and what the indexer computes its next revision from (Inputs: the lead's ruling on milestone 47/G finding 1), so that
// ModuleValueOf gives the module's value back from the entry alone. The acting rules are the revision's to add.
func CatalogEntryOf(module string, cur meshcatalog.ModulesByModuleValue, m catalogue.Manifest, n graph.Node) meshcatalog.CatalogEntry {
e := meshcatalog.CatalogEntry{V: CatalogEntryLayout, Module: module, Repository: cur.Repository, Path: cur.Path,
Commit: cur.Commit, Manifest: cur.Manifest, Verdict: cur.Verdict, Fp: n.FP, None: n.None, Bases: n.Bases,
Packages: maps.Clone(cur.Packages), Provides: n.Provides, Requires: n.Requires, Awaits: n.Awaits, Invokes: n.Invokes,
Serves: n.Serves, LeftOut: n.LeftOut, Main: MainOf(m)}
if n.FP == "" && n.None == "" {
// A module the recompute computed no node for (its manifest cannot be read) keeps what its value holds.
e.Fp, e.None = cur.Fp2, cur.None2
}
in := meshcatalog.EntryInputs{Toolchains: maps.Clone(cur.Toolchains), Paths: maps.Clone(cur.Paths), Problems: cur.Problems,
Warnings: cur.Warnings, Unknown: cur.Unknown, Src: cur.Src, FpSrc: cur.FpSrc}
if cur.Inputs2 != nil {
from := meshcatalog.FingerprintInputs(*cur.Inputs2)
in.From = &from
}
if !reflect.ValueOf(in).IsZero() {
e.Inputs = &in
}
for language, t := range n.Toolchains {
if e.Toolchains == nil {
e.Toolchains = map[string]meshcatalog.BuildEdge{}
}
e.Toolchains[language] = meshcatalog.BuildEdge{Module: t.Module, Version: t.Version, Recipe: t.Recipe}
}
return e
}
// ModuleValueOf is the module's value as the indexer computes with it, read back from its catalog graph entry: the
// inverse of CatalogEntryOf, by which the indexer's view of the trunk is the newest trunk revision's entries (the lead's
// ruling on milestone 47/G finding 1; ADR 0656 §6: one form of the fact). The old line's fields (the old-form
// fingerprint, the reader needs) are not kept.
func ModuleValueOf(e meshcatalog.CatalogEntry) meshcatalog.ModulesByModuleValue {
v := meshcatalog.ModulesByModuleValue{V: keys.Version, Repository: e.Repository, Path: e.Path, Commit: e.Commit, Manifest: e.Manifest,
Verdict: e.Verdict, Fp2: e.Fp, None2: e.None, Packages: maps.Clone(e.Packages)}
if in := e.Inputs; in != nil {
v.Toolchains, v.Paths, v.Problems, v.Warnings, v.Unknown = maps.Clone(in.Toolchains), maps.Clone(in.Paths), in.Problems, in.Warnings, in.Unknown
v.Src, v.FpSrc = in.Src, in.FpSrc
if in.From != nil {
from := meshcatalog.ModulesByModuleValueInputs2(*in.From)
v.Inputs2 = &from
}
}
return v
}
// NodeOf is the node of the module graph an entry carries, as CatalogEntryOf took it in: the graph the indexer holds in
// memory is read back from the newest trunk revision at its start (TrunkView), so a revision kept before its first
// recompute loses no edge.
func NodeOf(e meshcatalog.CatalogEntry) graph.Node {
n := graph.Node{V: graph.Version, FP: e.Fp, None: e.None, Repository: e.Repository, Path: e.Path, Bases: e.Bases,
Provides: e.Provides, Requires: e.Requires, Awaits: e.Awaits, Invokes: e.Invokes, Serves: e.Serves, LeftOut: e.LeftOut}
if len(e.Packages) > 0 {
n.Packages = slices.Sorted(maps.Keys(e.Packages))
}
for language, t := range e.Toolchains {
if n.Toolchains == nil {
n.Toolchains = map[string]graph.Toolchain{}
}
n.Toolchains[language] = graph.Toolchain{Module: t.Module, Version: t.Version, Recipe: t.Recipe}
}
return n
}
// EncodeEntry is an entry's bytes and their sha256 in hex, the name of its object: the same entry in two revisions is
// one object (design 47, *An unchanged entry is stored once*).
func EncodeEntry(e meshcatalog.CatalogEntry) (string, []byte, error) {
data, err := json.Marshal(e)
if err != nil {
return "", nil, fmt.Errorf("the entry of %s: %w", e.Module, err)
}
sum := sha256.Sum256(data)
return hex.EncodeToString(sum[:]), data, nil
}
// CatalogRevisionOf is a revision of entries, listed by module name with each entry's digest, and the entry objects it
// lists by digest, each once. The caller numbers the revision, names its kind (trunk or candidate) and what it was read
// from.
func CatalogRevisionOf(rev meshcatalog.CatalogRevision, entries map[string]meshcatalog.CatalogEntry) (meshcatalog.CatalogRevision, map[string][]byte, error) {
rev.V, rev.Entries = CatalogEntryLayout, make([]meshcatalog.EntryRef, 0, len(entries))
objects := map[string][]byte{}
for _, module := range slices.Sorted(maps.Keys(entries)) {
digest, data, err := EncodeEntry(entries[module])
if err != nil {
return meshcatalog.CatalogRevision{}, nil, err
}
objects[digest] = data
rev.Entries = append(rev.Entries, meshcatalog.EntryRef{Module: module, Digest: digest})
}
return rev, objects, nil
}
// MainOf is the Go main package the check's first tier builds when the module acts (the seat mesh-catalog's entry
// field main): the from of the first Go artifact a process resource of the manifest runs, as the manifest names it;
// empty when it runs none.
func MainOf(m catalogue.Manifest) string {
if m.Build == nil {
return ""
}
for _, r := range m.Resources {
if fmt.Sprint(r["type"]) != "process" {
continue
}
name := fmt.Sprint(r["artifact"])
for _, a := range m.Build.Artifacts {
if a.Name == name && a.Language == "go" && a.From != "" {
return a.From
}
}
}
return ""
}
@@ -0,0 +1,483 @@
package index
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strconv"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/sdk/graph"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/views"
)
// The indexer's object bucket catalog-revisions (design 47, Two graphs; novox/hq ADR 0619 §2: declared in the old
// `objects` section, its cap there): a revision is its list of entries' digests under trunk.<revision> or
// candidate.<group>.<heads>, and each entry one object entry.<digest>, written once and shared by every revision that
// lists it.
// CatalogRevisionsObjects is the bucket's local name in the indexer's manifest.
const CatalogRevisionsObjects = "catalog-revisions"
// ErrNoObject is an object the bucket does not hold.
var ErrNoObject = errors.New("no such object")
// Objects is an object bucket as the indexer keeps its revisions in it.
type Objects interface {
// Put stores data whole under name, replacing one of that name.
Put(ctx context.Context, name string, data []byte) error
// Get is the object's bytes; ErrNoObject when it holds none of that name.
Get(ctx context.Context, name string) ([]byte, error)
// Delete removes the object; one it does not hold is no error.
Delete(ctx context.Context, name string) error
// Names is every object's name; none is an empty list.
Names(ctx context.Context) ([]string, error)
// Fill is the bucket's bytes used and its cap in bytes.
Fill(ctx context.Context) (used, capacity int64, err error)
}
// EntryObject is the object name of the entry of that digest.
func EntryObject(digest string) string { return "entry." + digest }
// RevisionObject is the object name of a revision: trunk.<revision>, or candidate.<group>.<heads> for a candidate,
// which keeps its newest revision only.
func RevisionObject(rev meshcatalog.CatalogRevision) string {
if rev.Kind == "candidate" {
return "candidate." + rev.Group + "." + rev.Heads
}
return "trunk." + strconv.FormatInt(rev.Revision, 10)
}
// TrunkRevisionOf is the revision number a trunk object's name holds, and whether it is one.
func TrunkRevisionOf(name string) (int64, bool) {
n, ok := strings.CutPrefix(name, "trunk.")
if !ok {
return 0, false
}
r, err := strconv.ParseInt(n, 10, 64)
return r, err == nil
}
// KeepRevision stores a revision as CatalogRevisionOf made it: each entry object the bucket does not hold yet, then the
// revision's list, so a list is never kept whose entries are not; an entry already held is not written again.
func KeepRevision(ctx context.Context, o Objects, rev meshcatalog.CatalogRevision, entries map[string][]byte) error {
held, err := o.Names(ctx)
if err != nil {
return fmt.Errorf("the objects %s: %w", CatalogRevisionsObjects, err)
}
have := make(map[string]bool, len(held))
for _, n := range held {
have[n] = true
}
for _, ref := range rev.Entries {
name := EntryObject(ref.Digest)
if have[name] {
continue
}
data, ok := entries[ref.Digest]
if !ok {
return fmt.Errorf("revision %s lists the entry of %s (%s), which it does not carry", RevisionObject(rev), ref.Module, ref.Digest)
}
if err := o.Put(ctx, name, data); err != nil {
return fmt.Errorf("the entry of %s in %s: %w", ref.Module, CatalogRevisionsObjects, err)
}
have[name] = true
}
list, err := json.Marshal(rev)
if err != nil {
return fmt.Errorf("revision %s: %w", RevisionObject(rev), err)
}
if err := o.Put(ctx, RevisionObject(rev), list); err != nil {
return fmt.Errorf("revision %s in %s: %w", RevisionObject(rev), CatalogRevisionsObjects, err)
}
return nil
}
// ReadRevision is the revision kept under name and its entries whole, in the revision's order; an entry it lists that
// the bucket lost is an error naming it, never an entry left out.
func ReadRevision(ctx context.Context, o Objects, name string) (meshcatalog.CatalogRevision, []meshcatalog.CatalogEntry, error) {
var rev meshcatalog.CatalogRevision
data, err := o.Get(ctx, name)
if err != nil {
return rev, nil, fmt.Errorf("revision %s: %w", name, err)
}
if err := json.Unmarshal(data, &rev); err != nil {
return rev, nil, fmt.Errorf("revision %s: %w", name, err)
}
entries := make([]meshcatalog.CatalogEntry, 0, len(rev.Entries))
for _, ref := range rev.Entries {
data, err := o.Get(ctx, EntryObject(ref.Digest))
if err != nil {
return rev, nil, fmt.Errorf("revision %s, the entry of %s (%s): %w", name, ref.Module, ref.Digest, err)
}
var e meshcatalog.CatalogEntry
if err := json.Unmarshal(data, &e); err != nil {
return rev, nil, fmt.Errorf("revision %s, the entry of %s: %w", name, ref.Module, err)
}
entries = append(entries, e)
}
return rev, entries, nil
}
// newestTrunk is the newest trunk revision the bucket keeps, its list only; none is a zero revision and false.
func newestTrunk(ctx context.Context, o Objects) (meshcatalog.CatalogRevision, bool, error) {
names, err := o.Names(ctx)
if err != nil {
return meshcatalog.CatalogRevision{}, false, err
}
var newest int64
for _, n := range names {
if r, ok := TrunkRevisionOf(n); ok && r > newest {
newest = r
}
}
if newest == 0 {
return meshcatalog.CatalogRevision{}, false, nil
}
data, err := o.Get(ctx, "trunk."+strconv.FormatInt(newest, 10))
if err != nil {
return meshcatalog.CatalogRevision{}, false, err
}
var rev meshcatalog.CatalogRevision
if err := json.Unmarshal(data, &rev); err != nil {
return rev, false, fmt.Errorf("trunk.%d: %w", newest, err)
}
return rev, true, nil
}
// changedEntries are the modules whose entry differs between two revisions' lists, one gone or added included, by name.
func changedEntries(before, after []meshcatalog.EntryRef) []string {
was := map[string]string{}
for _, r := range before {
was[r.Module] = r.Digest
}
var changed []string
for _, r := range after {
if d, ok := was[r.Module]; !ok || d != r.Digest {
changed = append(changed, r.Module)
}
delete(was, r.Module)
}
for m := range was {
changed = append(changed, m)
}
slices.Sort(changed)
return changed
}
// keepTrunkRevision keeps the catalog graph revision a merge's or a recompute's module graph makes (design 47, Two
// graphs: a revision for each source revision taken in, and for a fingerprint moved with no merge), numbered after the
// newest kept, and says catalog-revision; a graph whose entries are the newest revision's makes none. A revision not
// kept is said in the log and returned, so the take-in that made it is not said taken.
func (ix *Indexer) keepTrunkRevision(ctx context.Context, edges map[string]graphEdges, whole graph.Graph) error {
if ix.Revisions == nil {
return nil
}
// Each entry is the module's value as this recompute left it in modules (its fingerprint, toolchains, package versions
// and inputs written), so the revision holds what the next one is computed from.
held, err := ix.heldModules(ctx)
if err != nil {
return err
}
// Every module the view holds is an entry, one the recompute computed no edges for (a manifest it cannot read) too.
entries := make(map[string]meshcatalog.CatalogEntry, len(held))
for name, cur := range held {
entries[name] = CatalogEntryOf(name, cur, edges[name].manifest, whole[name])
}
for name, e := range edges {
if _, ok := held[name]; !ok {
entries[name] = CatalogEntryOf(name, e.cur, e.manifest, whole[name])
}
}
return ix.keepEntries(ctx, entries)
}
// KeepView keeps the view of the trunk as a trunk revision after a write to it outside a recompute (the lead's ruling on
// milestone 47/G: the judge pass's verdicts, a build's reported imports, the source stamp, the modules the trunk no longer
// holds, the indexer's own entry a held revision writes): each entry is the module's value in the view with its node of
// the module graph in memory, so what the view holds is never lost at a restart. A view whose entries are the newest
// revision's keeps none.
func (ix *Indexer) KeepView(ctx context.Context) error {
if ix.Revisions == nil || ix.Modules == nil || ix.catchingUp.Load() {
return nil
}
held, err := ix.heldModules(ctx)
if err != nil {
return err
}
nodes := graph.Graph{}
if ix.Graph != nil {
keys, err := ix.Graph.Keys(ctx)
if err != nil && !errors.Is(err, process.ErrNotYet) {
return fmt.Errorf("the module graph: %w", err)
}
for _, k := range keys {
if value, _, found, err := ix.Graph.Get(ctx, k); err == nil && found {
if n, err := graph.Decode(value); err == nil {
nodes[k] = n
}
}
}
}
entries := make(map[string]meshcatalog.CatalogEntry, len(held))
for name, cur := range held {
var m catalogue.Manifest
if cur.Manifest != "" {
if _, read, err := ix.manifestRaw(ctx, cur.Manifest); err == nil {
m = read
}
}
entries[name] = CatalogEntryOf(name, cur, m, nodes[name])
}
return ix.keepEntries(ctx, entries)
}
// keepEntries keeps entries as the trunk revision after the newest, says catalog-revision, and keeps none when no entry
// moved; one keep at a time, so two never take the same number.
func (ix *Indexer) keepEntries(ctx context.Context, entries map[string]meshcatalog.CatalogEntry) error {
if ix.catchingUp.Load() {
return nil // the catch-up of a first start keeps its one revision at its end (CatchUp)
}
ix.keepMu.Lock()
defer ix.keepMu.Unlock()
ix.mu.Lock()
src := int64(ix.source.rev)
ix.mu.Unlock()
newest, any, err := newestTrunk(ctx, ix.Revisions)
if errors.Is(err, process.ErrNotYet) {
ix.logf("catalog graph: no revision is kept: its objects %s are not on the bus yet", CatalogRevisionsObjects)
return nil
}
if err != nil {
ix.logf("catalog graph: no revision is kept: the newest is not read: %v", err)
return fmt.Errorf("the newest catalog graph revision: %w", err)
}
at := ix.now().UTC()
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: newest.Revision + 1, Kind: "trunk", Source: src, At: &at}, entries)
if err != nil {
return err
}
changed := changedEntries(newest.Entries, rev.Entries)
if (any && len(changed) == 0) || (!any && len(entries) == 0) {
return nil // nothing moved; an empty first revision is never kept, so the first start's catch-up still runs
}
used, capacity, fillErr := ix.Revisions.Fill(ctx)
if fillErr == nil && capacity > 0 && used+newBytes(ctx, ix.Revisions, rev, objects) > capacity {
// W4.6 (design 47, Two graphs): a full store refuses the next revision, said once per source revision, and the
// take-in that made it holds its pending merge with the reason objects full until room is said.
refused := fmt.Sprintf("the next revision; source revision %d held pending with the reason %s", src, HeldObjectsFull)
ix.logf("catalog graph: revision %d of source revision %d is not kept: %s is full (%d of %d bytes)", rev.Revision, src, CatalogRevisionsObjects, used, capacity)
ix.sayFill(ctx, meshcatalog.EventFull, fmt.Sprintf("%d", src), meshcatalog.Fill{Bytes: used, Cap: capacity, Refused: refused})
return fmt.Errorf("revision %d of source revision %d: %w", rev.Revision, src, ErrObjectsFull)
}
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
ix.logf("catalog graph: revision %d of source revision %d is not kept: %v", rev.Revision, src, err)
return err
}
if after, capacity, err := ix.Revisions.Fill(ctx); fillErr == nil && err == nil && capacity > 0 && used*5 < capacity*4 && after*5 >= capacity*4 {
ix.sayFill(ctx, meshcatalog.EventFilling, fmt.Sprintf("%d", rev.Revision), meshcatalog.Fill{Bytes: after, Cap: capacity})
}
ix.logf("catalog graph: revision %d of source revision %d kept, %d entries, %d changed", rev.Revision, src, len(rev.Entries), len(changed))
if ix.SayCatalogRevision != nil {
if err := ix.SayCatalogRevision(ctx, meshcatalog.CatalogRevisionEvent{V: CatalogEntryLayout, Revision: rev.Revision, Kind: rev.Kind,
Source: src, At: &at, Changed: changed, Entries: int64(len(rev.Entries))}); err != nil {
ix.logf("catalog graph: revision %d is not said: %v", rev.Revision, err)
}
}
return nil
}
// HeldObjectsFull is the reason a pending merge is held while the catalog graph revisions are full (Pending.Held).
const HeldObjectsFull = "objects full"
// ErrObjectsFull is a revision refused because the catalog graph revisions are at their cap.
var ErrObjectsFull = errors.New("the catalog graph revisions are full")
// newBytes is the size of what keeping rev adds: its list and each entry the bucket does not hold yet.
func newBytes(ctx context.Context, o Objects, rev meshcatalog.CatalogRevision, entries map[string][]byte) int64 {
held, _ := o.Names(ctx)
have := make(map[string]bool, len(held))
for _, n := range held {
have[n] = true
}
list, _ := json.Marshal(rev)
size := int64(len(list))
for _, ref := range rev.Entries {
if !have[EntryObject(ref.Digest)] {
size += int64(len(entries[ref.Digest]))
}
}
return size
}
// sayFill says the seat's event filling, full or room of the catalog graph revisions, under the duplicate id of the
// event and what it is about, so the same one said twice is heard once.
func (ix *Indexer) sayFill(ctx context.Context, event, about string, f meshcatalog.Fill) {
if ix.SayFill == nil {
return
}
at := ix.now().UTC()
f.At, f.Store = &at, CatalogRevisionsObjects
if err := ix.SayFill(ctx, event, event+"-"+about, f); err != nil {
ix.logf("catalog graph: %s is not said: %v", event, err)
}
}
// RoomAgain says room and evaluates pending again when a merge held with the reason objects full finds the store
// below its filling mark (80 % of its cap): called after anything removes catalog graph revisions (no timer). It answers whether room was said.
func (ix *Indexer) RoomAgain(ctx context.Context) (bool, error) {
if ix.Revisions == nil || ix.Pending == nil {
return false, nil
}
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
if err != nil || !slices.ContainsFunc(entries, func(e PendingEntry) bool { return e.Value.Held == HeldObjectsFull }) {
return false, err
}
used, capacity, err := ix.Revisions.Fill(ctx)
if err != nil || (capacity > 0 && used*5 >= capacity*4) {
return false, err // room is said below the filling mark, 80 %, so one refusal is not followed by another at once
}
ix.sayFill(ctx, meshcatalog.EventRoom, fmt.Sprintf("%d", used), meshcatalog.Fill{Bytes: used, Cap: capacity})
return true, ix.PendingAgain(ctx) // room in catalog-revisions: pending evaluated again
}
// TrunkView is the indexer's view of the trunk read from the newest trunk revision the bucket keeps (the lead's ruling
// on milestone 47/G finding 1; ADR 0656 §6, one form of the fact): a store in memory holding each module's value as
// ModuleValueOf reads it back from its entry, under its modules key, a store in memory of each module's node of the module
// graph (NodeOf), and the revision it was read at (0 when the bucket keeps none yet: both start empty and the first start's
// catch-up fills them). A lost entry is an error naming the module (ReadRevision), never a module left out of the view.
func TrunkView(ctx context.Context, o Objects) (modules, nodes Store, at int64, err error) {
view, held := newOverlay(nil), newOverlay(nil)
newest, any, err := newestTrunk(ctx, o)
if err != nil || !any {
return view, held, 0, err
}
_, entries, err := ReadRevision(ctx, o, RevisionObject(newest))
if err != nil {
return nil, nil, 0, err
}
for _, e := range entries {
key, err := meshcatalogclient.ModulesKey(e.Module)
if err != nil {
return nil, nil, 0, fmt.Errorf("trunk.%d: the entry of %q: %w", newest.Revision, e.Module, err)
}
data, err := meshcatalogclient.EncodeModule(ModuleValueOf(e))
if err != nil {
return nil, nil, 0, fmt.Errorf("trunk.%d: the entry of %s: %w", newest.Revision, e.Module, err)
}
if err := view.Create(ctx, key, data); err != nil {
return nil, nil, 0, fmt.Errorf("trunk.%d: the entry of %s: %w", newest.Revision, e.Module, err)
}
gk, err := graph.Key(e.Module)
if err != nil {
continue // a module the graph cannot name has no node, as writeGraph writes none
}
node, err := graph.Encode(NodeOf(e))
if err != nil {
return nil, nil, 0, fmt.Errorf("trunk.%d: the node of %s: %w", newest.Revision, e.Module, err)
}
if err := held.Create(ctx, gk, node); err != nil {
return nil, nil, 0, fmt.Errorf("trunk.%d: the node of %s: %w", newest.Revision, e.Module, err)
}
}
return view, held, newest.Revision, nil
}
// MemoryStore is a store held in memory alone: the module graph's nodes, which each revision's entries carry, so the
// indexer keeps no state of them on the bus (milestone 47/G).
func MemoryStore() Store { return newOverlay(nil) }
// The view of the trunk is the newest catalog graph revision the indexer's objects catalog-revisions keep (milestone
// 47/G, ADR 0656 §6): the tools bundle answers the modules and the module graph it once read from the states modules and
// graph from that revision, at its number, and says it waits while the indexer keeps none yet (ADR 0615 §6), never an
// empty catalog.
// ErrNoRevision is a view asked before the indexer kept its first trunk revision.
var ErrNoRevision = errors.New("view-waits: the indexer keeps no catalog graph revision yet (its first start takes the trunk in); ask again on catalog-revision")
// TrunkStates reads the view of the trunk from the newest trunk revision; under answers every other state.
type TrunkStates struct {
Objects Objects
Under func(string) views.Source
}
// read is the view's modules and nodes, and the revision they were read at; none kept yet is ErrNoRevision.
func (t TrunkStates) read(nodes bool) (Store, int64, error) {
mods, held, at, err := TrunkView(context.Background(), t.Objects)
if err == nil && at == 0 {
err = ErrNoRevision
}
if nodes {
return held, at, err
}
return mods, at, err
}
// state is the source of a state by its name: modules and graph from the revision, their marks at its number, every
// other the indexer's own state.
func (t TrunkStates) State(name string) views.Source {
switch name {
case "modules", graph.State:
return revisionSource{t: t, nodes: name == graph.State}
case views.MarksState:
return markSource{t: t, under: t.Under(name)}
}
return t.Under(name)
}
// revisionSource is the modules or the module graph of the newest trunk revision, each value at its number.
type revisionSource struct {
t TrunkStates
nodes bool
}
func (r revisionSource) Keys() ([]string, error) {
s, _, err := r.t.read(r.nodes)
if err != nil {
return nil, err
}
return s.Keys(context.Background())
}
func (r revisionSource) Get(key string) (*views.Entry, bool, error) {
s, at, err := r.t.read(r.nodes)
if err != nil {
return nil, false, err
}
value, _, found, err := s.Get(context.Background(), key)
if err != nil || !found {
return nil, false, err
}
return &views.Entry{Value: value, Revision: uint64(at)}, true, nil
}
// markSource answers the mark of modules and graph as the newest trunk revision's number, which catalog-revision says
// the changes after; every other mark is the views family's.
type markSource struct {
t TrunkStates
under views.Source
}
func (m markSource) Keys() ([]string, error) { return m.under.Keys() }
func (m markSource) Get(state string) (*views.Entry, bool, error) {
if state != "modules" && state != graph.State {
return m.under.Get(state)
}
_, at, err := m.t.read(false)
if err != nil {
return nil, false, err
}
data, err := json.Marshal(views.Mark{V: views.Version, State: state, Revision: uint64(at), At: time.Now().UTC()})
if err != nil {
return nil, false, err
}
return &views.Entry{Value: data, Revision: uint64(at)}, true, nil
}
@@ -81,22 +81,7 @@ func (ix *Indexer) writeGraph(ctx context.Context, edges map[string]graphEdges)
GraphMost, len(names), names[GraphMost], len(names)-GraphMost-1)
names = names[:GraphMost]
}
providers := func(provision string) []catalogue.Manifest {
var out []catalogue.Manifest
for _, name := range slices.Sorted(maps.Keys(edges)) {
if slices.Contains(edges[name].manifest.Offers(), provision) {
out = append(out, edges[name].manifest)
}
}
return out
}
// The verb edges on a cycle are left out over the whole graph (ADR 0614 §3), every module it knows, so a module past
// GraphMost still closes the cycle it lies on.
whole := graph.Graph{}
for name, e := range edges {
whole[name] = graphNode(e, providers)
}
whole.LeaveOut()
whole := wholeGraph(edges)
for _, name := range names {
key, err := graph.Key(name)
if err != nil {
@@ -141,3 +126,23 @@ func (ix *Indexer) writeGraph(ctx context.Context, edges map[string]graphEdges)
}
return nil
}
// wholeGraph is the module graph of every module of edges, the verb edges on a cycle left out over the whole graph
// (ADR 0614 §3), so a module past GraphMost still closes the cycle it lies on.
func wholeGraph(edges map[string]graphEdges) graph.Graph {
providers := func(provision string) []catalogue.Manifest {
var out []catalogue.Manifest
for _, name := range slices.Sorted(maps.Keys(edges)) {
if slices.Contains(edges[name].manifest.Offers(), provision) {
out = append(out, edges[name].manifest)
}
}
return out
}
whole := graph.Graph{}
for name, e := range edges {
whole[name] = graphNode(e, providers)
}
whole.LeaveOut()
return whole
}
@@ -1,6 +1,12 @@
package index
import "encoding/json"
import (
"context"
"encoding/json"
"maps"
"slices"
"sync"
)
// rawList is the list a kept predefined value holds, as it was written.
func rawList(value []byte) json.RawMessage {
@@ -10,3 +16,69 @@ func rawList(value []byte) json.RawMessage {
_ = json.Unmarshal(value, &kept)
return kept.List
}
// memObjects is an object bucket in memory with a cap: a put past the cap is refused as the bus's DiscardNew refuses it.
type memObjects struct {
mu sync.Mutex
objs map[string][]byte
puts int
capacity int64
}
func newMemObjects(capacity int64) *memObjects {
return &memObjects{objs: map[string][]byte{}, capacity: capacity}
}
func (m *memObjects) used() int64 {
var n int64
for _, d := range m.objs {
n += int64(len(d))
}
return n
}
func (m *memObjects) Put(_ context.Context, name string, data []byte) error {
m.mu.Lock()
defer m.mu.Unlock()
if m.capacity > 0 && m.used()-int64(len(m.objs[name]))+int64(len(data)) > m.capacity {
return errObjectsFullForTest
}
m.objs[name] = slices.Clone(data)
m.puts++
return nil
}
func (m *memObjects) Get(_ context.Context, name string) ([]byte, error) {
m.mu.Lock()
defer m.mu.Unlock()
d, ok := m.objs[name]
if !ok {
return nil, ErrNoObject
}
return slices.Clone(d), nil
}
func (m *memObjects) Delete(_ context.Context, name string) error {
m.mu.Lock()
defer m.mu.Unlock()
delete(m.objs, name)
return nil
}
func (m *memObjects) Names(context.Context) ([]string, error) {
m.mu.Lock()
defer m.mu.Unlock()
return slices.Sorted(maps.Keys(m.objs)), nil
}
func (m *memObjects) Fill(context.Context) (int64, int64, error) {
m.mu.Lock()
defer m.mu.Unlock()
return m.used(), m.capacity, nil
}
var errObjectsFullForTest = &objectsFullError{}
type objectsFullError struct{}
func (*objectsFullError) Error() string { return "maximum bytes exceeded" }
@@ -27,6 +27,13 @@ import (
// Imports that cannot be resolved (a package the forge cannot list) are kept with no ImportForm: the next recompute that
// resolves them writes the import form, one rebuild too many, never a stale module.
func (ix *Indexer) ApplyBuildEnd(ctx context.Context, value []byte) error {
if err := ix.applyBuildEnd(ctx, value); err != nil {
return err
}
return ix.KeepView(ctx) // the imports kept with the view (KeepView), so a restart reads them back
}
func (ix *Indexer) applyBuildEnd(ctx context.Context, value []byte) error {
if !builds.IsBuildEntry(value) {
return nil
}
@@ -292,6 +292,13 @@ func (ix *Indexer) dropRemoved(ctx context.Context, repository string, held map[
// message (applyMerged). A repository whose trunk cannot be read, or reads as holding no module at all, is left as it is
// and said: a failure never empties the bucket.
func (ix *Indexer) Reconcile(ctx context.Context) error {
if err := ix.reconcile(ctx); err != nil {
return err
}
return ix.KeepView(ctx) // the modules dropped leave the view's next revision too
}
func (ix *Indexer) reconcile(ctx context.Context) error {
if ix.Modules == nil || ix.Trunk == nil {
return nil
}
@@ -1,7 +1,5 @@
package index
import "git.novox.be/novox/mesh-delivery/sdk/graph"
// The holders, states and buckets the indexer names, taken from its own manifest (module.json: its name, its `state`, its
// `reads`) and from the manifests of the holders it reads; a unit row checks them against the manifest
// (TestTheNamesAreTheManifests). No contract names a holder (novox/hq ADR 0625 §4, ADR 0634 §1).
@@ -14,13 +12,9 @@ const (
CandidatesBucket = Module + "_" + CandidatesState
CandidatesHistory = 10
ModulesState = "modules"
ModulesBucket = Module + "_" + ModulesState
ModulesHistory = 20
ManifestsState = "manifests"
ManifestsBucket = Module + "_" + ManifestsState
ManifestsHistory = 1
// GraphBucket is the bucket of the state graph (sdk/graph names the state, not its holder).
GraphBucket = Module + "_" + graph.State
// VerdictsState and VerdictsBucket are in verdicts.go.
VerdictsHistory = 5
PredefinedState = "predefined"
@@ -95,6 +95,9 @@ type Pending struct {
// UnreadSince is when the message was first found unread: past meshconditionsclient.CallUnansweredAfter, with the forge
// heard since, it is the finding call-unanswered (unreadcall.go, ADR 0668 §3).
UnreadSince time.Time `json:"unread-since,omitzero"`
// Held is why a ready merge is not taken in although its revision names this build: objects full (HeldObjectsFull),
// until room is said (RoomAgain).
Held string `json:"held,omitempty"`
}
// PendingEntry is one key of the pending state as read: its key, revision, value and the merged message decoded. For a
@@ -782,11 +785,32 @@ func (ix *Indexer) evaluate(ctx context.Context, own string, start bool) (Applie
if errors.Is(err, errVerdictsNotYet) {
return a, "the verdicts state is not on the bus yet: the revision waits for it, taken in when it binds", nil
}
if errors.Is(err, ErrObjectsFull) {
ix.holdFull(ctx, ready)
return a, ix.waitingLine(w) + "; held: " + HeldObjectsFull, nil
}
return a, ix.waitingLine(w), err
}
return Applied{}, ix.waitingLine(w), ix.holdFor(ctx, ready, names, own, dry)
}
// holdFull marks the ready merges held with the reason objects full (Pending.Held), read again at their revisions now
// (the take-in's claim moved them), so indexer_pending shows why they wait.
func (ix *Indexer) holdFull(ctx context.Context, ready []PendingEntry) {
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
if err != nil {
ix.logf("take-in: the merges held by a full %s are not marked: %v", CatalogRevisionsObjects, err)
return
}
var held []PendingEntry
for _, e := range entries {
if slices.ContainsFunc(ready, func(r PendingEntry) bool { return r.Key == e.Key }) && e.Value.Held != HeldObjectsFull {
held = append(held, e)
}
}
ix.markSaid(ctx, held, nil, func(p *Pending) { p.Held = HeldObjectsFull })
}
// holdLost holds a revision whose newest merge of a repository cannot be read (the forge bridge's stream no longer holds
// it, or the bus refuses the read): said once on take-in.pending with the lost keys (TakeIn.Lost), and on each lost entry
// (Pending.Lost, the reason, which indexer_pending shows), until a newer merge of its repository covers it or
@@ -884,7 +908,9 @@ func (ix *Indexer) dry() *dryRun {
// requirements, findings, seat definitions, holds and their bounds are not overlaid, nothing is said, and no hold is
// weighed (Holds and Counted nil hold nothing); ReadFile only reads.
Requirements: nil, Findings: nil, SeatDefinitions: nil, ReadFile: ix.ReadFile, SaySeatDefinition: nil,
Holds: nil, Counted: nil, AskEngines: nil, HoldBounds: nil, OwnBuildState: nil, underSeats: true}
Holds: nil, Counted: nil, AskEngines: nil, HoldBounds: nil, OwnBuildState: nil, underSeats: true,
// A dry run keeps no catalog graph revision (catalog_revisions.go): its revision is the take-in's to keep.
Revisions: nil, SayCatalogRevision: nil, SayFill: nil}
x.Logf = func(format string, args ...any) { ix.logf("take-in, computed and not written: "+format, args...) }
x.Store, d.candidates = over(ix.Store)
x.Modules, d.modules = over(ix.Modules)
@@ -1187,6 +1213,11 @@ func (ix *Indexer) holdFor(ctx context.Context, ready []PendingEntry, names, own
}
}
}
// The indexer's own entry the held revision names is kept as the trunk revision after the newest, every other entry as
// it was (milestone 47/G W4.7: a merged revision naming a newer indexer replaces only the indexer's entry).
if err := ix.KeepView(ctx); err != nil {
return fmt.Errorf("the revision with the indexer's own entry: %w", err)
}
t := revisionWait(takein.TakeInPending, ready, names)
t.Refused = refused
var ended, unmarked []PendingEntry
@@ -18,6 +18,7 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-delivery/sdk/trunk"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/busretry"
"git.novox.be/novox/mesh-sdk/go/catalogue"
@@ -744,6 +745,141 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
}
}
})
t.Run("W4.6 a full store holds the merge with the reason objects full, says full once, and room takes it in", func(t *testing.T) {
b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
named := b.fp(t, Module)
b.ix.ownMu.Lock()
b.ix.ownFP = named
b.ix.ownMu.Unlock()
store := newMemObjects(100) // 90 bytes held by another revision: full for this one
_ = store.Put(ctx, "trunk.0", make([]byte, 90))
b.ix.Revisions = store
var said []string
b.ix.SayFill = func(_ context.Context, event, id string, f meshcatalog.Fill) error {
said = append(said, event+" "+f.Refused)
return nil
}
if _, _, err := b.ix.TakeInAtStart(ctx); err != nil {
t.Fatal(err)
}
var p Pending
_ = json.Unmarshal(b.pending.vals[b.keys()[0]], &p)
if len(b.keys()) != 1 || p.Held != HeldObjectsFull || len(said) != 1 || !strings.HasPrefix(said[0], "full ") || !strings.Contains(said[0], HeldObjectsFull) {
t.Fatalf("pending %v held %q, said %q: want the merge held with %q and full said", b.keys(), p.Held, said, HeldObjectsFull)
}
if roomed, err := b.ix.RoomAgain(ctx); err != nil || roomed {
t.Fatalf("room while still full: %v %v", roomed, err)
}
_ = store.Delete(ctx, "trunk.0") // the other revision removed: room
store.capacity = 0
if roomed, err := b.ix.RoomAgain(ctx); err != nil || !roomed || len(b.keys()) != 0 || said[len(said)-1] != "room " {
t.Errorf("room said %v (%v), said %q, pending %v: want room said and the merge taken in", roomed, err, said, b.keys())
}
// The same take-in into a store it fills to 85 % of its cap says filling once, and is taken in.
kept := store.used()
b2 := newTakeInBed(t)
b2.arrive(t, 2, b2.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b2.ix.ownMu.Lock()
b2.ix.ownFP = b2.fp(t, Module)
b2.ix.ownMu.Unlock()
b2.ix.Revisions = newMemObjects(kept * 100 / 85)
var filling []meshcatalog.Fill
b2.ix.SayFill = func(_ context.Context, event, _ string, f meshcatalog.Fill) error {
if event == meshcatalog.EventFilling {
filling = append(filling, f)
}
return nil
}
if _, _, err := b2.ix.TakeInAtStart(ctx); err != nil || len(b2.keys()) != 0 || len(filling) != 1 || filling[0].Store != CatalogRevisionsObjects {
t.Errorf("a take-in to 85 %% said filling %+v (%v), pending %v: want it said once and the merge taken in", filling, err, b2.keys())
}
})
t.Run("W4.12 a candidate's half dropped by hand frees its own entries, says room and takes the held merge in; the read tools list it; a recompute that moves nothing is equal", func(t *testing.T) {
bed := func(store *memObjects) *takeInBed {
b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.ix.ownMu.Lock()
b.ix.ownFP = b.fp(t, Module)
b.ix.ownMu.Unlock()
b.ix.Revisions = store
return b
}
free := newMemObjects(0)
if _, _, err := bed(free).ix.TakeInAtStart(ctx); err != nil {
t.Fatal(err)
}
size := free.used() // what the merge's revision takes
store := newMemObjects(2 * size)
shared := []byte(`{"module":"b"}`)
if err := KeepRevision(ctx, store, meshcatalog.CatalogRevision{Kind: "candidate", Group: "g1", Heads: "h1", Revision: 1,
Entries: []meshcatalog.EntryRef{{Module: "a", Digest: "da"}, {Module: "b", Digest: "db"}}},
map[string][]byte{"da": make([]byte, size+10), "db": shared}); err != nil {
t.Fatal(err)
}
if err := KeepRevision(ctx, store, meshcatalog.CatalogRevision{Kind: "candidate", Group: "g2", Heads: "h2", Revision: 1,
Entries: []meshcatalog.EntryRef{{Module: "b", Digest: "db"}}}, map[string][]byte{"db": shared}); err != nil {
t.Fatal(err)
}
b := bed(store)
var said []string
b.ix.SayFill = func(_ context.Context, event, _ string, _ meshcatalog.Fill) error { said = append(said, event); return nil }
if _, _, err := b.ix.TakeInAtStart(ctx); err != nil || len(b.keys()) != 1 {
t.Fatalf("pending %v (%v): want the merge held for objects full", b.keys(), err)
}
var recorded []string
asks := 0
tools := map[string]stdio.Tool{}
for _, tool := range CatalogRevisionTools(store, func(_ string, value []byte) (TakeInAsk, error) {
asks++
a, err := DecodeTakeIn(value)
if err != nil {
return a, err
}
var r TakenIn
if a.Recompute {
r = b.ix.RecomputeByHand(ctx, a.By, a.Why)
} else {
r = b.ix.DropCandidateGraph(ctx, a.CandidateGroup, a.CandidateHeads, a.By, a.Why)
}
a.Result = &r
return a, nil
}, time.Now, func(what, why string) error {
recorded = append(recorded, what)
if why == "unrecorded" {
return errors.New("mesh-hand-acts did not answer")
}
return nil
}) {
tools[tool.Name] = tool
}
if _, err := tools[DropCandidateGraphTool].Run(map[string]any{"group": "g1", "heads": "h1", "why": "unrecorded"}); err == nil || asks != 0 {
t.Fatalf("a drop whose hand act is not recorded: %v, %d asks: want refused before it is asked", err, asks)
}
got, err := tools[DropCandidateGraphTool].Run(map[string]any{"group": "g1", "heads": "h1", "why": "its gate event never came"})
r, _ := got.(*TakenIn)
if err != nil || r == nil || !r.Stored || !slices.Equal(r.Dropped, []string{"entry.da"}) || len(recorded) != 2 {
t.Fatalf("drop answered %+v (%v), recorded %q: want candidate.g1.h1 dropped with entry.da alone, after its hand act", r, err, recorded)
}
if _, err := store.Get(ctx, "entry.db"); err != nil || len(b.keys()) != 0 || !slices.Contains(said, meshcatalog.EventRoom) {
t.Errorf("entry.db %v, pending %v, said %q: want the shared entry kept, room said and the merge taken in", err, b.keys(), said)
}
list, err := tools[CatalogRevisionsTool].Run(map[string]any{})
l, _ := list.(RevisionsAnswer)
if err != nil || len(l.Revisions) != 2 || l.Revisions[0].Name != "candidate.g2.h2" || l.Revisions[1].Name != "trunk.1" || !l.Revisions[1].Current ||
l.Fill.Cap != 2*size || l.Fill.Bytes != store.used() || len(l.Unlisted) != 0 {
t.Errorf("indexer_catalog_revisions = %+v (%v): want candidate.g2.h2 and the current trunk.1, the fill against its cap", l, err)
}
if c, err := tools[CandidateGraphsTool].Run(map[string]any{}); err != nil || len(c.(RevisionsAnswer).Revisions) != 1 {
t.Errorf("indexer_candidate_graphs = %+v (%v): want g2's alone", c, err)
}
if one, err := tools[CatalogRevisionTool].Run(map[string]any{}); err != nil || one.(map[string]any)["name"] != "trunk.1" {
t.Errorf("indexer_catalog_revision = %v (%v): want the newest trunk revision whole", one, err)
}
if got, err := tools[RecomputeTool].Run(map[string]any{"why": "a check"}); err != nil || !strings.Contains(got.(*TakenIn).Answer, "equal to the kept revision 1") {
t.Errorf("indexer_recompute = %+v (%v): want equal to the kept revision 1", got, err)
}
})
t.Run("T7 a toolchain report evaluates pending again: the revision then names the running build and is taken in", func(t *testing.T) {
b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
@@ -970,7 +1106,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
for _, f := range []string{"Store", "Modules", "Manifests", "Predefined", "ConfigCandidates", "Taken", "Verdicts", "OldSections", "Unjudged",
"JudgeID", "SayRefused", "SayAccepted", "Packages", "Graph", "Files", "GoSum", "GoMod", "Now", "Builds", "Named", "Trunk",
"Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn",
"Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "Holds", "Counted", "AskEngines", "HoldBounds", "OpenTries", "TryRecovery", "OwnBuildState"} {
"Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "Holds", "Counted", "AskEngines", "HoldBounds", "OpenTries", "TryRecovery", "OwnBuildState", "Revisions", "SayCatalogRevision", "SayFill"} {
named[f] = true
}
ty := reflect.TypeOf(Indexer{})
@@ -220,6 +220,8 @@ type TakenIn struct {
// and deleted.
Handed []string `json:"handed,omitempty"`
Dropped []string `json:"dropped,omitempty"`
// Unread are the repositories a catch-up (CatchUp) could not take in: their merged message was not read.
Unread []string `json:"unread,omitempty"`
// Fingerprints are the modules whose fingerprint a recompute by hand moved (FingerprintsAgain), each old and new.
Fingerprints []FingerprintMove `json:"fingerprints,omitempty"`
}
@@ -0,0 +1,326 @@
package index
import (
"context"
"encoding/json"
"errors"
"fmt"
"regexp"
"slices"
"strconv"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// The indexer's tools of its catalog graph revisions (design 47, the indexer's tools; the instruction "tools are how a
// module is spoken to"): indexer_catalog_revision and indexer_catalog_revisions read the objects catalog-revisions,
// indexer_candidate_graphs every candidate's catalog half; indexer_recompute and indexer_drop_candidate_graph steer the
// indexer by an ask in takeins, as indexer_take_in does, since the indexer's process serves no request subject.
const (
CatalogRevisionTool = "indexer_catalog_revision"
CatalogRevisionsTool = "indexer_catalog_revisions"
CandidateGraphsTool = "indexer_candidate_graphs"
RecomputeTool = "indexer_recompute"
DropCandidateGraphTool = "indexer_drop_candidate_graph"
)
// RecomputeKey is the takeins key of a recompute by hand: one at a time.
const RecomputeKey = "recompute"
var candidateToken = regexp.MustCompile(`^[A-Za-z0-9_=-]+$`)
// NewRecomputeAsk is the value indexer_recompute writes: its key in takeins and the value.
func NewRecomputeAsk(why, by string, at time.Time) (string, []byte, error) {
data, err := json.Marshal(TakeInAsk{V: 1, Recompute: true, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
return RecomputeKey, data, err
}
// NewDropCandidateGraphAsk is the value indexer_drop_candidate_graph writes: its key in takeins,
// candidate-graph.<group>.<heads>, and the value.
func NewDropCandidateGraphAsk(group, heads, why, by string, at time.Time) (string, []byte, error) {
group, heads = strings.TrimSpace(group), strings.TrimSpace(heads)
switch {
case !candidateToken.MatchString(group) || !candidateToken.MatchString(heads):
return "", nil, errors.New("name the candidate by its group and its heads, as indexer_candidate_graphs names them")
case strings.TrimSpace(why) == "":
return "", nil, errors.New("say why (why): a drop is a hand act, recorded through mesh-hand-acts")
}
data, err := json.Marshal(TakeInAsk{V: 1, CandidateGroup: group, CandidateHeads: heads, Drop: true, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
return "candidate-graph." + group + "." + heads, data, err
}
// RevisionSummary is one kept catalog graph revision as indexer_catalog_revisions and indexer_candidate_graphs list it.
type RevisionSummary struct {
Name string `json:"name"`
Kind string `json:"kind"`
Revision int64 `json:"revision"`
Source int64 `json:"source"`
Group string `json:"group,omitempty"`
Heads string `json:"heads,omitempty"`
At *time.Time `json:"at,omitempty"`
Entries int `json:"entries"`
// Current says it is the trunk's newest revision, which is kept whatever pins it.
Current bool `json:"current,omitempty"`
}
// RevisionsAnswer is what indexer_catalog_revisions answers: every kept revision, the entries stored once by digest, the
// store's fill against its cap, and each object it could not read.
type RevisionsAnswer struct {
Revisions []RevisionSummary `json:"revisions"`
Entries int `json:"entries"`
Unlisted []string `json:"unlisted,omitempty"`
Fill meshcatalog.Fill `json:"fill"`
FillUnread string `json:"fill-unread,omitempty"`
Unread []string `json:"unread,omitempty"`
}
// ListRevisions reads every revision the objects keep, candidates included (kind "candidate" alone when candidates).
func ListRevisions(ctx context.Context, o Objects, candidates bool) (RevisionsAnswer, error) {
out := RevisionsAnswer{Revisions: []RevisionSummary{}}
names, err := o.Names(ctx)
if err != nil {
return out, fmt.Errorf("the objects %s cannot be listed: %w", CatalogRevisionsObjects, err)
}
listed := map[string]bool{}
var newest int64
entries := []string{}
for _, n := range names {
if strings.HasPrefix(n, "entry.") {
entries = append(entries, n)
continue
}
data, err := o.Get(ctx, n)
var rev meshcatalog.CatalogRevision
if err == nil {
err = json.Unmarshal(data, &rev)
}
if err != nil {
out.Unread = append(out.Unread, n+": "+err.Error())
continue
}
for _, ref := range rev.Entries {
listed[EntryObject(ref.Digest)] = true
}
if rev.Kind == "trunk" && rev.Revision > newest {
newest = rev.Revision
}
if candidates && rev.Kind != "candidate" {
continue
}
out.Revisions = append(out.Revisions, RevisionSummary{Name: n, Kind: rev.Kind, Revision: rev.Revision, Source: rev.Source,
Group: rev.Group, Heads: rev.Heads, At: rev.At, Entries: len(rev.Entries)})
}
for i := range out.Revisions {
out.Revisions[i].Current = out.Revisions[i].Kind == "trunk" && out.Revisions[i].Revision == newest
}
slices.SortFunc(out.Revisions, func(a, b RevisionSummary) int { return strings.Compare(a.Name, b.Name) })
out.Entries = len(entries)
for _, e := range entries {
if !listed[e] {
out.Unlisted = append(out.Unlisted, e)
}
}
slices.Sort(out.Unlisted)
used, capacity, err := o.Fill(ctx)
out.Fill = meshcatalog.Fill{Bytes: used, Cap: capacity, Store: CatalogRevisionsObjects}
if err != nil {
out.FillUnread = err.Error()
}
return out, nil
}
// removeUnlisted deletes every entry no kept revision lists any more: an entry is stored once and shared, so it goes
// only with the last revision that lists it. It answers the entries removed.
func removeUnlisted(ctx context.Context, o Objects) ([]string, error) {
list, err := ListRevisions(ctx, o, false)
if err != nil {
return nil, err
}
if len(list.Unread) > 0 {
// A revision that cannot be read may list any entry: none is removed rather than one it still needs.
return nil, fmt.Errorf("no entry is removed while a revision cannot be read: %s", strings.Join(list.Unread, "; "))
}
var removed []string
for _, e := range list.Unlisted {
if err := o.Delete(ctx, e); err != nil {
return removed, fmt.Errorf("%s: %w", e, err)
}
removed = append(removed, e)
}
return removed, nil
}
// DropCandidateGraph drops a candidate's catalog half by hand (indexer_drop_candidate_graph, design 47: for a candidate
// whose gate event never came), the entries no other revision lists with it, and says room when that takes the store
// below its filling mark, which takes in a merge held for objects full (RoomAgain).
func (ix *Indexer) DropCandidateGraph(ctx context.Context, group, heads, by, why string) TakenIn {
name := RevisionObject(meshcatalog.CatalogRevision{Kind: "candidate", Group: group, Heads: heads})
r := TakenIn{Module: name}
if ix.Revisions == nil {
r.Answer = "the indexer keeps no catalog graph revisions: nothing dropped"
return r
}
if _, err := ix.Revisions.Get(ctx, name); err != nil {
r.Answer = name + " is not kept (" + err.Error() + "): nothing dropped"
return r
}
ix.keepMu.Lock()
err := ix.Revisions.Delete(ctx, name)
var removed []string
if err == nil {
removed, err = removeUnlisted(ctx, ix.Revisions)
}
ix.keepMu.Unlock()
if err != nil {
r.Answer = name + " could not be dropped whole: " + err.Error()
ix.logf("%s", r.Answer)
return r
}
r.Stored, r.Dropped = true, removed
r.Answer = fmt.Sprintf("%s is dropped by %s (%s), with %d entries no other revision lists", name, by, why, len(removed))
if roomed, err := ix.RoomAgain(ctx); err != nil {
r.Answer += "; room is not evaluated: " + err.Error()
} else if roomed {
r.Answer += "; room said, what waited for it evaluated again"
}
ix.logf("%s", r.Answer)
return r
}
// RecomputeByHand computes the catalog graph again now from the view of the trunk, as a toolchain report or a merge
// does, and compares it with the newest revision kept: the modules whose entry moved, and the revision kept for them
// (none when the recompute equals the one kept).
func (ix *Indexer) RecomputeByHand(ctx context.Context, by, why string) TakenIn {
r := TakenIn{Module: "catalog graph"}
if ix.Revisions == nil {
r.Answer = "the indexer keeps no catalog graph revisions: nothing recomputed"
return r
}
before, _, err := newestTrunk(ctx, ix.Revisions)
if err != nil {
r.Answer = "the newest catalog graph revision cannot be read: " + err.Error()
return r
}
if err := ix.Recompute(ctx); err != nil {
r.Answer = "the recompute failed: " + err.Error()
return r
}
after, _, err := newestTrunk(ctx, ix.Revisions)
if err != nil {
r.Answer = "the newest catalog graph revision cannot be read after the recompute: " + err.Error()
return r
}
r.Stored = true
if after.Revision == before.Revision {
r.Answer = fmt.Sprintf("recomputed by %s (%s): equal to the kept revision %d", by, why, before.Revision)
} else {
r.Handed = changedEntries(before.Entries, after.Entries)
r.Answer = fmt.Sprintf("recomputed by %s (%s): revision %d differs from the kept %d in %s; revision %d is kept", by, why,
after.Revision, before.Revision, strings.Join(r.Handed, ", "), after.Revision)
}
ix.logf("%s", r.Answer)
return r
}
// CatalogRevisionTools are the tools of the catalog graph revisions; ask writes an ask in takeins and waits for the
// indexer's answer, record records a hand act through mesh-hand-acts before a drop is asked.
func CatalogRevisionTools(o Objects, ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time, record func(what, why string) error) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
asked := func(key string, value []byte, err error) (any, error) {
if err != nil {
return nil, err
}
a, err := ask(key, value)
if err != nil {
return nil, err
}
return a.Result, nil
}
return []stdio.Tool{
{Name: CatalogRevisionTool,
Description: "One catalog graph revision the indexer keeps in its objects catalog-revisions, whole: its list (kind, number, " +
"source revision, trunk heads, a candidate's group and heads) and every entry it lists, each module as its manifest " +
"declares it at that source revision (bases, packages, toolchains, provides, requires, invokes, serves, fingerprint, " +
"acting rules, inputs). The trunk's newest with no input; revision a kept trunk revision; group and heads a candidate's. " +
"An entry the store lost is an error naming it. indexer_catalog_revisions lists what is kept.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"revision": map[string]any{"type": "integer", "description": "a kept trunk revision's number"},
"group": map[string]any{"type": "string", "description": "a candidate's group, with heads"},
"heads": map[string]any{"type": "string", "description": "a candidate's heads, with group"}}},
Run: func(args map[string]any) (any, error) {
ctx := context.Background()
n, _ := args["revision"].(float64)
name := ""
switch {
case str(args, "group") != "" || str(args, "heads") != "":
name = RevisionObject(meshcatalog.CatalogRevision{Kind: "candidate", Group: str(args, "group"), Heads: str(args, "heads")})
case n > 0:
name = "trunk." + strconv.FormatInt(int64(n), 10)
default:
newest, ok, err := newestTrunk(ctx, o)
if err != nil {
return nil, fmt.Errorf("the catalog graph revisions cannot be read: %w", err)
}
if !ok {
return nil, ErrNoRevision
}
name = RevisionObject(newest)
}
rev, entries, err := ReadRevision(ctx, o, name)
if err != nil {
return nil, err
}
return map[string]any{"name": name, "revision": rev, "entries": entries}, nil
}},
{Name: CatalogRevisionsTool,
Description: "Every catalog graph revision the indexer keeps in its objects catalog-revisions, trunk and candidate: each " +
"one's name, kind, number, source revision, when, how many entries it lists and whether it is the trunk's newest " +
"(current); how many entries are stored once by digest, those no revision lists (unlisted), and the store's fill " +
"in bytes against its cap (filling is said at 80 %, full refuses the next revision). An object it cannot read is " +
"named in unread. indexer_catalog_revision reads one whole.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) {
return ListRevisions(context.Background(), o, false)
}},
{Name: CandidateGraphsTool,
Description: "Every candidate's catalog half the indexer keeps (a delivery group's or a pull request's heads merged onto " +
"a trunk source revision): its group, its heads, its revision, the source revision it is merged onto, since when, " +
"and how many entries it lists; and the store's fill. indexer_drop_candidate_graph drops one whose gate event never came.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) {
return ListRevisions(context.Background(), o, true)
}},
{Name: RecomputeTool,
Description: "Compute the catalog graph again now, as the indexer does on a toolchain report or a merge, from the view of " +
"the trunk (each module's manifest, its bases, the agreed toolchains and the published packages), and compare it with " +
"the newest revision kept: answers equal, or the modules whose entry differs and the revision kept for them.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"why": map[string]any{"type": "string", "description": "why it is recomputed now, in plain words"}}},
Run: func(args map[string]any) (any, error) {
key, value, err := NewRecomputeAsk(str(args, "why"), stdio.CallerOf(args), now())
return asked(key, value, err)
}},
{Name: DropCandidateGraphTool,
Description: "Drop one candidate's catalog half by hand, for a candidate whose gate event never came: its revision and the " +
"entries no other revision lists. A hand act, recorded through mesh-hand-acts before it is asked. When the drop takes " +
"the store below its filling mark, room is said and a merge held for objects full is taken in. indexer_candidate_graphs " +
"names the candidates.",
Input: map[string]any{"type": "object", "required": []string{"group", "heads", "why"}, "properties": map[string]any{
"group": map[string]any{"type": "string", "description": "the candidate's group"},
"heads": map[string]any{"type": "string", "description": "the candidate's heads"},
"why": map[string]any{"type": "string", "description": "why it is dropped, in plain words"}}},
Run: func(args map[string]any) (any, error) {
key, value, err := NewDropCandidateGraphAsk(str(args, "group"), str(args, "heads"), str(args, "why"), stdio.CallerOf(args), now())
if err != nil {
return nil, err
}
if err := record("dropped the catalog half of the candidate "+str(args, "group")+" at "+str(args, "heads")+" by hand", str(args, "why")); err != nil {
return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was dropped: %w", err)
}
return asked(key, value, nil)
}},
}
}
@@ -0,0 +1,102 @@
package index
import (
"context"
"encoding/json"
"fmt"
"slices"
"strconv"
"strings"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/views"
)
// RevisionInvoke is how a part declares the seat mesh-catalog's verb revision in its manifest's invokes, the one way a
// part reads the catalog half (novox/hq ADR 0663 §2).
const RevisionInvoke = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbRevision
// RevisionCallers are the parts the newest trunk revision's entries say declare the verb revision in their invokes,
// sorted, the holder itself never among them (ADR 0663 §2: granted to exactly the parts that declare it).
func RevisionCallers(entries []meshcatalog.CatalogEntry, holder string) stdio.Callers {
var out stdio.Callers
for _, e := range entries {
if e.Module != holder && slices.Contains(e.Invokes, RevisionInvoke) {
out = append(out, e.Module)
}
}
slices.Sort(out)
return out
}
// Revision answers the seat mesh-catalog's verb revision from the indexer's own objects (ADR 0663 §1, §3): one catalog
// graph revision, the trunk's newest when the input names none, a kept trunk revision by its number, or a candidate's by
// its group and heads, with one page of its entries whole, by module name. The caller is the runtime's proof: it is
// answered only when the newest trunk revision's entry of its module declares the verb in invokes, and the holder is
// refused too. With no trunk revision kept yet nobody is answered and the reader waits (ErrNoRevision), never an empty
// catalog.
func Revision(ctx context.Context, o Objects, holder, caller string, in meshcatalog.RevisionInput) (meshcatalog.RevisionAnswer, error) {
var out meshcatalog.RevisionAnswer
newest, ok, err := newestTrunk(ctx, o)
if err != nil {
return out, fmt.Errorf("the catalog graph revisions cannot be read: %w", err)
}
if !ok {
return out, ErrNoRevision
}
_, trunkEntries, err := ReadRevision(ctx, o, RevisionObject(newest))
if err != nil {
return out, err
}
if err := views.Refusal(meshcatalog.Seat+"."+meshcatalog.VerbRevision, RevisionCallers(trunkEntries, holder), caller); err != nil {
return out, fmt.Errorf("%w (novox/hq ADR 0663 §2: a part reads it by declaring %s in invokes)", err, RevisionInvoke)
}
name := RevisionObject(newest)
switch {
case in.Group != "" || in.Heads != "":
if in.Group == "" || in.Heads == "" {
return out, fmt.Errorf("a candidate's revision is named by its group and its heads, both")
}
name = RevisionObject(meshcatalog.CatalogRevision{Kind: "candidate", Group: in.Group, Heads: in.Heads})
case in.Revision != 0 && in.Revision != newest.Revision:
name = "trunk." + strconv.FormatInt(in.Revision, 10)
}
rev, entries := newest, trunkEntries
if name != RevisionObject(newest) {
if rev, entries, err = ReadRevision(ctx, o, name); err != nil {
return out, err
}
if in.Revision != 0 && rev.Revision != in.Revision {
return out, fmt.Errorf("the candidate %s.%s keeps its newest revision only, %d, not %d", in.Group, in.Heads, rev.Revision, in.Revision)
}
}
slices.SortFunc(entries, func(a, b meshcatalog.CatalogEntry) int { return strings.Compare(a.Module, b.Module) })
out.Revision, out.Entries = rev, []meshcatalog.CatalogEntry{}
size := 0
for _, e := range entries {
if e.Module <= in.After {
continue
}
data, _ := json.Marshal(e)
if len(out.Entries) > 0 && size+len(data) > views.PageBytes {
out.Next = out.Entries[len(out.Entries)-1].Module
break
}
size += len(data)
out.Entries = append(out.Entries, e)
}
return out, nil
}
// RevisionTool is the verb revision as the tool mesh-catalog.revision of the indexer's own runtime (ADR 0663 §1): the
// tool runner describes it with the seat's definition, so it carries no description or input of its own.
func RevisionTool(o Objects, holder string) stdio.Tool {
return stdio.Tool{Name: meshcatalog.Seat + "." + meshcatalog.VerbRevision, Run: func(args map[string]any) (any, error) {
var in meshcatalog.RevisionInput
if raw, err := json.Marshal(args); err == nil {
_ = json.Unmarshal(raw, &in)
}
return Revision(context.Background(), o, holder, stdio.CallerOf(args), in)
}}
}
@@ -2,9 +2,14 @@ package index
import (
"context"
"errors"
"fmt"
"maps"
"slices"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
@@ -56,6 +61,9 @@ func (ix *Indexer) ApplySource(ctx context.Context, a meshdelivery.SourceAnswer)
if ix.Modules == nil {
return ix.RecomputeCandidates(ctx)
}
if err := ix.kickCatchUp(ctx, s.Heads); err != nil {
return err
}
// A registered module the forge could not read is read again at each new source revision (issue 1945), in the
// background: a forge that does not answer must not hold this revision. One taken in later is stamped when it is written
// (applyManifests reads srcOf).
@@ -100,6 +108,124 @@ func (ix *Indexer) ApplySource(ctx context.Context, a meshdelivery.SourceAnswer)
return fmt.Errorf("modules.%s: %w", key, err)
}
}
// The stamps are kept with the view (KeepView), so a restart reads them back.
if err := ix.KeepView(ctx); err != nil {
return err
}
// A trunk that moved makes the previews made onto the old one out of date.
return ix.RecomputeCandidates(ctx)
}
// kickCatchUp takes the trunk in at a first start, when the objects keep no trunk revision yet (the lead's ruling A on
// milestone 47/G: the catch-up path): the head the source names for every repository, by the merged message the forge
// bridge gives for it, each applied as its merge, and revision 1 kept from them all at its end. Until then the revision
// the tools answer waits, never an empty catalog. The catch-up runs off the source handler, in its own goroutine, said
// when it ends (lead-47's ruling R1 on the rebase): a bridge that does not answer holds the kept call (ADR 0668) to its
// bound, and must not hold this source revision's stamps and the pulls' candidates with it. A repository whose message is
// not read keeps no revision, said: the next source revision runs the catch-up again (no timer). Objects not on the bus
// yet run none, said, until then.
func (ix *Indexer) kickCatchUp(ctx context.Context, heads map[string]string) error {
if ix.Revisions == nil || ix.Message == nil || len(heads) == 0 {
return nil
}
if _, any, err := newestTrunk(ctx, ix.Revisions); errors.Is(err, process.ErrNotYet) {
ix.logf("catalog graph: the trunk is not taken in yet: its objects %s are not on the bus", CatalogRevisionsObjects)
return nil
} else if err != nil || any {
return err
}
if !ix.catchingUp.CompareAndSwap(false, true) {
return nil
}
heads = maps.Clone(heads)
ix.catchUpWG.Add(1)
go func() {
defer ix.catchUpWG.Done()
unread := ix.takeInHeads(ctx, heads)
ix.catchingUp.Store(false)
if len(unread) > 0 {
ix.logf("catalog graph: the first start's catch-up ended and no first revision is kept: %d repositories were not taken in (%s); taken in again at the next source revision",
len(unread), strings.Join(unread, ", "))
return
}
if err := ix.KeepView(ctx); err != nil {
ix.logf("catalog graph: the first start's catch-up took in the trunk of %d repositories, but its revision could not be kept: %v; taken in again at the next source revision", len(heads), err)
return
}
ix.logf("catalog graph: the first start's catch-up ended: the trunk of %d repositories taken in", len(heads))
}()
return nil
}
// takeInHeads applies the forge bridge's merged message of each repository's head, as its merge, and answers the
// repositories whose message was not read or not applied.
func (ix *Indexer) takeInHeads(ctx context.Context, heads map[string]string) []string {
var unread []string
for _, repository := range slices.Sorted(maps.Keys(heads)) {
head := heads[repository]
if head == "" {
continue
}
f, err := ix.mergedAt(ctx, repository, head)
if err == nil {
_, err = ix.ApplyMessage(ctx, f)
}
if err != nil {
unread = append(unread, repository)
ix.logf("catalog graph: the trunk of %s at %.12s is not taken in: %v", repository, head, err)
}
}
return unread
}
// CatchUp is the indexer's catch-up tool's work (indexer_take_in with no module and no repository, the catch-up its
// manifest names beside the catalog graph revisions): after a restore point of them is restored, the node agent calls it
// with no arguments, and the indexer takes in the head the gate's source names for every repository, as the first
// start's catch-up does, whatever revision the objects keep, then keeps the view as the next revision when an entry
// moved. Its answer names the repositories taken in and those whose merged message could not be read.
func (ix *Indexer) CatchUp(ctx context.Context) TakenIn {
out := TakenIn{Module: "catch-up"}
ix.mu.Lock()
heads := maps.Clone(ix.source.heads)
ix.mu.Unlock()
switch {
case len(heads) == 0:
out.Answer = "the gate's source names no head yet, so nothing is taken in; the next source revision is taken in as it comes"
return out
case ix.Modules == nil || ix.Message == nil:
out.Answer = "the indexer holds no view of the trunk or reads no merged message, so nothing is taken in"
return out
case !ix.catchingUp.CompareAndSwap(false, true):
out.Answer = "a catch-up runs already; its end keeps the revision"
return out
}
unread := ix.takeInHeads(ctx, heads)
ix.catchingUp.Store(false)
if err := ix.KeepView(ctx); err != nil {
out.Answer = "the repositories were taken in but the revision could not be kept: " + err.Error()
return out
}
out.Stored = len(unread) == 0
out.Unread = unread
out.Answer = fmt.Sprintf("the heads the gate's source names for %d repositories taken in", len(heads)-len(unread))
if len(unread) > 0 {
out.Answer += fmt.Sprintf("; %d not (%s): their merged message was not read, and the call may be made again", len(unread), strings.Join(unread, ", "))
}
return out
}
// mergedAt is the forge bridge's merged message of a repository's head.
func (ix *Indexer) mergedAt(ctx context.Context, repository, head string) (forgemsg.Forge, error) {
data, err := ix.Message(ctx, repository, head, 0)
if err != nil {
return forgemsg.Forge{}, fmt.Errorf("the forge bridge could not answer: %w", err)
}
f, err := forgemsg.DecodeForge(data)
if err != nil {
return f, fmt.Errorf("the forge bridge's answer is not a forge message: %w", err)
}
if f.Kind != forgemsg.ForgeMerged || f.Merge != head || !trunk.SameRepository(f.Repository, repository) {
return f, fmt.Errorf("the forge bridge's answer is not the merge of %s at %s", repository, head)
}
return f, nil
}
@@ -32,6 +32,9 @@ type TakeInAsk struct {
// Repository asks the head the gate's source names for a repository to be taken in (TakeInRepository) instead of a module.
Repository string `json:"repository,omitempty"`
Why string `json:"why,omitempty"`
// CatchUp asks the head the gate's source names for every repository to be taken in (CatchUp): the catch-up tool,
// indexer_take_in called with no module and no repository, as the node agent calls it after a restore.
CatchUp bool `json:"catch-up,omitempty"`
// Judge asks the manifests to be judged again now (indexer_judge_again) instead of the module to be taken in.
Judge bool `json:"judge,omitempty"`
// Digest names, with Judge, the one manifest digest of the module to judge (indexer_judge with a digest).
@@ -63,12 +66,17 @@ type TakeInAsk struct {
// packages as they are read (indexer_fingerprints_again, fingerprints.go).
Fingerprints bool `json:"fingerprints,omitempty"`
// RecoverTries asks the open tries' candidates to be recovered now (indexer_recover_tries).
RecoverTries bool `json:"recover-tries,omitempty"`
Drop bool `json:"drop,omitempty"`
By string `json:"by,omitempty"`
At time.Time `json:"at"`
Result *TakenIn `json:"result,omitempty"`
Answered time.Time `json:"answered,omitzero"`
RecoverTries bool `json:"recover-tries,omitempty"`
// Recompute asks the catalog graph to be computed again now and compared with the newest kept revision (indexer_recompute).
Recompute bool `json:"recompute,omitempty"`
// CandidateGroup and CandidateHeads, with Drop, ask that candidate's catalog half to be dropped (indexer_drop_candidate_graph).
CandidateGroup string `json:"candidate-group,omitempty"`
CandidateHeads string `json:"candidate-heads,omitempty"`
Drop bool `json:"drop,omitempty"`
By string `json:"by,omitempty"`
At time.Time `json:"at"`
Result *TakenIn `json:"result,omitempty"`
Answered time.Time `json:"answered,omitzero"`
}
// Done says the ask is answered.
@@ -83,7 +91,7 @@ func DecodeTakeIn(data []byte) (TakeInAsk, error) {
if a.V != 1 {
return a, fmt.Errorf("a takeins value of version %d, not 1", a.V)
}
if a.Module == "" && a.Repository == "" && a.Pending == "" && a.Merge == "" && a.TryCandidate == "" && !a.RecoverTries && !a.Fingerprints {
if a.Module == "" && a.Repository == "" && a.Pending == "" && a.Merge == "" && a.TryCandidate == "" && !a.RecoverTries && !a.Fingerprints && !a.CatchUp && !a.Recompute && a.CandidateGroup == "" {
return a, errors.New("a takeins value naming no module, repository, pending wait, merge or try")
}
return a, nil
@@ -114,6 +122,20 @@ func NewTakeInRepository(repository, why, by string, at time.Time) (string, []by
return "repository." + key, data, err
}
// CatchUpKey is the takeins key of a catch-up: one at a time, a second written over an unanswered first.
const CatchUpKey = "catch-up"
// NewCatchUp is the value indexer_take_in writes when it names no module and no repository: the catch-up of every
// repository the gate's source names (CatchUp).
func NewCatchUp(why, by string, at time.Time) (string, []byte, error) {
why = strings.TrimSpace(why)
if why == "" {
why = "the catch-up after a restore"
}
data, err := json.Marshal(TakeInAsk{V: 1, CatchUp: true, Why: why, By: by, At: at.UTC()})
return CatchUpKey, data, err
}
// TakeInMergeTool takes one merge in by hand, its merged message asked of the forge bridge (ADR 0633 §5, issue 1585).
const TakeInMergeTool = "indexer_take_in_merge"
@@ -372,7 +372,10 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc
if err := ix.writeOverlay(ctx, overlay, finals, w); err != nil {
return err
}
return ix.writeGraph(ctx, edges)
if err := ix.writeGraph(ctx, edges); err != nil {
return err
}
return ix.keepTrunkRevision(ctx, edges, wholeGraph(edges))
}
// writeOverlay writes the records of a merge, each once and only if it differs from what the bucket holds; finals has
@@ -2,12 +2,15 @@ package index
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"os"
"reflect"
"slices"
"strings"
"sync"
"testing"
"time"
@@ -18,8 +21,10 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/views"
"git.novox.be/novox/mesh-delivery/sdk/graph"
@@ -285,16 +290,128 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { //
if g := graphOf(t, ix, "other"); g.FP != "" || !strings.Contains(g.None, "ts-base") || !reflect.DeepEqual(g.Bases, []string{"ts-base"}) {
t.Errorf("graph.other = %+v, want its base and its reason", g)
}
// Milestone 47/G W4.1, W4.2 (design 47, Two graphs): the catalog graph entry of a module is its graph node with where the
// trunk holds it; a revision lists one entry per module by digest, an unchanged entry has the same digest in the next
// revision, and a changed one only its own.
entries := map[string]meshcatalog.CatalogEntry{}
for name := range mods.vals {
m, _ := moduleOf(t, mods, name)
entries[name] = CatalogEntryOf(name, m, catalogue.Manifest{}, graphOf(t, ix, name))
}
// W3.1's main: the Go main package the module's process runs, read from its manifest (here the indexer's own); a
// manifest that runs no process names none.
own, err := os.ReadFile("../../module.json")
if err != nil {
t.Fatal(err)
}
if m, err := catalogue.Decode(own); err != nil || MainOf(m) != "cmd/mesh-delivery-indexer" || MainOf(catalogue.Manifest{}) != "" {
t.Errorf("the indexer's main = %q (%v), want cmd/mesh-delivery-indexer", MainOf(m), err)
}
if e := entries["avahi"]; e.Fp != want["avahi"] || e.Commit == "" || e.Manifest == "" || !reflect.DeepEqual(e.Bases, []string{"mesh-tools"}) ||
e.Toolchains["go"] != (meshcatalog.BuildEdge{Module: "mesh-build-go", Version: "go1.27.1", Recipe: "1"}) {
t.Errorf("the catalog entry of avahi = %+v", e)
}
// W3.1's inputs (the lead's ruling on finding 1): an entry carries what the next revision is computed from (the
// fingerprint's inputs, the toolchains used, the package versions), so the module's value read back from it is the
// value modules held, but for the old line's fields.
for name := range mods.vals {
m, _ := moduleOf(t, mods, name)
m.Fp, m.None, m.Needs = "", "", nil
if back := ModuleValueOf(entries[name]); !reflect.DeepEqual(back, m) {
t.Errorf("%s read back from its entry = %+v, want %+v", name, back, m)
}
}
if e := entries["avahi"]; e.Inputs == nil || e.Inputs.From == nil || e.Inputs.Toolchains["go"].Version != "go1.27.1" {
t.Errorf("the catalog entry of avahi carries inputs %+v, want what its fingerprint is computed from", e.Inputs)
}
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk", Source: 7}, entries)
if err != nil || len(rev.Entries) != len(mods.vals) || len(objects) != len(mods.vals) || rev.V != CatalogEntryLayout {
t.Fatalf("revision %+v with %d objects (%v), want one entry per module of %d", rev, len(objects), err, len(mods.vals))
}
moved := maps.Clone(entries)
tools := moved["mesh-tools"]
tools.Fp = "src2:" + strings.Repeat("b", 64)
moved["mesh-tools"] = tools
next, _, _ := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 2, Kind: "trunk", Source: 8}, moved)
for i, ref := range next.Entries {
if same := ref.Digest == rev.Entries[i].Digest; same == (ref.Module == "mesh-tools") {
t.Errorf("entry %s: digest %s against %s before; only mesh-tools' moves", ref.Module, ref.Digest, rev.Entries[i].Digest)
}
}
// W4.2's store: both revisions kept in catalog-revisions hold one object per entry once, the moved entry twice, and
// each list; a revision read back is its entries whole; an entry the bucket lost is an error, never left out.
kept := newMemObjects(0)
_, nextObjects, _ := CatalogRevisionOf(next, moved)
if err := KeepRevision(context.Background(), kept, rev, objects); err != nil {
t.Fatal(err)
}
if err := KeepRevision(context.Background(), kept, next, nextObjects); err != nil {
t.Fatal(err)
}
if names, _ := kept.Names(context.Background()); len(names) != len(mods.vals)+1+2 || kept.puts != len(names) {
t.Errorf("catalog-revisions holds %v after %d puts, want each entry once, the moved one twice, and two lists", names, kept.puts)
}
if back, got, err := ReadRevision(context.Background(), kept, "trunk.2"); err != nil || back.Source != 8 || len(got) != len(mods.vals) ||
!slices.ContainsFunc(got, func(e meshcatalog.CatalogEntry) bool { return e.Module == "mesh-tools" && e.Fp == tools.Fp }) {
t.Errorf("trunk.2 read back = %+v, %d entries, %v", back, len(got), err)
}
// W4.3's second part: the view of the trunk read at start is the newest trunk revision, each module's value as its
// entry gives it back; a bucket that keeps none is an empty view at 0.
view, nodes, at, err := TrunkView(context.Background(), kept)
if err != nil || at != 2 {
t.Fatalf("the view is read at %d (%v), want 2", at, err)
}
for name, e := range moved {
key, _ := meshcatalogclient.ModulesKey(name)
raw, _, found, _ := view.Get(context.Background(), key)
got, err := meshcatalogclient.DecodeModule(raw)
if want := ModuleValueOf(e); !found || err != nil || !reflect.DeepEqual(got, want) {
t.Errorf("the view's %s = %+v (%v), want %+v", name, got, err, want)
}
// The node of the module graph the entry carries is read back with the view (NodeOf), so a revision kept before the
// first recompute after a start loses no edge.
gk, _ := graph.Key(name)
rawNode, _, found, _ := nodes.Get(context.Background(), gk)
if back, err := graph.Decode(rawNode); !found || err != nil || !reflect.DeepEqual(NodeOf(e), back) || back.FP != e.Fp ||
!reflect.DeepEqual(back.Bases, e.Bases) || !reflect.DeepEqual(CatalogEntryOf(name, ModuleValueOf(e), catalogue.Manifest{}, back).Bases, e.Bases) {
t.Errorf("the view's node of %s = %+v (%v), want %+v", name, back, err, NodeOf(e))
}
}
if empty, _, at, err := TrunkView(context.Background(), newMemObjects(0)); err != nil || at != 0 || func() bool { k, _ := empty.Keys(context.Background()); return len(k) != 0 }() {
t.Errorf("an empty bucket's view is read at %d (%v), want an empty view at 0", at, err)
}
_ = kept.Delete(context.Background(), EntryObject(next.Entries[0].Digest))
if _, _, err := ReadRevision(context.Background(), kept, "trunk.2"); err == nil || !strings.Contains(err.Error(), next.Entries[0].Module) {
t.Errorf("trunk.2 with a lost entry read without error naming it: %v", err)
}
// W4.3: a recompute keeps a trunk revision of the module graph and says catalog-revision; one that moves no entry keeps
// none; a module leaving makes the next revision, naming it changed.
ix.Revisions = newMemObjects(0)
var revisions []meshcatalog.CatalogRevisionEvent
ix.SayCatalogRevision = func(_ context.Context, e meshcatalog.CatalogRevisionEvent) error {
revisions = append(revisions, e)
return nil
}
if err := ix.Recompute(context.Background()); err != nil || len(revisions) != 1 || revisions[0].Revision != 1 || revisions[0].Kind != "trunk" ||
revisions[0].Entries != int64(len(mods.vals)) || len(revisions[0].Changed) != len(mods.vals) {
t.Fatalf("the first recompute said %+v (%v), want trunk revision 1 with every module", revisions, err)
}
graph := ix.Graph.(*memStore)
written := graph.writes
if err := ix.Recompute(context.Background()); err != nil || graph.writes != written {
t.Errorf("a recompute that moves nothing wrote the graph %d times (%v)", graph.writes-written, err)
if err := ix.Recompute(context.Background()); err != nil || graph.writes != written || len(revisions) != 1 {
t.Errorf("a recompute that moves nothing wrote the graph %d times and said %d revisions (%v)", graph.writes-written, len(revisions), err)
}
// A module the modules state no longer holds (a merge dropped it, issue 1900) leaves the graph at the next recompute.
delete(mods.vals, "other")
if err := ix.Recompute(context.Background()); err != nil {
t.Fatal(err)
}
if len(revisions) != 2 || revisions[1].Revision != 2 || !reflect.DeepEqual(revisions[1].Changed, []string{"other"}) {
t.Errorf("after other left, said %+v, want revision 2 naming other changed", revisions)
}
if _, kept, err := ReadRevision(context.Background(), ix.Revisions, "trunk.2"); err != nil || len(kept) != len(mods.vals) {
t.Errorf("trunk.2 holds %d entries (%v), want %d", len(kept), err, len(mods.vals))
}
if _, held := graph.vals["other"]; held {
t.Error("graph.other is still held after its module went")
}
@@ -339,6 +456,137 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { //
t.Errorf("%s moved with a build of build-agent: fp2 = %q, want %s", name, m.Fp2, fp)
}
}
// The lead's ruling A on milestone 47/G: a first start whose objects keep no trunk revision takes in the head the source
// names for every repository, by the forge bridge's merged message, and keeps revision 1 with every module; a message
// the bridge does not give keeps none, and the next source revision takes the trunk in again.
first, _, merged := newFormIndexerUnmerged(t, builderModules(t)...)
first.Revisions = newMemObjects(0)
var firstSaid []meshcatalog.CatalogRevisionEvent
first.SayCatalogRevision = func(_ context.Context, e meshcatalog.CatalogRevisionEvent) error {
firstSaid = append(firstSaid, e)
return nil
}
// Lead-47's ruling R1: the catch-up runs off the source handler, so a bridge that has not answered yet holds no source
// revision (the first message is answered only once ApplySource returned; held in the handler it is answered as failed).
answers, applied := 0, make(chan struct{})
first.Message = func(_ context.Context, repository, head string, _ int) ([]byte, error) {
if answers++; answers == 1 {
select {
case <-applied:
return nil, errors.New("the forge bridge does not answer")
case <-time.After(10 * time.Second):
return nil, errors.New("the catch-up held the source handler")
}
}
if repository != catalogueRepo || head != merged.Merge {
return nil, fmt.Errorf("no merge of %s at %s", repository, head)
}
return json.Marshal(merged)
}
heads := meshdelivery.SourceByMainValue{Heads: map[string]string{catalogueRepo: merged.Merge}}
var firstLog []string
var logMu sync.Mutex
first.Logf = func(f string, a ...any) {
logMu.Lock()
firstLog = append(firstLog, fmt.Sprintf(f, a...))
logMu.Unlock()
}
err = first.ApplySource(context.Background(), 5, heads)
close(applied)
first.catchUpWG.Wait()
if err != nil || len(firstSaid) != 0 || !strings.Contains(strings.Join(firstLog, "\n"), "the forge bridge does not answer") {
t.Fatalf("a catch-up whose message is not read said %+v (%v), log %q, want no revision and the bridge's silence said", firstSaid, err, firstLog)
}
err = first.ApplySource(context.Background(), 6, heads)
first.catchUpWG.Wait()
if err != nil || len(firstSaid) != 1 || firstSaid[0].Revision != 1 ||
firstSaid[0].Entries != int64(len(builderModules(t))) || firstSaid[0].Source != 6 {
t.Fatalf("the first start's catch-up said %+v (%v), want trunk revision 1 of source 6 with every module", firstSaid, err)
}
// KeepView: a write to the view outside a recompute (here a verdict carried into one module) is the next revision.
viewKey, _ := meshcatalogclient.ModulesKey("avahi")
_ = write(context.Background(), first.Modules, viewKey, func(value []byte, _ bool) ([]byte, bool, error) {
m, _ := meshcatalogclient.DecodeModule(value)
m.Verdict, m.Problems = meshcatalogclient.VerdictRefused, []string{"a field the judge refuses"}
data, err := meshcatalogclient.EncodeModule(m)
return data, false, err
})
if err := first.KeepView(context.Background()); err != nil || len(firstSaid) != 2 || !reflect.DeepEqual(firstSaid[1].Changed, []string{"avahi"}) {
t.Errorf("KeepView after a write said %+v (%v), want revision 2 naming avahi", firstSaid, err)
}
if view, _, at, err := TrunkView(context.Background(), first.Revisions); err != nil || at != 2 {
t.Errorf("the view at a restart is read at %d (%v), want 2", at, err)
} else if raw, _, _, _ := view.Get(context.Background(), viewKey); !strings.Contains(string(raw), "a field the judge refuses") {
t.Errorf("the view at a restart lost the verdict: %s", raw)
}
// W4.14: the catch-up tool (indexer_take_in with no module and no repository, as the node agent calls it after a
// restore) takes in the head the source names for every repository again, whatever revision is kept, and keeps none
// when nothing moved; a repository whose message is not read is named in unread.
if key, value, err := NewCatchUp("", "node-engine@ace", time.Now()); err != nil || key != CatchUpKey {
t.Fatalf("NewCatchUp = %q (%v), want key %q", key, err, CatchUpKey)
} else if a, err := DecodeTakeIn(value); err != nil || !a.CatchUp {
t.Fatalf("a catch-up ask decodes to %+v (%v), want a catch-up", a, err)
}
asked := answers
if got := first.CatchUp(context.Background()); !got.Stored || answers != asked+1 || len(firstSaid) != 2 || len(got.Unread) != 0 {
t.Errorf("the catch-up answered %+v after %d asks, said %d revisions, want the one repository taken in, no new revision", got, answers-asked, len(firstSaid))
}
first.Message = func(context.Context, string, string, int) ([]byte, error) {
return nil, errors.New("the forge bridge does not answer")
}
if got := first.CatchUp(context.Background()); got.Stored || !reflect.DeepEqual(got.Unread, []string{catalogueRepo}) {
t.Errorf("a catch-up whose message is not read answered %+v, want %s unread", got, catalogueRepo)
}
// The tools bundle answers modules and graph from the newest trunk revision, their marks at its number; with none kept
// it says it waits (view-waits), never an empty catalog.
trunkStates := TrunkStates{Objects: first.Revisions, Under: func(string) views.Source { return nil }}
if keys, err := trunkStates.State("modules").Keys(); err != nil || len(keys) != len(builderModules(t)) {
t.Errorf("the bundle's modules = %v (%v), want every module of revision 2", keys, err)
}
if e, ok, err := trunkStates.State(views.MarksState).Get("graph"); err != nil || !ok || !strings.Contains(string(e.Value), `"revision":2`) {
t.Errorf("the bundle's mark of graph = %+v %v (%v), want revision 2", e, ok, err)
}
if _, err := (TrunkStates{Objects: newMemObjects(0)}).State("graph").Keys(); !errors.Is(err, ErrNoRevision) {
t.Errorf("the bundle's graph with no revision kept answered %v, want it to wait", err)
}
// W4.4, ADR 0663: the seat mesh-catalog's verb revision answers, from the indexer's own objects, exactly the parts whose
// entry in the newest trunk revision declares it, the holder refused too; a candidate's by its group and heads; none
// kept, everyone waits.
verbObjects := newMemObjects(0)
keep := func(rev meshcatalog.CatalogRevision, declares map[string]bool) {
entries := map[string][]byte{}
for _, name := range []string{"avahi", Module, "mesh-grants", "node-tools"} {
e := meshcatalog.CatalogEntry{Module: name, Repository: catalogueRepo}
if declares[name] {
e.Invokes = []string{RevisionInvoke}
}
data, _ := json.Marshal(e)
digest := fmt.Sprintf("%s-%d-%s", rev.Kind, rev.Revision, name)
entries[digest] = data
rev.Entries = append(rev.Entries, meshcatalog.EntryRef{Module: name, Digest: digest})
}
if err := KeepRevision(context.Background(), verbObjects, rev, entries); err != nil {
t.Fatal(err)
}
}
if _, err := Revision(context.Background(), verbObjects, Module, "n1.node-tools", meshcatalog.RevisionInput{}); !errors.Is(err, ErrNoRevision) {
t.Errorf("revision with none kept answered %v, want it to wait", err)
}
keep(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, map[string]bool{"node-tools": true, Module: true})
keep(meshcatalog.CatalogRevision{Revision: 4, Kind: "candidate", Group: "feat/x", Heads: "0123456789ab"}, map[string]bool{"node-tools": true, "mesh-grants": true})
for caller, answered := range map[string]bool{"n1.node-tools": true, "n1.mesh-grants": false, "n1." + Module: false, "person.someone": false, "unproven": false} {
got, err := Revision(context.Background(), verbObjects, Module, caller, meshcatalog.RevisionInput{})
if (err == nil) != answered || (answered && (got.Revision.Revision != 3 || len(got.Entries) != 4 || got.Next != "")) {
t.Errorf("revision to %s answered %+v (%v), want answered %v with trunk revision 3 whole", caller, got, err, answered)
}
}
if got, err := Revision(context.Background(), verbObjects, Module, "n1.node-tools", meshcatalog.RevisionInput{Group: "feat/x", Heads: "0123456789ab", After: "mesh-grants"}); err != nil ||
got.Revision.Kind != "candidate" || len(got.Entries) != 1 || got.Entries[0].Module != "node-tools" {
t.Errorf("a candidate's revision after mesh-grants answered %+v (%v), want its one entry node-tools", got, err)
}
if _, err := Revision(context.Background(), verbObjects, Module, "n1.node-tools", meshcatalog.RevisionInput{Revision: 9}); err == nil {
t.Error("a trunk revision not kept was answered")
}
// ADR 0614 §2, §3: the graph names what each module invokes and serves, and leaves out, over the whole graph, the
// verb edges of two modules calling each other, from either side; a third calling one of them keeps its edge.
invoking := func(name, invokes string) mod {
@@ -252,6 +252,10 @@ func (ix *Indexer) judgeOverHeld(ctx context.Context, fresh map[string]bool, ove
ix.logf("judge: modules.%s does not carry its verdict: %v", name, err)
}
}
// The verdicts carried are kept with the view (KeepView), so a restart reads them back.
if err := ix.KeepView(ctx); err != nil {
ix.logf("judge: the verdicts carried are not kept in a catalog graph revision: %v", err)
}
if over != nil {
return out, nil
}
@@ -11,10 +11,11 @@ import (
const MarksBucket = Module + "_" + views.MarksState
// FirstViewTools are the indexer's first-view tools (ADR 0621 §1, ADR 0622 §1, ADR 0631 §1): the tool, the state it answers, and
// the words of what that state holds. Each answers only its callers list (CallersEnv), which the manifest writes.
// the words of what that state holds. Each answers only its callers list (CallersEnv), which the manifest writes. The fourth
// first view, of which modules exist, is the seat mesh-catalog's verb revision (ADR 0663 §1, RevisionTool), granted by
// declaration, not by a list.
var FirstViewTools = []ViewedTool{
{Tool: "manifests", State: "manifests", Words: "every manifest the indexer keeps, by its digest."},
{Tool: "modules", State: "modules", Words: "every module the indexer holds, its fingerprint, inputs and manifest digest."},
{Tool: "verdicts", State: VerdictsState, Words: "the judge's verdict on each manifest: accepted, waiting or refused."},
{Tool: "graph", State: graph.State, Words: "the module graph: each module's build, provision and verb edges, whose verb edges a want waits on."},
}
@@ -43,13 +44,11 @@ type ViewedTool struct {
// ViewedStates are the states the indexer's views family says, each by its bucket: the ones its readers read.
var ViewedStates = map[string]string{
"manifests": ManifestsBucket,
"modules": ModulesBucket,
VerdictsState: VerdictsBucket,
PredefinedState: PredefinedBucket,
ConfigCandidatesState: ConfigCandidatesBucket,
"candidates": CandidatesBucket,
PackagesState: PackagesBucket,
graph.State: GraphBucket,
HeardState: HeardBucket,
"taken": TakenBucket,
mcc.FindingsState: mcc.FindingsBucket(Module),
+44 -15
View File
@@ -16,7 +16,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/catalogue"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-delivery/sdk/graph"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-indexer/internal/index"
@@ -57,7 +56,13 @@ type manifest struct {
Slug string `json:"slug"`
RunsAs string `json:"runs-as"`
Streams []stream `json:"streams"`
Claims []struct {
Objects []struct {
Name string `json:"name"`
MaxMiB int `json:"max-mib"`
Form int `json:"form"`
CatchUp string `json:"catch-up"`
} `json:"objects"`
Claims []struct {
Name string `json:"name"`
Scope string `json:"scope"`
Serves []string `json:"serves"`
@@ -113,19 +118,26 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
t.Errorf("emits = %v, want the two take-in events (the verdict events are the seat mesh-catalog's)", m.Emits)
}
// 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard.
if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: "try-recovery", History: 1}, {Name: "own-build", History: 1}, {Name: index.PackagesState, History: index.PackagesHistory},
{Name: graph.State, History: graph.History}, {Name: views.MarksState, History: 1}, {Name: mcc.FindingsState, History: 5}, {Name: views.StatusState, History: 1},
if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: "try-recovery", History: 1}, {Name: "own-build", History: 1}, {Name: index.PackagesState, History: index.PackagesHistory},
{Name: views.MarksState, History: 1}, {Name: mcc.FindingsState, History: 5}, {Name: views.StatusState, History: 1},
{Name: index.RequirementsState, History: 5}, {Name: index.SeatDefinitionsState, History: 5},
{Name: index.HoldsState, History: 5}, {Name: index.EngineReadsState, History: 1}, {Name: index.HoldBoundsState, History: 1, KeyTTL: true}, {Name: index.PendingState, History: 1}, {Name: index.UnjudgedState, History: 1}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17, ADR 0632 §2
t.Errorf("state = %+v, want %+v", m.State, want)
}
// Milestone 47/G W4.2 (design 47, Two graphs; ADR 0619 §2): the catalog graph revisions are objects in the old section,
// capped at 512 MiB, under the local name the indexer opens. W4.14 (design 47, Reverted): their form is 1, and the
// catch-up tool a restore of them is followed by is indexer_take_in, which the bundle serves.
if len(m.Objects) != 1 || m.Objects[0].Name != index.CatalogRevisionsObjects || m.Objects[0].MaxMiB != 512 ||
m.Objects[0].Form != 1 || m.Objects[0].CatchUp != index.TakeInTool || !slices.Contains(m.Tools, index.TakeInTool) {
t.Errorf("objects = %+v, want %s capped at 512 MiB, form 1, catch-up %s (a tool of its own)", m.Objects, index.CatalogRevisionsObjects, index.TakeInTool)
}
// D-11 (ADR 0634 §1): the holders and buckets the indexer names (internal/index/names.go) are its manifest's: its own
// name and states with their revisions, and the states it reads, each as `<holder>.<state>`.
if index.Module != m.Module {
t.Errorf("the indexer names itself %q, its manifest's module is %q", index.Module, m.Module)
}
for bucket, history := range map[string]int{index.CandidatesBucket: index.CandidatesHistory, index.ModulesBucket: index.ModulesHistory, index.ManifestsBucket: index.ManifestsHistory,
index.VerdictsBucket: index.VerdictsHistory, index.PackagesBucket: index.PackagesHistory, index.TakenBucket: 1, index.PredefinedBucket: 5, index.ConfigCandidatesBucket: 5, index.GraphBucket: graph.History} {
for bucket, history := range map[string]int{index.CandidatesBucket: index.CandidatesHistory, index.ManifestsBucket: index.ManifestsHistory,
index.VerdictsBucket: index.VerdictsHistory, index.PackagesBucket: index.PackagesHistory, index.TakenBucket: 1, index.PredefinedBucket: 5, index.ConfigCandidatesBucket: 5} {
if !slices.ContainsFunc(m.State, func(st state) bool { return m.Module+"_"+st.Name == bucket && st.History == history }) {
t.Errorf("the indexer names its bucket %s with %d revisions, its manifest's states are %+v", bucket, history, m.State)
}
@@ -163,25 +175,30 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
// 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none. The same bundle
// reads the verdicts state (indexer_verdicts) and asks the indexer to judge again through the takeins state
// (indexer_judge; issue 774, clause 3). 1966/AC11: indexer_old_sections reads the count of the old sections (state oldsections).
// ADR 0621 §1, ADR 0622 §1, ADR 0631 §1: the first-view tools manifests, modules, verdicts and graph, and design 25 § 12.2's read
// ADR 0621 §1, ADR 0622 §1, ADR 0631 §1, ADR 0663 §1: the first-view tools manifests, verdicts and graph (modules is the seat
// verb revision's, served by the claim), and design 25 § 12.2's read
// tools, all of the tools bundle, which runs in a runtime of its own on the indexer's account (runs-as).
want := []string{"indexer_take_in", "indexer_old_sections", "indexer_verdicts", "indexer_judge"}
for _, v := range append(append([]index.ViewedTool{}, index.FirstViewTools...), index.ReadTools...) {
want = append(want, v.Tool)
if _, ok := index.ViewedStates[v.State]; !ok {
// Milestone 47/G: modules and graph are the newest catalog graph revision's, which catalog-revision says the changes of.
if _, ok := index.ViewedStates[v.State]; !ok && v.State != index.ModulesState && v.State != "graph" {
t.Errorf("%s answers the state %s, which the views family does not say", v.Tool, v.State)
}
}
want = append(want, "indexer_first_views", "indexer_first_view_again", index.ConfigReadersTool, index.BusStateTool, index.RequirementsTool, index.PackageUsersTool, index.SayAcceptedTool, // DLV-F3r, DLV-I10, DLV-I11, DLV-I7, DLV-I8, DLV-I9c
index.SeatDefinitionTool, index.SeatDefinitionsAgainTool, index.OutOfStepTool, // DLV-I19
index.HoldsTool, index.EngineReadsTool, // ADR 0632 §2: the mesh-wide holds and the engines' reads they weigh
index.PendingTool, index.TakeInPendingTool, index.DropPendingTool, index.TryCandidatesTool, index.DropTryCandidateTool, index.RecoverTriesTool, index.TakeInMergeTool, index.FingerprintsAgainTool) // issue 1966; 1585/T12, T13 (ADR 0633 §5)
if !reflect.DeepEqual(m.Tools, want) || len(want) != 40 {
index.PendingTool, index.TakeInPendingTool, index.DropPendingTool, index.TryCandidatesTool, index.DropTryCandidateTool, index.RecoverTriesTool, index.TakeInMergeTool, index.FingerprintsAgainTool, // issue 1966; 1585/T12, T13 (ADR 0633 §5)
// milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered.
index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool)
if !reflect.DeepEqual(m.Tools, want) || len(want) != 45 || slices.Contains(m.Tools, "modules") {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
// Every read tool of the indexer a reader of this repository invokes is one it serves (the checker's seventh first
// view, indexer_read_seat_definitions, was invoked and not served).
readers, _ := filepath.Glob("../*/module.json")
var declaring []string
for _, p := range readers {
raw, err := os.ReadFile(p)
if err != nil {
@@ -194,24 +211,36 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
t.Fatalf("%s: %v", p, err)
}
for _, inv := range r.Invokes {
// The grants part reads modules until its own subtask (W8) moves it to revision; the join removes this line.
if p == "../mesh-grants/module.json" && inv == "mesh-delivery-indexer.modules" {
continue
}
if tool, ok := strings.CutPrefix(inv, "mesh-delivery-indexer."); ok && !slices.Contains(m.Tools, tool) {
t.Errorf("%s invokes %s, which the indexer does not serve", p, inv)
}
if inv == index.RevisionInvoke {
declaring = append(declaring, filepath.Base(filepath.Dir(p)))
}
}
}
// ADR 0663 §2: the assigner reads which modules exist by declaring the seat's verb revision (the grants part too, from W8).
if !slices.Contains(declaring, "mesh-delivery-assigner") {
t.Errorf("the parts of this repository declaring %s are %v, want the assigner", index.RevisionInvoke, declaring)
}
if m.RunsAs != m.Module || m.SecretsOwner != m.RunsAs {
t.Errorf("runs-as = %q, secrets-owner %q: the tools bundle runs on the indexer's own account (ADR 0621 §1)", m.RunsAs, m.SecretsOwner)
}
// ADR 0621 §2, ADR 0622 §1, ADR 0631 §1: each first-view tool answers exactly its readers; the indexer is on none, and
// graph answers the assigner alone.
callers := map[string]string{"manifests": "mesh-grants,node-tools,mesh-delivery-assigner", "modules": "mesh-grants,node-tools,mesh-delivery-assigner", "verdicts": "mesh-grants",
// ADR 0663 §2: revision has no callers list; it answers the parts whose entry declares it.
callers := map[string]string{"manifests": "mesh-grants,node-tools,mesh-delivery-assigner", "verdicts": "mesh-grants",
"graph": "mesh-delivery-assigner"}
for _, a := range m.Build.Artifacts {
if a.Name != "tools" {
continue
}
if len(a.Env) != len(callers) {
t.Errorf("tools env = %v, want only the four callers lists", a.Env)
t.Errorf("tools env = %v, want only the three callers lists", a.Env)
}
for tool, list := range callers {
if got := a.Env[views.CallersEnv(tool)]; got != list {
@@ -241,10 +270,10 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
}
}
// ADR 0604 §2, ADR 0636 §4: it holds the seat mesh-catalog, serving its four verbs and seat-definitions from its tools
// bundle (which stays without a request subject).
// bundle (which stays without a request subject); ADR 0663 §1: and revision, in the same runtime of its own.
if len(m.Claims) != 1 || m.Claims[0].Name != index.CatalogSeat || m.Claims[0].Scope != catalogue.ScopeMesh ||
!slices.Equal(m.Claims[0].Serves, []string{"modules", "module", "provides", "dependents", index.SeatDefinitionsVerb}) {
t.Errorf("claims = %+v, want mesh-catalog with its five verbs", m.Claims)
!slices.Equal(m.Claims[0].Serves, []string{"modules", "module", "provides", "dependents", index.SeatDefinitionsVerb, "revision"}) {
t.Errorf("claims = %+v, want mesh-catalog with its six verbs", m.Claims)
}
for _, e := range []string{index.SeatDefinitionChangedEvent, index.SeatDefinitionRemovedEvent} {
if slices.Contains(m.Emits, e) {
+11 -6
View File
@@ -12,7 +12,8 @@
"module",
"provides",
"dependents",
"seat-definitions"
"seat-definitions",
"revision"
]
}
],
@@ -22,7 +23,6 @@
"indexer_verdicts",
"indexer_judge",
"manifests",
"modules",
"verdicts",
"graph",
"indexer_read_manifests",
@@ -56,7 +56,12 @@
"indexer_drop_try_candidate",
"indexer_recover_tries",
"indexer_take_in_merge",
"indexer_fingerprints_again"
"indexer_fingerprints_again",
"indexer_catalog_revision",
"indexer_catalog_revisions",
"indexer_candidate_graphs",
"indexer_recompute",
"indexer_drop_candidate_graph"
],
"consumes": [
"go-package-registry.published",
@@ -77,7 +82,6 @@
{ "name": "candidates", "history": 10 },
{ "name": "heard", "history": 64 },
{ "name": "manifests", "history": 1 },
{ "name": "modules", "history": 20 },
{ "name": "predefined", "history": 5 },
{ "name": "config-candidates", "history": 5 },
{ "name": "taken", "history": 1 },
@@ -87,7 +91,6 @@
{ "name": "try-recovery", "history": 1 },
{ "name": "own-build", "history": 1 },
{ "name": "packages", "history": 5 },
{ "name": "graph", "history": 5 },
{ "name": "view-marks", "history": 1 },
{ "name": "findings", "history": 5 },
{ "name": "first-views", "history": 1 },
@@ -99,6 +102,9 @@
{ "name": "pending", "history": 1 },
{ "name": "unjudged", "history": 1 }
],
"objects": [
{ "name": "catalog-revisions", "max-mib": 512, "form": 1, "catch-up": "indexer_take_in" }
],
"emits": [
"take-in.pending",
"take-in.taken",
@@ -194,7 +200,6 @@
],
"env": {
"MESH_VIEW_CALLERS_MANIFESTS": "mesh-grants,node-tools,mesh-delivery-assigner",
"MESH_VIEW_CALLERS_MODULES": "mesh-grants,node-tools,mesh-delivery-assigner",
"MESH_VIEW_CALLERS_VERDICTS": "mesh-grants",
"MESH_VIEW_CALLERS_GRAPH": "mesh-delivery-assigner"
}