Merge fix/a479-catalog-answers-in-time (ea106ea8) into the catalog-answers-from-memory branch

Builds: design 47 G A-487..A-487c, A-479

# Conflicts:
#	modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go
#	modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go
This commit is contained in:
2026-11-19 01:24:39 +01:00
33 changed files with 382 additions and 1565 deletions
@@ -1,8 +1,6 @@
package main
import (
"fmt"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
stdio "git.novox.be/novox/mesh-sdk/go"
@@ -19,20 +17,13 @@ func catalogRead(modules views.Source, manifest func(digest string) (string, err
return index.CatalogRead{
SeatDefinitions: seatDefinitions, Marks: marks,
Modules: func() (map[string]meshcatalog.ModulesByModuleValue, error) {
keys, err := modules.Keys()
values, err := index.SourceValues(modules)
if err != nil {
return nil, err
}
out := map[string]meshcatalog.ModulesByModuleValue{}
for _, k := range keys {
e, ok, err := modules.Get(k)
if err != nil {
return nil, fmt.Errorf("modules.%s: %w", k, err)
}
if !ok {
continue
}
if m, err := meshcatalogclient.DecodeModule(e.Value); err == nil {
for k, v := range values {
if m, err := meshcatalogclient.DecodeModule(v); err == nil {
out[k] = m
}
}
@@ -52,20 +43,13 @@ func catalogRead(modules views.Source, manifest func(digest string) (string, err
},
Manifest: manifest,
Graph: func() (graph.Graph, error) {
keys, err := nodes.Keys()
values, err := index.SourceValues(nodes)
if err != nil {
return nil, err
}
g := graph.Graph{}
for _, k := range keys {
e, ok, err := nodes.Get(k)
if err != nil {
return nil, fmt.Errorf("graph.%s: %w", k, err)
}
if !ok {
continue
}
if n, err := graph.Decode(e.Value); err == nil {
for k, v := range values {
if n, err := graph.Decode(v); err == nil {
g[k] = n
}
}
@@ -75,7 +59,27 @@ func catalogRead(modules views.Source, manifest func(digest string) (string, err
}
// catalogTools are the seat's verbs over the indexer's own states, served beside the indexer's own tools, since the indexer's manifest claims the seat (ADR 0604 §2).
func catalogTools(state func(string) views.Source, kept *index.KeptTrunk) []stdio.Tool {
// Every state a verb reads whole is answered from memory (the view of the trunk held, or a watched copy), so a call costs no
// round trip per key: four node-engines ask every module at once.
func catalogTools(state func(string) views.Source, kept *index.KeptTrunk, watched *stdio.WatchedStates) []stdio.Tool {
return index.CatalogTools(catalogRead(state("modules"), kept.Manifest, state(graph.State),
keptSource(index.SeatDefinitionsState), keptSource(views.MarksState)))
watchedSource{watched, index.SeatDefinitionsState}, keptSource(views.MarksState)))
}
// watchedSource is a state as its watched copy holds it (stdio.WatchedStates), as a Source; it offers every value in one read.
type watchedSource struct {
w *stdio.WatchedStates
state string
}
func (s watchedSource) Keys() ([]string, error) { return s.w.Keys(s.state) }
func (s watchedSource) Get(key string) (*views.Entry, bool, error) {
v, ok, err := s.w.Get(s.state, key)
if err != nil || !ok {
return nil, false, err
}
return &views.Entry{Value: v}, true, nil
}
func (s watchedSource) Values() (map[string][]byte, error) { return s.w.Values(s.state) }
@@ -307,7 +307,8 @@ func main() {
os.Exit(1)
}
follow.Start(context.Background())
tools = append(tools, catalogTools(trunk.State, kept)...)
watched := stdio.NewWatchedStates()
tools = append(tools, catalogTools(trunk.State, kept, watched)...)
// The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly
// the parts whose entry declares it (novox/hq ADR 0663).
tools = append(tools, index.RevisionTool(revisions, kept, os.Getenv(stdio.ServedAs)), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}),
@@ -336,15 +337,11 @@ func main() {
})...)
tools = append(tools, viewTools(os.Getenv, trunk.State, time.Now)...)
tools = append(tools, index.PendingTools(keptRead{stdio.State(index.PendingState)}, keptRead{stdio.State(index.OwnBuildStateName)}, asks, time.Now)...)
tools = append(tools, index.ValidationTools(asks, time.Now, func(what, why string) error {
_, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.ValidateAgainTool, Verb: "validate-again"})
return err
})...)
tools = append(tools, index.FingerprintsTools(asks, time.Now, func(what, why string) error {
_, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.FingerprintsAgainTool, Verb: "fingerprints-again"})
return err
}, 10*time.Second)...)
tools = append(tools, configReadersTool(stdio.NewWatchedStates(), trunk.State("modules")), busStateTool(trunk.State), requirementsTool(keptSource),
tools = append(tools, configReadersTool(watched, trunk.State("modules")), busStateTool(trunk.State), requirementsTool(keptSource),
packageUsersTool(trunk.State), seatDefinitionTool(keptSource), outOfStepTool(os.Getenv(stdio.ServedAs), keptSource))
if err := stdio.Serve("", tools); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err)
@@ -22,7 +22,6 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"github.com/nats-io/nats-server/v2/server"
"github.com/nats-io/nats.go"
@@ -97,11 +96,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
// The forge bridge's tool bridge_merged_message (issue 1585): answers the merged message published for a merge, by
// repository@merge, as the bridge does; a restarted indexer holds none in memory and asks it.
var bridgeMessages sync.Map
// The build seat node-build-go's stand-in (design 47 G §712, §713): it hears the indexer's validation asks and ends each with
// the manifest validator's verdict; refusing names, by the commit the job runs at, the manifest a revision's change refuses.
var validations *standIn
var refusing sync.Map
js, ctx, candidates, _, restart := startIndexerRun(t, func(ctx context.Context, builds buildLog) {
js, ctx, candidates, _, _ := startIndexerRun(t, func(ctx context.Context, builds buildLog) {
// The builder's log entries of two builds' ends (ADR 0571 clause 1), as the builder appends them.
builds.end(t, "novox/mesh-delivery", "a", "modules/a")
builds.end(t, "novox/mesh-catalog", "c", "modules/c")
@@ -128,9 +123,8 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
if _, err := tc.Put(ctx, key, value); err != nil {
t.Fatal(err)
}
// The trunk holds the indexer's entry, taken in at an earlier commit with its own manifest; the validation names no Go module (A-268).
// The trunk holds the indexer's entry, taken in at an earlier commit with its own manifest; the take-in asks no validation (A-476 a).
seedBuildModules(t, ctx, js, map[string]meshcatalog.CatalogEntry{testIndexer: trunkIndexerEntry(t, ctx, js)})
validations = standInBuildSeat(t, ctx, js, &refusing)
// The node-engine's composition on n1 (process.Conn.Call's subjects): the build of the indexer that runs there.
answer := func(m *nats.Msg) {
raw, _ := json.Marshal(map[string]any{"result": map[string]any{"runs": map[string]string{"fp": *running.Load()}}, "node": "n1"})
@@ -281,17 +275,13 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
}
return takein.TakeIn{}, false
}
held, ok := said(takein.TakeInPending, m1Key)
if !ok || held.What != takein.TakeInRevision || held.WaitsFor == "" || held.WaitsFor == first || len(held.Merges) != 1 {
t.Fatalf("take-in.pending.%s = %+v (said %v) within 20 s, want the revision waiting for another build than %s", m1Key, held, ok, first)
// A-476 a, b: the revision changing the indexer's own tree is taken in at once, with the build this program runs and no
// validation ask: it is never said pending, and the build the revision names is not waited for.
if taken, ok := said(takein.TakeInTaken, m1Key); !ok || len(taken.Merges) != 1 || taken.Merges[0].Merge != m1.Merge {
t.Fatalf("take-in.taken.%s = %+v (said %v) within 20 s, want the revision taken in at once", m1Key, taken, ok)
}
named := held.WaitsFor
// IX5 (§712): the revision was validated first, by one run ask whose job names the indexer entry's repository, path and
// the revision's head, and no heads; accepted, it is never asked again while it waits for the named indexer.
if asks := validations.asked(); len(asks) != 1 || asks[0].Run == nil || asks[0].Kind != sdkstep.AskCheck ||
asks[0].Run.Program.Repository != m1.Repository || asks[0].Run.Program.Dir != "modules/"+ix || asks[0].Run.Program.Commit != m1.Merge ||
len(asks[0].Run.Heads) != 0 || asks[0].Run.Program.Package != "./cmd/mesh-judge" {
t.Fatalf("validation asks = %+v, want one run of the indexer's manifest validator at %s", asks, m1.Merge)
if held := saidAll(takein.TakeInPending, m1Key); len(held) != 0 {
t.Errorf("the revision was said pending: %+v", held)
}
get := func(bucket, key string) ([]byte, bool) {
t.Helper()
@@ -312,11 +302,10 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
}
return e.Value(), true
}
// Only what building and moving the named indexer needs is written.
raw, ok := get(index.ModulesState, ix)
own, err := meshcatalogclient.DecodeModule(raw)
if !ok || err != nil || own.Fp2 != named || own.Commit != m1.Merge {
t.Fatalf("modules.%s = %+v (%v), want the named build %s at the merge", ix, own, err, named)
if !ok || err != nil || own.Commit != m1.Merge {
t.Fatalf("modules.%s = %+v (%v), want the indexer at the merge", ix, own, err)
}
if _, ok := get(index.Bucket(testIndexer, index.ManifestsState), own.Manifest); !ok {
t.Errorf("manifests.%s is not written", own.Manifest)
@@ -325,16 +314,11 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
t.Errorf("the verdict on the indexer's manifest %s is not written", own.Manifest)
}
repoKey, _ := trunk.RepositoryKey(m1.Repository)
if _, ok := get(index.ModulesState, "a"); ok {
t.Errorf("modules.a is written while the revision is pending")
if raw, ok := get(index.ModulesState, "a"); !ok || !strings.Contains(string(raw), m1.Merge) {
t.Errorf("modules.a = %s, want it at the merge %s", raw, m1.Merge)
}
if _, ok := get(index.Bucket(testIndexer, index.TakenState), repoKey); ok {
t.Errorf("taken.%s is written while the revision is pending", repoKey)
}
if e, err := candidates.Get(ctx, "feat/x"); err != nil {
t.Errorf("the candidate feat/x is gone while the merge of #12 is pending: %v", err)
} else if c, err := index.DecodeCandidate(e.Value()); err != nil || len(c.Pulls) != 2 {
t.Errorf("the candidate feat/x = %+v while the merge of #12 is pending, want its two pull requests", c)
if _, ok := get(index.Bucket(testIndexer, index.TakenState), repoKey); !ok {
t.Errorf("taken.%s is not written", repoKey)
}
// A group's first member arrives meanwhile; its end is still to come (the knowledge base's remark: it stays across the restart).
@@ -365,34 +349,13 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
}
var w []takein.PendingWait
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if w = waits(); len(w) == 2 && w[0].WaitsFor == named && w[1].Said != "" {
if w = waits(); len(w) == 1 && w[0].Said != "" {
break
}
}
if len(w) != 2 || w[0].Key != m1Key || w[0].WaitsFor != named || !slices.Contains(w[0].Holds, "a") || !slices.Contains(w[0].Holds, ix) ||
w[1].Key != groupKey || w[1].What != takein.TakeInGroup || len(w[1].Merges) != 1 || w[1].Said != takein.TakeInPendingEvent {
t.Fatalf("indexer_pending = %+v, want the revision %s waiting for %s holding a and the indexer, then the group gathered", w, m1Key, named)
}
// The named indexer starts (the node-engine now runs that build): it takes the revision in, and leaves the group.
running.Store(&named)
restart()
if taken, ok := said(takein.TakeInTaken, m1Key); !ok || len(taken.Merges) != 1 || taken.Merges[0].Merge != m1.Merge {
t.Fatalf("take-in.taken.%s = %+v (said %v) within 20 s of the start of %s", m1Key, taken, ok, named)
}
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if w = waits(); len(w) == 1 {
break
}
}
if raw, ok := get(index.ModulesState, "a"); !ok || !strings.Contains(string(raw), m1.Merge) {
t.Errorf("modules.a = %s after the take-in at the start, want it at the merge %s", raw, m1.Merge)
}
if _, ok := get(index.Bucket(testIndexer, index.TakenState), repoKey); !ok {
t.Errorf("taken.%s is not written by the take-in at the start", repoKey)
}
if len(w) != 1 || w[0].Key != groupKey || len(w[0].Keys) != 1 || w[0].Keys[0] != g1Key {
t.Fatalf("indexer_pending after the start = %+v, want only the group gathered, its first member %s", w, g1Key)
if len(w) != 1 || w[0].Key != groupKey || w[0].What != takein.TakeInGroup || len(w[0].Merges) != 1 || len(w[0].Keys) != 1 || w[0].Keys[0] != g1Key ||
w[0].Said != takein.TakeInPendingEvent {
t.Fatalf("indexer_pending = %+v, want only the group gathered, its first member %s", w, g1Key)
}
if _, ok := get(index.ModulesState, "c"); ok {
t.Errorf("modules.c is written while its group is gathered")
@@ -425,23 +388,6 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
t.Errorf("the group's first member %s was said pending as a revision of its own", g1Key)
}
// IX5 (§712, §713): a merge whose changed manifest gains a problem the trunk case lacks stays pending, refused naming it,
// said in take-in.pending; nothing of it is taken in.
m3 := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/y", Number: 13, Head: h1,
Merge: "3333333333333333333333333333333333333333", Root: "r0", Trees: map[string]string{"modules/a": "tree-a-3"},
Manifests: map[string]string{"modules/a": `{"module":"a","version":"2.0"}`}}
refusing.Store(m3.Merge, "a")
m3Key := takein.PendingKey(merged(m3))
if refused, ok := said(takein.TakeInPending, m3Key); !ok || !strings.Contains(refused.Refused, "a: the stand-in's problem") || refused.Failed != "" {
t.Fatalf("take-in.pending.%s = %+v (said %v) within 20 s, want the revision refused naming a's problem", m3Key, refused, ok)
}
if w = waits(); len(w) != 1 || w[0].Key != m3Key || w[0].Validation == nil || w[0].Validation.Accepted != 0 || len(validations.asked()) != 3 {
t.Fatalf("indexer_pending = %+v, asks %d: want the refused revision pending with its validation, asked once after m1 and the group", w, len(validations.asked()))
}
if raw, ok := get(index.ModulesState, "a"); !ok || strings.Contains(string(raw), m3.Merge) {
t.Errorf("modules.a = %s: the refused revision was taken in", raw)
}
}
// trunkIndexerEntry keeps the indexer's own manifest by its digest, accepted, and answers its trunk entry at an earlier commit.
@@ -475,63 +421,6 @@ func trunkIndexerEntry(t *testing.T, ctx context.Context, js jetstream.JetStream
return e
}
// standIn is the build seat node-build-go as the indexer's validation meets it: the asks it heard, oldest first.
type standIn struct {
mu sync.Mutex
asks []sdkstep.StepAsk
}
func (s *standIn) asked() []sdkstep.StepAsk {
s.mu.Lock()
defer s.mu.Unlock()
return slices.Clone(s.asks)
}
// standInBuildSeat keeps the seat's check queue as a stream and hears the indexer's asks on it (accept.check.indexer): each is
// ended at once with step-ended on the seat's family, its answer inline (A-254), the program's last line: the manifest validator's verdict over both cases,
// the change case refusing the manifest refusing names for the job's commit.
func standInBuildSeat(t *testing.T, ctx context.Context, js jetstream.JetStream, refusing *sync.Map) *standIn {
t.Helper()
subject, err := sdkstep.AskSubjectBy(sdkstep.AskGo, sdkstep.AskCheck, index.ValidationAsker)
if err != nil || subject != "mesh.seat.node-build-go.accept.check.mesh-catalog" {
t.Fatalf("the validation's subject %q %v, want the indexer's seat name its asker (A-163, A-232)", subject, err)
}
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "SEAT_NODE_BUILD_GO_CHECK", Subjects: []string{"mesh.seat.node-build-go.accept.check.>"}}); err != nil {
t.Fatal(err)
}
// The indexer's state inputs and its objects, where the validation's request is kept for the build seat's holder (§354).
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, index.InputsState), History: 1}); err != nil {
t.Fatal(err)
}
if _, err := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: catalogue.BusName(testIndexer, catalogue.StateObjectsName(index.InputsState))}); err != nil {
t.Fatal(err)
}
s := &standIn{}
if _, err := js.Conn().Subscribe(subject, func(m *nats.Msg) {
a, err := sdkstep.DecodeStepAsk(m.Data)
if err != nil || a.Run == nil {
t.Errorf("the stand-in build seat heard what is no run ask: %v", err)
return
}
s.mu.Lock()
s.asks = append(s.asks, a)
s.mu.Unlock()
change := composer.CaseVerdict{Name: "change", Manifests: map[string]composer.ManifestRead{}}
if module, ok := refusing.Load(a.Run.Program.Commit); ok {
change.Manifests[module.(string)] = composer.ManifestRead{Refused: "the stand-in's problem"}
}
verdict, _ := json.Marshal(composer.Verdict{V: 1, Cases: []composer.CaseVerdict{change, {Name: judge.TrunkCase, Manifests: map[string]composer.ManifestRead{}}}})
end, _ := json.Marshal(sdkstep.StepEnded{V: 1, ID: a.ID, Ask: a.ID, Asker: index.ValidationAsker, Node: "n1", Outcome: sdkstep.OutcomePassed,
Output: string(verdict), At: time.Now().UTC()})
if _, err := js.Publish(ctx, nodebuildgo.SubjectStepEnded("n1"), end); err != nil {
t.Errorf("the stand-in build seat could not say step-ended: %v", err)
}
}); err != nil {
t.Fatal(err)
}
return s
}
// jsSource is a bucket as a views.Source.
type jsSource struct {
ctx context.Context
@@ -1529,7 +1418,8 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
t.Fatal(err)
}
toolchains = kv
for node, version := range map[string]string{"n1": "go1.27.3", "n2": "go1.27.2"} {
// A-471: a toolchain is agreed by its language version (go1.27), not its patch; n2 reports another language version.
for node, version := range map[string]string{"n1": "go1.27", "n2": "go1.28.1"} {
key, _ := index.ToolchainKey(node, "go")
value, err := index.EncodeToolchain(nodebuildgo.Toolchain{Language: "go", Version: version, Recipe: "1", At: ptr(time.Now())})
if err != nil {
@@ -1573,15 +1463,15 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
return got
}
agreed := func(m meshcatalog.ModulesByModuleValue) bool {
return m.Fp2 != "" && m.Toolchains["go"].Version == "go1.27.3"
return m.Fp2 != "" && m.Toolchains["go"].Version == "go1.27"
}
held := read(agreed)
if !agreed(held) {
t.Fatalf("within 20 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27.3 alone", held.Fp2, held.None2, held.Toolchains)
t.Fatalf("within 20 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27 alone", held.Fp2, held.None2, held.Toolchains)
}
place(ctx, t, "n2", "mesh-build-go")
key, _ := index.ToolchainKey("n2", "go")
value, err := index.EncodeToolchain(nodebuildgo.Toolchain{Language: "go", Version: "go1.27.2", Recipe: "1", At: ptr(time.Now().Add(time.Second))})
value, err := index.EncodeToolchain(nodebuildgo.Toolchain{Language: "go", Version: "go1.28.1", Recipe: "1", At: ptr(time.Now().Add(time.Second))})
if err != nil {
t.Fatal(err)
}
@@ -1604,7 +1494,7 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
t.Fatal(err)
}
disagree := func(m meshcatalog.ModulesByModuleValue) bool {
return m.Fp2 == "" && strings.HasPrefix(m.None2, "builders disagree: go1.27.2 recipe 1 on n2")
return m.Fp2 == "" && strings.HasPrefix(m.None2, "builders disagree: go1.27 recipe 1 on n1; go1.28 recipe 1 on n2")
}
if m := read(disagree); !disagree(m) {
t.Fatalf("within 20 s of a merge after n2's assignments avahi has fp2 %q, none2 %q: want the builders to disagree", m.Fp2, m.None2)
@@ -479,11 +479,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear;
// until then every merge is taken in at once, as before.
Pending: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.PendingState), "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)},
SayTakeIn: sayTakeIn(js, cred),
// The validation's run ask (design 47 G §713, A-247, A-248), put under the indexer's own asker.
ValidationAsks: buildrun.Asker{Name: index.ValidationAsker, Publisher: buildrun.JetStreamAsks{JS: js}},
// The build seat's verb output, for a validation's answer kept as an object (A-254).
AskVerb: calls.CallFile}
SayTakeIn: sayTakeIn(js, cred)}
// The registrations and the changes that wait for their warrant (design 47 G §2, §667): their states are bound when they
// appear; a warrant is heard on the router's decided event for this module, below.
registry := newRegistry(js, conn, cred, calls.Call,
@@ -206,6 +206,7 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T)
if pub.Commit != m2 || len(pub.Packages) != 3 {
t.Fatalf("packages.%s = %+v within 20 s, want the merge %s with a digest of go, announce and buildstep", key, pub, m2)
}
// A-476a: mesh-sdk's merge moves importer, of another repository, which imports a package it moved.
moved := module("importer", func(m meshcatalog.ModulesByModuleValue) bool { return m.Fp2 != importer.Fp2 })
if moved.Packages[sdkGo] != m2 {
t.Errorf("importer moved to %+v, want it built against %s", moved.Packages, m2)
@@ -28,7 +28,6 @@ type takerIn interface {
NamedDrop(ctx context.Context, fingerprint, member, by, why string) index.TakenIn
Unpin(ctx context.Context, revision, record, by, why string) index.TakenIn
DropRunByHand(ctx context.Context, run, by, why string) index.TakenIn
ValidateAgainByHand(ctx context.Context, key, by, why string) index.TakenIn
RegistryAct(ctx context.Context, a index.TakeInAsk) index.TakenIn
}
@@ -75,9 +74,6 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream
} else if a.Merge != "" {
asked, r = "hand take-in of a merge", ix.TakeInMerge(ctx, a.MergeRepository, a.Merge, a.Sequence, a.By, a.Why)
a.Module = r.Module
} else if a.Pending != "" && a.ValidateAgain {
asked, r = "validation asked again", ix.ValidateAgainByHand(ctx, a.Pending, a.By, a.Why)
a.Module = a.Pending
} else if a.Pending != "" && a.Drop {
asked, r = "drop of a pending merge", ix.DropPending(ctx, a.Pending, a.By, a.Why, a.Abandon)
a.Module = a.Pending
@@ -7,8 +7,6 @@ import (
"sync/atomic"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
@@ -155,15 +153,6 @@ func heardStepEnded(ctx context.Context, ix *index.Indexer, m jetstream.Msg, log
_ = m.Ack()
return
}
// The end of a validation ask resumes its pending revision (design 47 G §713); its own fault is given again by the bus.
var end buildstep.StepEnded
if err := json.Unmarshal(m.Data(), &end); err == nil {
if err := ix.HearValidationEnded(ctx, end); err != nil {
logf("take-in: the end of the validation ask %s is not taken in, so the bus gives it again: %v", end.Ask, err)
leftForBus(m, "the end of the validation ask "+end.Ask, err, logf)
return
}
}
if err := ix.HearInputEnded(ctx, e); err != nil {
logf("inputs: the input of the ask %s could not be deleted, so the bus gives it again: %v", e.Ask, err)
leftForBus(m, "the step-ended of the ask "+e.Ask, err, logf)
@@ -91,7 +91,7 @@ func (ix *Indexer) TakeInBehind(ctx context.Context) []string {
}
switch {
case held[head]:
// A head the pending holds (a member waiting for its group's end or its build) is taken in by that path, whole with
// A head the pending holds (a member waiting for its group's end) is taken in by that path, whole with
// its group: it is not behind (A-463a).
ix.clearUntaken(ctx, key, "the head of "+repository+" waits in the indexer's pending")
case merge == head:
@@ -6,7 +6,6 @@ import (
"context"
"errors"
"fmt"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/stateobjects"
"reflect"
"slices"
@@ -234,12 +233,6 @@ type Indexer struct {
Pending Store
// SayTakeIn publishes a take-in event (take-in.pending, take-in.taken; ADR 0633 §5); nil says none.
SayTakeIn func(ctx context.Context, t takein.TakeIn) error
// ValidationAsks puts the validation's run ask (design 47 G §713, A-247: buildrun.Asker, named as the indexer's asker,
// A-248); nil validates nothing, and a revision is taken in as before.
ValidationAsks ValidationAsks
// AskVerb asks a seat verb as the runtime calls it: the build seat's verb output, for a run's answer the seat keeps as an
// object (design 47 G A-254, buildstep.StepEnded.ReadOutput); nil reads none, and such an answer could not validate.
AskVerb buildstep.AskVerb
// Runs is the state runs: the runs on request the indexer holds a candidate for (runcandidates.go, A-174); nil keeps none.
Runs Store
runMu sync.Mutex // one run event at a time (runcandidates.go)
@@ -0,0 +1,72 @@
package index
import (
"context"
"encoding/json"
"fmt"
"strings"
"sync/atomic"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/views"
)
// listingObjects counts the listings of the bucket: each costs the whole bucket (56 ms at 11 000 objects, more through the runtime).
type listingObjects struct {
*memObjects
listings atomic.Int64
}
func (l *listingObjects) Names(ctx context.Context) ([]string, error) {
l.listings.Add(1)
return l.memObjects.Names(ctx)
}
// A seat verb that answers from the whole view of the trunk (modules, provides, dependents, module) reads the catalog-revisions
// bucket's listing once per call, never once per module: the node-engines ask `module` for every module at once, and 33 asks
// in 5 minutes passed the tool runner's 30 s (2026-11-17).
func TestTrunkViewIsReadOnceForAWholeAnswer(t *testing.T) {
objects := &listingObjects{memObjects: newMemObjects(0)}
const modules = 40
var refs []meshcatalog.EntryRef
for m := 0; m < modules; m++ {
name := fmt.Sprintf("mod%02d", m)
digest := fmt.Sprintf("%064x", m)
e := meshcatalog.CatalogEntry{Module: name, Repository: "o/r", Path: "m/" + name, Commit: "c", Fp: "src2:" + strings.Repeat("a", 64),
Manifest: map[string]any{"name": name}, ManifestDigest: "sha256:" + digest}
b, _ := json.Marshal(e)
if err := objects.Put(context.Background(), EntryObject(digest), b); err != nil {
t.Fatal(err)
}
refs = append(refs, meshcatalog.EntryRef{Module: name, Digest: digest})
}
b, _ := json.Marshal(meshcatalog.CatalogRevision{Kind: "trunk", Revision: 1, Entries: refs})
if err := objects.Put(context.Background(), "trunk.1", b); err != nil {
t.Fatal(err)
}
ts := TrunkStates{Objects: objects, Under: func(string) views.Source { return nil }, Kept: &KeptTrunk{}}
for _, state := range []string{"modules", "graph"} {
objects.listings.Store(0)
values, err := SourceValues(ts.State(state))
if err != nil || len(values) != modules {
t.Fatalf("the %s of the view = %d values (%v), want %d", state, len(values), err, modules)
}
if n := objects.listings.Load(); n > 2 {
t.Errorf("a whole read of the %s listed the bucket %d times, want at most 2 (once per call, not once per module)", state, n)
}
}
if _, err := meshcatalogclient.DecodeModule(mustValue(t, ts)); err != nil {
t.Errorf("a value of the view does not decode: %v", err)
}
}
func mustValue(t *testing.T, ts TrunkStates) []byte {
t.Helper()
v, err := SourceValues(ts.State("modules"))
if err != nil {
t.Fatal(err)
}
return v["mod00"]
}
@@ -691,6 +691,57 @@ func (r revisionSource) Get(key string) (*views.Entry, bool, error) {
return &views.Entry{Value: value, Revision: uint64(at)}, true, nil
}
// Values is every value of the modules (or the module graph) of the newest trunk revision from one read of it: a seat verb
// that answers from the whole view asks which revision is newest once, never once per key (each ask lists the whole
// catalog-revisions bucket: 6 s for the 110 modules of a bucket of 11 000 objects, and 35 s with 12 callers at once).
func (r revisionSource) Values() (map[string][]byte, error) {
s, _, err := r.t.read(r.nodes)
if err != nil {
return nil, err
}
ctx := context.Background()
keys, err := s.Keys(ctx)
if err != nil {
return nil, err
}
out := make(map[string][]byte, len(keys))
for _, k := range keys {
value, _, found, err := s.Get(ctx, k)
if err != nil {
return nil, err
}
if found {
out[k] = value
}
}
return out, nil
}
// SourceValues is every key's value of a source: in one read where the source offers it (the view of the trunk, a
// watched copy of a state), else a get per key.
func SourceValues(src views.Source) (map[string][]byte, error) {
if all, ok := src.(interface {
Values() (map[string][]byte, error)
}); ok {
return all.Values()
}
keys, err := src.Keys()
if err != nil {
return nil, err
}
out := make(map[string][]byte, len(keys))
for _, k := range keys {
e, ok, err := src.Get(k)
if err != nil {
return nil, fmt.Errorf("%s: %w", k, err)
}
if ok && e != nil {
out[k] = e.Value
}
}
return out, nil
}
// Page is the values in key order after a key from one read of the revision, as many as fit maxBytes and at least one
// (views.Pager): a first view's page reads the revision once, never once per key.
func (r revisionSource) Page(after string, maxBytes int, _ time.Duration) ([]views.KeyedEntry, string, error) {
@@ -60,7 +60,7 @@ func (ix *Indexer) FingerprintsAgain(ctx context.Context, module string) TakenIn
ix.freshMu.Unlock()
// The path of a merge's recompute: the modules, then the candidates their fingerprints reach (recomputeWith), then the
// pending merges a changed fingerprint settles (PendingAgain, as after a toolchain report).
err = ix.recomputeWith(ctx, nil, nil)
err = ix.recomputeWith(ctx, nil, nil, "")
ix.freshMu.Lock()
ix.fresh = nil
ix.freshMu.Unlock()
@@ -144,7 +144,7 @@ func (ix *Indexer) applyMoving(ctx context.Context, f forgemsg.Forge, drop bool,
ix.logf("%s at %s: its packages are not published: %v", f.Repository, f.Merge, err)
}
// What this merge moved reaches the modules standing on it (a base's tree, build-agent's fingerprint), in the graph's order.
return w, ix.recomputeWith(ctx, overlay, w)
return w, ix.recomputeWith(ctx, overlay, w, f.Repository)
}
// baseFP answers the new-form fingerprint of a base module as modules.<base> holds it, or why it has none.
@@ -13,7 +13,6 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
@@ -23,13 +22,10 @@ import (
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-indexer/judge"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
)
// The take-in step (novox/hq ADR 0633): a source revision is taken in only by the indexer it names. A merged message is
// kept in the indexer's own state pending, one key per merge, until the take-in that ends it; a revision naming another
// build of the indexer writes only what building and moving that indexer needs, and the rest stays pending for it.
// The take-in step (novox/hq ADR 0633, A-476): a merged message is kept in the indexer's own state pending, one key per
// merge, until the take-in that ends it; a revision is taken in by the running indexer whatever build of the indexer it names.
const (
// PendingState is the indexer's state of held merges: the bucket mesh-delivery-indexer_pending, keyed
// takein.PendingKey, so the key order is the merge order. An old `state` entry (the indexer is foundation, ADR
@@ -48,10 +44,6 @@ var errClaimLost = errors.New("another take-in claimed the merges first, so this
// runs such a build, so the revision waits until a later merge gives it one or a hand take-in (indexer_take_in_pending).
const NoBuild = "no build: "
// ErrOwnBuildUnknown is what a merged message answers while the indexer cannot learn the build it runs (ADR 0633 §1):
// the message is left unacknowledged, so the bus delivers it again, and nothing is written.
var ErrOwnBuildUnknown = errors.New("the build of the indexer this program runs is not known")
// PendingRef is what a pending value keeps of its merged message instead of a copy (a merge of a repository with many
// modules is about 0.5 MB, over the state's 256 KiB value cap; issue 1585): the small fields the waits are worked out
// from. The message itself is read from the forge bridge's stream by Pending.Seq when a take-in needs it.
@@ -98,8 +90,6 @@ type Pending struct {
// Held is why a ready merge is not taken in although its revision names this build: objects full (HeldObjectsFull),
// until room is said (RoomAgain).
Held string `json:"held,omitempty"`
// Validation is the revision's validation (design 47 G §713, A-247), on the revision's newest value alone.
Validation *takein.Validation `json:"validation,omitempty"`
}
// PendingEntry is one key of the pending state as read: its key, revision, value and the merged message decoded. For a
@@ -389,26 +379,13 @@ func (ix *Indexer) pendingOnBus(ctx context.Context) bool {
}
// ownBuildNow is the build this program runs, as FixOwnBuild fixed it at the start: empty while unknown, and never asked again.
// It names only the writer of what is written (LastWriter); nothing waits for it (A-476 b).
func (ix *Indexer) ownBuildNow(context.Context) string {
ix.ownMu.Lock()
defer ix.ownMu.Unlock()
return ix.ownFP
}
// sayOwnBuildUnknown says, for a merge this process cannot take in because it does not know its own build (ADR 0633 §1), a
// take-in.pending that names why and since when, so the watchman's take-in-waits bound covers it. The merge is not written to
// pending; the stream delivers it again to the process that knows its build.
func (ix *Indexer) sayOwnBuildUnknown(ctx context.Context, seq uint64, f forgemsg.Forge) {
ix.ownMu.Lock()
since, why := ix.ownSince, ix.ownWhy
ix.ownMu.Unlock()
if since.IsZero() { // never fixed: the start did not run (no engine wired)
since, why = ix.now(), "the build this program runs was never asked"
}
ix.sayTakeIn(ctx, takein.TakeIn{Kind: takein.TakeInPending, Key: takein.PendingKey(seq), What: takein.TakeInRevision,
Merges: []takein.Merge{{Repository: f.Repository, Number: f.Number, Merge: f.Merge}}, Since: since, Unknown: why})
}
// sayTakeIn publishes a take-in event and keeps what it said on the entries, so indexer_pending reads the last event of
// each wait from the state. A publish that fails is said in the log: the next evaluation says it again.
func (ix *Indexer) sayTakeIn(ctx context.Context, t takein.TakeIn) bool {
@@ -456,10 +433,6 @@ func (ix *Indexer) Arrive(ctx context.Context, seq uint64, f forgemsg.Forge) (Ap
return a, "", err
}
own := ix.ownBuildNow(ctx)
if own == "" {
ix.sayOwnBuildUnknown(ctx, seq, f)
return Applied{}, "", ErrOwnBuildUnknown
}
ix.pendingMu.Lock()
defer ix.pendingMu.Unlock()
key := takein.PendingKey(seq)
@@ -502,9 +475,6 @@ func (ix *Indexer) EndGroup(ctx context.Context, end forgemsg.ForgeGroupEnd) (Ap
return Applied{}, "no pending state: the group's merges were taken in one by one", nil
}
own := ix.ownBuildNow(ctx)
if own == "" {
return Applied{}, "", ErrOwnBuildUnknown
}
ix.pendingMu.Lock()
defer ix.pendingMu.Unlock()
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
@@ -677,21 +647,15 @@ func (ix *Indexer) takenAlready(ctx context.Context, m forgemsg.GroupMerge) (boo
}
// TakeInAtStart is the take-in at the start (ADR 0633 §2), before the consumer on the forge bridge's stream starts:
// it evaluates pending once with the build this program runs, taking in the whole revisions that name this build; a
// group still being gathered stays, and a revision naming yet another build is held for that one. A claim left by an
// indexer that ran before is void here. When the build is not known (the node-engine did not answer), it waits for
// nothing: it says so and the consumer starts all the same; a merged message is then left unacknowledged until the
// build is known (clause 1), and its redelivery, or a toolchain report (ToolchainReported), asks again. No timer.
// it evaluates pending once, taking in the whole revisions whatever build of the indexer they name (A-476 b); a group
// still being gathered stays. A claim left by an indexer that ran before is void here. The build this program runs, known
// or not, only names the writer. No timer.
func (ix *Indexer) TakeInAtStart(ctx context.Context) (Applied, string, error) {
if ix.Pending == nil || !ix.pendingOnBus(ctx) {
// Nothing can wait in a state not on the bus: the start asks no build and does not wait for one.
return Applied{}, "the pending state is not on the bus yet: nothing waits", nil
}
own := ix.ownBuildNow(ctx)
if own == "" {
ix.logf("take-in: the build this program runs is not known at its start, so nothing pending is taken in now; the forge consumer starts, and a merged message left unacknowledged asks it again")
return Applied{}, "the build this program runs is not known yet: nothing pending is taken in at the start", nil
}
ix.pendingMu.Lock()
defer ix.pendingMu.Unlock()
a, words, err := ix.evaluate(ctx, own, true)
@@ -699,8 +663,8 @@ func (ix *Indexer) TakeInAtStart(ctx context.Context) (Applied, string, error) {
return a, "the pending state is not on the bus yet: nothing waits", nil
}
if err == nil && len(a.Written) == 0 {
// A start that takes nothing in says so, with the build it learnt (the wait's own build is in the line holdFor says).
ix.logf("take-in: at the start, this program runs build %s and took nothing in: %s", own, words)
// A start that takes nothing in says so.
ix.logf("take-in: at the start, this program (build %q) took nothing in: %s", own, words)
}
return a, words, err
}
@@ -731,16 +695,12 @@ func (ix *Indexer) UnreadAgain(ctx context.Context) error {
}
// PendingAgain evaluates pending again after something a fingerprint is computed from changed, a toolchain report
// (ADR 0633 §1, §3), or the verdicts state bound (issue 1966): a revision that now names the running build is taken in, and one that names another build is held
// for that one. It does nothing while the pending state is not on the bus or the build this program runs is not known.
// (ADR 0633 §3), or the verdicts state bound (issue 1966): what waits is taken in. It does nothing while the pending state is not on the bus.
func (ix *Indexer) PendingAgain(ctx context.Context) error {
if ix.Pending == nil || !ix.pendingOnBus(ctx) {
return nil
}
own := ix.ownBuildNow(ctx)
if own == "" {
return nil
}
ix.pendingMu.Lock()
defer ix.pendingMu.Unlock()
_, words, err := ix.evaluate(ctx, own, false)
@@ -786,31 +746,37 @@ func (ix *Indexer) evaluate(ctx context.Context, own string, start bool) (Applie
if len(lost) > 0 {
return Applied{}, ix.waitingLine(w), ix.holdLost(ctx, w.Ready, lost, reasons, unread)
}
if wait, words, err := ix.validationGate(ctx, ready, own); wait || err != nil {
return Applied{}, words, err
}
names, dry, err := ix.computeOver(ctx, ready)
dry, err := ix.computeOver(ctx, ready)
if err != nil {
return Applied{}, "", err
}
if words, refused, err := ix.refuseCycle(ctx, ready, dry); refused || err != nil {
return Applied{}, words, err
}
if names == own {
a, err := ix.takeInWhole(ctx, ready, own, "")
if errors.Is(err, errClaimLost) {
return a, "claimed by another take-in", nil
}
if errors.Is(err, errVerdictsNotYet) {
return a, "the verdicts state is not on the bus yet: the revision waits for it, taken in when it binds", nil
}
if errors.Is(err, ErrObjectsFull) {
ix.holdFull(ctx, ready)
return a, ix.waitingLine(w) + "; held: " + HeldObjectsFull, nil
}
return a, ix.waitingLine(w), err
// The indexer takes in with the build it runs, whatever build of the indexer the revision names (A-476 b).
a, err := ix.takeInWhole(ctx, ready, own, "")
if errors.Is(err, errClaimLost) {
return a, "claimed by another take-in", nil
}
return Applied{}, ix.waitingLine(w), ix.holdFor(ctx, ready, names, own, dry)
if errors.Is(err, errVerdictsNotYet) {
return a, "the verdicts state is not on the bus yet: the revision waits for it, taken in when it binds", nil
}
if errors.Is(err, ErrObjectsFull) {
ix.holdFull(ctx, ready)
return a, ix.waitingLine(w) + "; held: " + HeldObjectsFull, nil
}
return a, ix.waitingLine(w), err
}
// newestOfRevision is the index in ready of the revision's newest merge.
func newestOfRevision(ready []PendingEntry) int {
n := 0
for i, e := range ready {
if e.Value.Seq > ready[n].Value.Seq {
n = i
}
}
return n
}
// refuseCycle refuses the revision whose catalog graph would hold a cycle with a build edge the take-in refuses (design 47 G
@@ -951,7 +917,6 @@ func (ix *Indexer) dry() *dryRun {
x := &Indexer{Module: ix.Module, Manifests: ix.Manifests, Packages: ix.Packages, JudgeID: ix.JudgeID, Files: ix.Files, GoSum: ix.GoSum,
GoMod: ix.GoMod, Now: ix.Now, Builds: ix.Builds, Named: ix.Named, Trunk: ix.Trunk, BranchHead: ix.BranchHead, Read: ix.Read, Contains: ix.Contains,
ReadOpen: nil, OwnBuild: nil, Message: ix.Message, SayRefused: nil, SayAccepted: nil, SayTakeIn: nil, Pending: nil,
ValidationAsks: nil, AskVerb: nil, // a dry run asks no validation (§713)
// The registry milestone's fields: what a take-in would write on the bus stays out of a dry run, so the states of the
// requirements, findings, seat definitions, holds and their bounds are not overlaid, nothing is said, and no hold is
// weighed (Holds and Counted nil hold nothing); ReadFile only reads.
@@ -991,16 +956,15 @@ func (ix *Indexer) dry() *dryRun {
}
// computeOver applies the ready merges, in merge order, to a dry indexer, as one overlay: each later merge's trees
// govern (ADR 0633 §3). It answers the fingerprint the indexer's own module has once they are applied (none is "",
// with its reason), and the dry run, whose overlays hold what the take-in would write. A merge that can never be read
// govern (ADR 0633 §3). It answers the dry run, whose overlays hold what the take-in would write. A merge that can never be read
// is left out, as its take-in will drop it.
func (ix *Indexer) computeOver(ctx context.Context, ready []PendingEntry) (string, *dryRun, error) {
func (ix *Indexer) computeOver(ctx context.Context, ready []PendingEntry) (*dryRun, error) {
d := ix.dry()
// The manifests are judged against the seat definitions the revision carries (seatsof.go, ADR 0633 §1), so the indexer
// it names is not refused for a verb or an event the built-in list lacks.
seats, err := ix.revisionSeats(ctx, ready)
if err != nil {
return "", d, fmt.Errorf("reading the seat definitions of %s: %w", ready[0].Key, err)
return d, fmt.Errorf("reading the seat definitions of %s: %w", ready[0].Key, err)
}
var applyErr error
judge.WithSeats(seats, func() {
@@ -1018,24 +982,9 @@ func (ix *Indexer) computeOver(ctx context.Context, ready []PendingEntry) (strin
}
})
if applyErr != nil {
return "", d, applyErr
return d, applyErr
}
key, _ := meshcatalogclient.ModulesKey(ix.Module)
value, _, found, err := d.ix.Modules.Get(ctx, key)
if err != nil {
return "", d, err
}
if !found {
return NoBuild + "the indexer holds no record of itself", d, nil
}
m, err := meshcatalogclient.DecodeModule(value)
if err != nil {
return "", d, nil
}
if m.Fp2 == "" {
return NoBuild + m.None2, d, nil
}
return m.Fp2, d, nil
return d, nil
}
// claim marks the entries as taken in by this take-in, by compare-and-set on the revisions read. A lost one (another
@@ -1238,8 +1187,7 @@ func (ix *Indexer) nextSource(branch string) int64 {
// takenWaits are the taken events of a take-in: every wait the entries were said under (saidWaits), every wait
// indexer_pending names the claimed entries by now (WaitsOf: a revision by its oldest merge's key, a group by its g. key) and
// the key of every claimed entry, since indexer_pending named it by that key earlier when it waited behind a group. So a wait
// the watchman learned of without the indexer marking it (seeded from indexer_pending, or said by the own-build-unknown
// path) ends too (design 47, Taken in by the indexer it names). No key is said twice.
// the watchman learned of without the indexer marking it (seeded from indexer_pending) ends too. No key is said twice.
func takenWaits(entries []PendingEntry) []takein.TakeIn {
out := saidWaits(entries)
said := map[string]bool{}
@@ -1298,83 +1246,6 @@ func saidWaits(entries []PendingEntry) []takein.TakeIn {
return out
}
// holdFor keeps the ready merges pending for the indexer build they name and writes only what building and moving
// that indexer needs (ADR 0633 §1): the indexer's own module entry, its key of the module graph and the verdict on its
// manifest, from the dry run; the manifests and the packages are written by the dry run itself. The revision is said
// pending under its oldest merge's key, and a wait it was said under before by another key is said taken.
func (ix *Indexer) holdFor(ctx context.Context, ready []PendingEntry, names, own string, d *dryRun) error {
key, _ := meshcatalogclient.ModulesKey(ix.Module)
if err := copyKey(ctx, d.modules, ix.Modules, key); err != nil {
return fmt.Errorf("the indexer's own module entry: %w", err)
}
if gk, err := graph.Key(ix.Module); err == nil {
if err := copyKey(ctx, d.graph, ix.Graph, gk); err != nil && !errors.Is(err, process.ErrNotYet) {
return fmt.Errorf("the indexer's key of the module graph: %w", err)
}
}
// Why the manifest of the indexer the revision names was refused, when it was (judged with the seat definitions the
// revision carries): said with the wait, as a merge the forge bridge could not give is (holdLost).
refused := ""
if value, _, found, err := d.ix.Modules.Get(ctx, key); err == nil && found {
if m, err := meshcatalogclient.DecodeModule(value); err == nil && m.Manifest != "" {
if vk, err := meshcatalogclient.VerdictsKey(m.Manifest); err == nil {
if err := copyKey(ctx, d.verdicts, ix.Verdicts, vk); err != nil && !errors.Is(err, process.ErrNotYet) {
return fmt.Errorf("the verdict on the indexer's manifest: %w", err)
}
if vv, _, ok, err := d.ix.Verdicts.Get(ctx, vk); err == nil && ok {
if v, err := meshcatalogclient.DecodeVerdict(vv); err == nil && meshcatalogclient.VerdictRefuses(v) {
refused = strings.Join(v.Problems, "; ")
}
}
}
}
}
// The indexer's own entry the held revision names is kept as the trunk revision after the newest, every other entry as
// it was (milestone 47/G W4.7: a merged revision naming a newer indexer replaces only the indexer's entry).
if err := ix.KeepView(ctx); err != nil {
return fmt.Errorf("the revision with the indexer's own entry: %w", err)
}
t := revisionWait(takein.TakeInPending, ready, names)
t.Refused = refused
var ended, unmarked []PendingEntry
said, changed := false, false
for _, e := range ready {
switch {
case e.Value.Said == t.Key && e.Value.SaidKind == takein.TakeInPending:
said = true
changed = changed || e.Value.WaitsFor != names || e.Value.Refused != refused
if e.Value.WaitsFor != names || e.Value.Refused != refused {
unmarked = append(unmarked, e)
}
case e.Value.Said != "" && e.Value.SaidKind == takein.TakeInPending:
ended = append(ended, e)
unmarked = append(unmarked, e)
default:
unmarked = append(unmarked, e)
}
}
for _, old := range saidWaits(ended) {
ix.sayTakeIn(ctx, old)
}
if said && !changed {
// Nothing to say on the bus, but the start decided nothing and a reader of the journal sees why: the build learnt, the build the wait names.
ix.logf("take-in: %d merges, oldest %s, still name the indexer build %s, this program runs %s: they wait, as already said", len(ready), t.Key, names, own)
}
if !said || changed {
ix.logf("take-in: %d merges, oldest %s, name the indexer build %s, not %s, which runs: they wait for it", len(ready), t.Key, names, own)
if refused != "" {
ix.logf("take-in: the manifest of the indexer they name is refused, judged with the seat definitions they carry, so no such indexer runs: %s", refused)
}
if !ix.sayTakeIn(ctx, t) {
ix.markSaid(ctx, unmarked, nil, func(p *Pending) { p.WaitsFor, p.LastWriter, p.Refused = names, own, refused })
return nil
}
unmarked = ready
}
ix.markSaid(ctx, unmarked, &t, func(p *Pending) { p.WaitsFor, p.LastWriter, p.Refused = names, own, refused })
return nil
}
// copyKey writes what the overlay holds of key to the store, when it wrote or deleted it.
func copyKey(ctx context.Context, from *overlayStore, to Store, key string) error {
if from == nil || to == nil {
@@ -1439,9 +1310,6 @@ func waitOf(entries []PendingEntry, what, behind string) takein.PendingWait {
if e.Value.Refused != "" {
pw.Refused = e.Value.Refused
}
if e.Value.Validation != nil {
pw.Validation = e.Value.Validation
}
if e.Value.Said != "" && e.Value.SaidAt.After(pw.SaidAt) {
pw.Said, pw.SaidAt = "take-in."+e.Value.SaidKind, e.Value.SaidAt
}
@@ -1450,9 +1318,9 @@ func waitOf(entries []PendingEntry, what, behind string) takein.PendingWait {
return pw
}
// TakeInPendingByHand is the hand take-in (indexer_take_in_pending, ADR 0633 §5): the wait named by its key, the
// ready revision (its m. key) or a group gathered so far (its g. key), is taken in with this indexer and its judge,
// whatever build it names, for when the named indexer can never run or the group's end never comes. by and why are
// TakeInPendingByHand is the hand take-in (indexer_take_in_pending, ADR 0633 §5, A-476, principle 4): the pending revision (its
// m. key) or the group gathered so far (its g. key) is taken in now with this indexer and its judge,
// for a revision whose message is unread or lost or held for room, or a group whose end never comes. by and why are
// said in the journal and on the take-in.taken event. What waited behind it is evaluated again after.
func (ix *Indexer) TakeInPendingByHand(ctx context.Context, key, by, why string) TakenIn {
r := TakenIn{Module: key}
@@ -106,6 +106,16 @@ func (b *takeInBed) merge(number int, c, avahi, indexer string) forgemsg.Forge {
return f
}
// arriveHeld arrives f while the verdicts state is not on the bus, so the take-in holds it (ADR 0668 §4): the way a merge is
// left pending now that no merge waits for the indexer build it names (A-476 b).
func (b *takeInBed) arriveHeld(t *testing.T, seq uint64, f forgemsg.Forge) {
t.Helper()
bound := b.ix.Verdicts
b.ix.Verdicts = &memStore{unbound: process.ErrNotYet}
defer func() { b.ix.Verdicts = bound }()
b.arrive(t, seq, f)
}
func (b *takeInBed) arrive(t *testing.T, seq uint64, f forgemsg.Forge) {
t.Helper()
b.publish(t, f)
@@ -176,63 +186,21 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
})
}
})
t.Run("T6 a merge while the own build is unknown writes nothing", func(t *testing.T) {
t.Run("T6 a merge while the own build is unknown is taken in all the same (A-476 b)", func(t *testing.T) {
b := newTakeInBed(t)
b.ix.ownFP, b.ix.ownFixed = "", false // this process is started again and asks the engine, which does not answer
b.ix.OwnBuild = func(context.Context) (string, error) { return "", errors.New("the engine does not answer") }
b.ix.FixOwnBuild(ctx)
b.said = nil
_, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", ""))
if !errors.Is(err, ErrOwnBuildUnknown) || len(b.keys()) != 0 || b.commit(t, "avahi") == sha('b') {
t.Errorf("err %v, pending %v, avahi at %s: want ErrOwnBuildUnknown and nothing written", err, b.keys(), b.commit(t, "avahi"))
if _, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", "tree-indexer-2")); err != nil || len(b.keys()) != 0 || b.commit(t, "avahi") != sha('b') {
t.Errorf("err %v, pending %v, avahi at %s: want the merge taken in at once", err, b.keys(), b.commit(t, "avahi"))
}
// The merge is said pending with why and since when (so the watchman's bound covers it), and indexer_pending says the
// build is unknown, from the own-build state.
if len(b.said) != 1 || b.said[0].Kind != takein.TakeInPending || b.said[0].Key != takein.PendingKey(1) || !strings.Contains(b.said[0].Unknown, "does not answer") ||
b.said[0].Since.IsZero() || b.said[0].WaitsFor != "" {
t.Errorf("said %+v, want one take-in.pending of m.1 saying why the own build is unknown", b.said)
} else if _, err := takein.EncodeTakeIn(b.said[0]); err != nil {
t.Errorf("the unknown-build event is refused by its own contract: %v", err)
if len(b.said) != 1 || b.said[0].Kind != takein.TakeInTaken {
t.Errorf("said %+v, want one take-in.taken and no wait said", b.said)
}
answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{})
if a, _ := answer.(takein.PendingAnswer); err != nil || !strings.Contains(a.OwnBuildUnknown, "does not answer") || a.OwnBuildSince.IsZero() {
t.Errorf("indexer_pending = %+v %v, want the build unknown with why and since", answer, err)
}
// Before the pending state is on the bus nothing waits: the start asks no build, and the merge is taken in at once,
// as before ADR 0633, though the build is unknown (TestIndexerWhole found the start waiting on it for ever).
b.ix.Pending = &memStore{unbound: process.ErrNotYet}
if _, _, err := b.ix.TakeInAtStart(ctx); err != nil {
t.Fatal(err)
}
if _, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", "")); err != nil || b.commit(t, "avahi") != sha('b') {
t.Errorf("no pending state: err %v, avahi at %s: want the merge taken in at once", err, b.commit(t, "avahi"))
}
})
t.Run("1966/AC195-2 AC195-3 a merge said pending with its build unknown and taken in on redelivery ends that wait under its key; a revision wait is said once", func(t *testing.T) {
b := newTakeInBed(t)
ownBuild := b.ix.OwnBuild
b.ix.ownFP, b.ix.ownFixed = "", false
b.ix.OwnBuild = func(context.Context) (string, error) { return "", errors.New("the engine does not answer") }
b.ix.FixOwnBuild(ctx)
b.said = nil
if _, _, err := b.ix.Arrive(ctx, 7, b.merge(30, sha('b'), "tree-avahi-2", "")); !errors.Is(err, ErrOwnBuildUnknown) {
t.Fatal(err)
}
b.ix.OwnBuild, b.ix.ownFP, b.ix.ownFixed = ownBuild, "", false
b.ix.FixOwnBuild(ctx)
b.arrive(t, 7, b.merge(30, sha('b'), "tree-avahi-2", "")) // the stream delivers it again
if want := []string{"pending " + takein.PendingKey(7), "taken " + takein.PendingKey(7)}; !slices.Equal(b.events(), want) || len(b.keys()) != 0 {
t.Fatalf("said %v, pending %v: want %v and nothing pending", b.events(), b.keys(), want)
}
// Two merges said pending under the oldest's key (waiting for a named indexer), then taken in together: one taken, under
// that key, and no entry's key said twice.
b.said = nil
b.arrive(t, 8, b.merge(31, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.arrive(t, 9, b.merge(32, sha('d'), "tree-avahi-4", "tree-indexer-2"))
b.said = nil
b.arrive(t, 10, b.merge(33, sha('e'), "tree-avahi-5", b.base.Trees["modules/"+testSelf]))
if want := []string{"taken " + takein.PendingKey(8), "taken " + takein.PendingKey(9), "taken " + takein.PendingKey(10)}; !slices.Equal(b.events(), want) || len(b.keys()) != 0 {
t.Errorf("said %v, pending %v: want %v, the wait's key (m.8, said pending) once and each other entry's own", b.events(), b.keys(), want)
if a, _ := answer.(takein.PendingAnswer); err != nil || a.OwnBuild != "" || a.OwnBuildSince.IsZero() || len(a.Waits) != 0 {
t.Errorf("indexer_pending = %+v %v, want no wait and no known build", answer, err)
}
})
t.Run("T7 and T3 a merge alone, an indexer change, a merge behind it and a revert", func(t *testing.T) {
@@ -259,42 +227,28 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
t.Fatalf("alone: avahi %s (was %s), pending %v, said %v: want taken in, said taken under its key (1966/AC195-1)", avahi1, avahi0, b.keys(), b.events())
}
b.said = nil
avahi1, verdicts := b.fp(t, "avahi"), len(b.ix.Verdicts.(*memStore).vals)
// T4: a change to the indexer writes only its entry (with its graph key and verdict); the rest is pending.
avahi1 := b.fp(t, "avahi")
// A-476 b: a merge naming another indexer build is taken in at once, with the build this program runs: the indexer's entry
// and avahi are written, nothing is held and said pending.
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
named := b.fp(t, testSelf)
if named == own || b.fp(t, "avahi") != avahi1 || b.commit(t, "avahi") != sha('b') {
t.Fatalf("indexer change: indexer %s (own %s), avahi %s at %s: want only the indexer written", named, own, b.fp(t, "avahi"), b.commit(t, "avahi"))
if named == own || b.fp(t, "avahi") == avahi1 || b.commit(t, "avahi") != sha('c') || len(b.keys()) != 0 {
t.Fatalf("indexer change: indexer %s (own %s), avahi %s at %s, pending %v: want the whole revision taken in at once", named, own, b.fp(t, "avahi"), b.commit(t, "avahi"), b.keys())
}
if g := graphOf(t, b.ix, testSelf); g.FP != named {
t.Errorf("graph of the indexer names %s, want %s", g.FP, named)
}
if n := len(b.ix.Verdicts.(*memStore).vals); n > verdicts+1 {
t.Errorf("verdicts %d, want at most %d: only the new indexer manifest's", n, verdicts+1)
if want := []string{"taken " + takein.PendingKey(2)}; !slices.Equal(b.events(), want) {
t.Errorf("said %v, want %v: nothing said pending", b.events(), want)
}
if want := []string{takein.PendingKey(2)}; !reflect.DeepEqual(b.keys(), want) || !slices.Equal(b.events(), []string{"pending " + takein.PendingKey(2)}) || b.said[0].WaitsFor != named {
t.Fatalf("pending %v, said %+v: want m.2 pending for %s", b.keys(), b.said, named)
}
// T5: the merge delivered again writes no second key and says nothing again.
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
if len(b.keys()) != 1 || len(b.said) != 1 {
t.Errorf("redelivered: pending %v, said %v", b.keys(), b.events())
}
// A later merge joins: the newest governs and still names the new indexer.
// A later merge is taken in at once too, and a revert to the first indexer.
b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
if len(b.keys()) != 2 || b.commit(t, "avahi") != sha('b') || len(b.said) != 1 {
t.Fatalf("behind: pending %v, avahi at %s, said %v: want both pending and nothing said again", b.keys(), b.commit(t, "avahi"), b.events())
}
// A revert brings the indexer back to the running build: everything is taken in, oldest first.
b.arrive(t, 4, b.merge(23, sha('e'), "tree-avahi-5", b.base.Trees["modules/"+testSelf]))
if len(b.keys()) != 0 || b.fp(t, testSelf) != own || b.commit(t, "avahi") != sha('e') {
t.Fatalf("revert: pending %v, indexer %s, avahi at %s: want all taken in", b.keys(), b.fp(t, testSelf), b.commit(t, "avahi"))
}
if want := []string{"pending " + takein.PendingKey(2), "taken " + takein.PendingKey(2), "taken " + takein.PendingKey(3), "taken " + takein.PendingKey(4)}; !slices.Equal(b.events(), want) {
t.Errorf("said %v, want %v", b.events(), want)
}
})
t.Run("IX22 a merged revision naming a newer indexer is the previous catalog graph revision with only the indexer's entry replaced", func(t *testing.T) { // design 47 G §273, §237
t.Run("IX22 a merged revision naming a newer indexer is a catalog graph revision with its entry and the merge's others replaced", func(t *testing.T) { // design 47 G §273, §237
b := newTakeInBed(t)
store := newMemObjects(0)
b.ix.Revisions = store
@@ -305,14 +259,14 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
if err != nil || before.Revision != 1 {
t.Fatalf("the view's revision %+v (%v), want revision 1", before, err)
}
// The merge changes avahi and the indexer: only the indexer is written, the revision is made at once.
// The merge changes avahi and the indexer: both are taken in at once (A-476 b), in a revision after the first.
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
after, _, err := newestTrunk(ctx, store)
if err != nil || after.Revision != 2 {
t.Fatalf("after the merge naming a newer indexer the newest revision is %+v (%v), want revision 2", after, err)
if err != nil || after.Revision <= before.Revision {
t.Fatalf("after the merge naming a newer indexer the newest revision is %+v (%v), want one after revision %d", after, err, before.Revision)
}
if changed := changedEntries(before.Entries, after.Entries); !slices.Equal(changed, []string{testSelf}) {
t.Errorf("the revision differs from the previous in %v, want the indexer's entry alone", changed)
if changed := changedEntries(before.Entries, after.Entries); !slices.Equal(changed, []string{"avahi", testSelf}) && !slices.Equal(changed, []string{testSelf, "avahi"}) {
t.Errorf("the revision differs from the previous in %v, want avahi and the indexer", changed)
}
})
t.Run("P1 a revision that adds a Go module its own indexer uses is computed against that module's packages at its merge", func(t *testing.T) {
@@ -353,71 +307,13 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
t.Fatal(err)
}
b.arrive(t, 2, f)
if len(b.said) != 1 || !strings.HasPrefix(b.said[0].WaitsFor, "src2:") {
t.Errorf("said %+v, want m.2 waiting for the indexer build its merge's packages give, not for a version no take-in has published", b.said)
if len(b.said) != 1 || b.said[0].Kind != takein.TakeInTaken || b.said[0].WaitsFor != "" || len(b.keys()) != 0 {
t.Errorf("said %+v, pending %v, want m.2 taken in at once", b.said, b.keys())
}
if _, ok := b.ix.publishedAll(ctx)[sibling]; !ok {
t.Errorf("the packages the revision publishes at its merge lack %s", sibling)
}
})
t.Run("K1 the indexer a revision names is judged with the seat definitions the revision carries", func(t *testing.T) {
// A successor's manifest invokes a verb of the node-engine's seat that the seat list lacks and the revision's
// definitions add (ADR 0633 §1, §6): refused without them, with its reason said on the wait and in indexer_pending;
// accepted with them, and the revision waits for the build it names.
const verb, contracts = "revision-verb", "modules/seat-contracts"
// The third row's definer is the Delivery domain's contracts module defining a Core seat: not taken (ADR 0636 §5).
for _, row := range []struct {
name string
defined bool
refused bool
module string
}{{"definitions the revision does not carry", false, true, ""}, {"definitions the revision carries", true, false, "seat-contracts"},
{"definitions of a seat outside the carrier's domain", true, true, "mesh-delivery-contracts"}} {
b := newTakeInBed(t)
f := b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")
f.Manifests = map[string]string{}
for dir, raw := range b.base.Manifests {
f.Manifests[dir] = raw
}
dir := "modules/" + testSelf
f.Manifests[dir] = strings.Replace(f.Manifests[dir], `"module": "`+testSelf+`"`,
`"module": "`+testSelf+`", "invokes": ["seat:node-engine.`+verb+`"]`, 1)
if row.defined {
f.Trees[contracts] = "tree-seat-contracts"
f.Locks = map[string]map[string]string{}
for d, l := range b.base.Locks {
f.Locks[d] = l
}
f.Locks[contracts] = map[string]string{}
f.Manifests[contracts] = `{"module":"` + row.module + `","version":"1.0","build":{"artifacts":[{"name":"go-seats","kind":"package","language":"go","paths":["go"],"series":"0.1","generated-from":"."}]}}`
b.ix.Files = func(_ context.Context, repository, commit string, dirs []string) (map[string]map[string]string, error) {
if repository != f.Repository || commit != f.Merge || !slices.Equal(dirs, []string{contracts + "/seats"}) {
t.Errorf("listed %v of %s at %s", dirs, repository, commit)
}
return map[string]map[string]string{dirs[0]: {"node-engine/seat.json": "1", "node-engine/fixtures/a.json": "2"}}, nil
}
b.ix.ReadFile = func(_ context.Context, _, _, p string) ([]byte, error) {
switch p {
case contracts + "/seats/node-engine/seat.json":
return []byte(`{"format":1,"seat":"node-engine","scope":"node","grain":"seat-list","decision":"d","domain":"Core","verbs":[{"name":"` + verb + `"}]}`), nil
}
return nil, nil
}
}
b.arrive(t, 2, f)
waits := PendingWaits(readPending(t, b))
refused := ""
if len(b.said) > 0 {
refused = b.said[0].Refused
}
if got := strings.Contains(refused, `serves no verb "`+verb+`"`); got != row.refused || (len(waits) == 1 && waits[0].Refused != refused) {
t.Errorf("%s: said %q, indexer_pending %+v: want a refusal naming the verb %v", row.name, refused, waits, row.refused)
}
if _, ok := catalogue.SeatNamed("node-engine"); !ok || judgeSeatsHave("node-engine", verb) {
t.Errorf("%s: the seats are not put back after the dry run", row.name)
}
}
})
t.Run("A-326 a revision whose catalog graph holds a cycle with a build edge no build seat breaks is refused, naming it", func(t *testing.T) {
// avahi's artifact needs the indexer's (kind 2), and the indexer is built on avahi (kind 3): avahi holds no build seat, so
// no running version breaks the cycle, and the revision is refused at take-in naming every artifact on it (design 47 G §49).
@@ -443,15 +339,15 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
t.Errorf("indexer_pending %+v, want the refusal naming the cycle", waits)
}
})
t.Run("T10 the named indexer takes the revision in at its start; a gathered group stays", func(t *testing.T) {
t.Run("T10 a start takes what waits in whatever its build; a gathered group stays", func(t *testing.T) {
b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
g := b.merge(30, sha('f'), "", "")
g.Repository, g.Trees, g.Manifests, g.Locks, g.Group = "novox/mesh-sdk", map[string]string{}, nil, nil, "feat/x"
g.Members = []meshdeliveryclient.Head{{Repository: "novox/mesh-sdk", Number: 30, Head: g.Head}, {Repository: "novox/mesh-host", Number: 4, Head: sha('9')}}
b.arrive(t, 5, g)
named := b.fp(t, testSelf)
// The new indexer starts; its build is not known (the engine does not answer at its start): a hand ask is answered
b.arriveHeld(t, 5, g)
own, _ := b.ix.OwnBuild(ctx)
// The indexer starts; its build is not known (the engine does not answer at its start): a hand ask is answered
// meanwhile (R5), and the start waits for nothing. It is never asked again, so it stays unknown until it is started again.
mk := func(ask func(context.Context) (string, error)) *Indexer {
n := &Indexer{Module: testSelf, Store: b.ix.Store, Modules: b.ix.Modules, Manifests: b.ix.Manifests, Graph: b.ix.Graph, Verdicts: b.ix.Verdicts,
@@ -462,39 +358,21 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
n.FixOwnBuild(ctx)
return n
}
// A start that decides nothing says so (issue 1921): the build it learnt, the build the held wait names, also when the
// wait was said before; and indexer_pending names the known build and since when.
oldBuild, _ := b.ix.OwnBuild(ctx)
oldStart := mk(func(context.Context) (string, error) { return oldBuild, nil })
b.logs.Reset()
if _, words, err := oldStart.TakeInAtStart(ctx); err != nil || len(b.keys()) != 2 || !strings.Contains(b.logs.String(), "still name the indexer build") ||
!strings.Contains(b.logs.String(), "at the start, this program runs build") {
t.Fatalf("a start that holds: %q %v, journal %q: want the build learnt and the build the wait names said", words, err, b.logs.String())
}
if answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{}); err != nil || answer.(takein.PendingAnswer).OwnBuild != oldBuild || answer.(takein.PendingAnswer).OwnBuildSince.IsZero() {
t.Errorf("indexer_pending = %+v %v, want the known build and since", answer, err)
}
asked := 0
next := mk(func(context.Context) (string, error) { asked++; return "", errors.New("not yet") })
// The build this program runs is not known (the engine does not answer at its start): nothing waits for it, so the start
// takes the revision in all the same (A-476 b), and a hand ask names what waits.
next := mk(func(context.Context) (string, error) { return "", errors.New("not yet") })
if r := next.TakeInPendingByHand(ctx, "m.nothing", "lead@shanks", "a test"); r.Stored || !strings.Contains(r.Answer, "nothing waits") {
t.Errorf("a hand ask while the own build is unknown: %+v", r)
t.Errorf("a hand ask for what does not wait: %+v", r)
}
if _, words, err := next.TakeInAtStart(ctx); err != nil || len(b.keys()) != 2 || !strings.Contains(words, "not known") {
t.Fatalf("a start with its build unknown: %q %v, pending %v: want it to answer at once and take nothing in", words, err, b.keys())
}
// The engine answers now, and the same process does not ask it again: it learnt nothing it did not know at its start.
next.OwnBuild = func(context.Context) (string, error) { asked++; return named, nil }
if _, words, err := next.TakeInAtStart(ctx); err != nil || asked != 1 || len(b.keys()) != 2 || !strings.Contains(words, "not known") {
t.Fatalf("the window of a replacement: %q %v, asked %d, pending %v: want the build fixed at the start, never learnt later", words, err, asked, b.keys())
}
// The module started again fixes the build at its start and takes the revision in.
next = mk(func(context.Context) (string, error) { return named, nil })
if _, _, err := next.TakeInAtStart(ctx); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(b.keys(), []string{takein.PendingKey(5)}) || b.commit(t, "avahi") != sha('c') {
t.Errorf("start: pending %v, avahi at %s: want the revision taken in, the group left", b.keys(), b.commit(t, "avahi"))
}
if answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{}); err != nil || answer.(takein.PendingAnswer).OwnBuild != own {
t.Errorf("indexer_pending = %+v %v, want the build the original program knows", answer, err)
}
})
t.Run("T8 T9 T11 a group gathered until its end, a merge behind it, one alone, an end again", func(t *testing.T) {
b := newTakeInBed(t)
@@ -544,7 +422,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
if _, words, err := b.ix.EndGroup(ctx, end); err != nil || !strings.Contains(words, "taken in already") {
t.Errorf("an end again: %q %v", words, err)
}
// T11 (the lead's row): a group whose revision changes the indexer: the group's wait is taken, then the revision pending.
// T11: a group whose revision changes the indexer: the group's wait is taken and the revision taken in at once (A-476 b).
b.said = nil
g := member(catalogueRepo, 40, sha('c'))
g.Group, g.Trees = "feat/two", b.merge(0, "", "", "tree-indexer-7").Trees
@@ -553,7 +431,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
if _, _, err := b.ix.EndGroup(ctx, forgemsg.ForgeGroupEnd{Group: "feat/two", Members: []forgemsg.GroupMerge{{Repository: catalogueRepo, Number: 40, Merge: sha('c')}}}); err != nil {
t.Fatal(err)
}
if want := []string{"pending g.feat/two", "taken g.feat/two", "pending " + takein.PendingKey(9)}; !slices.Equal(b.events(), want) {
if want := []string{"pending g.feat/two", "taken g.feat/two", "taken " + takein.PendingKey(9)}; !slices.Equal(b.events(), want) {
t.Errorf("a group naming another indexer: said %v, want %v", b.events(), want)
}
})
@@ -593,11 +471,8 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
}
held := func() *takeInBed {
b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
b.ix.ownMu.Lock()
b.ix.ownFP = b.fp(t, testSelf)
b.ix.ownMu.Unlock()
b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.arriveHeld(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
return b
}
// A probe that fails keeps the group's wait and stops nothing else: an unrelated merge is still taken in.
@@ -719,7 +594,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
})
t.Run("T5 two take-ins of one pending revision: the one that loses the claim writes nothing", func(t *testing.T) {
b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
named := b.fp(t, testSelf)
b.ix.ownMu.Lock()
b.ix.ownFP = named
@@ -774,8 +649,8 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
{"a take-in that fails", func(b *takeInBed) { b.mods.failWrite = once(func(string) bool { return true }) }},
} {
b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.arriveHeld(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
named := b.fp(t, testSelf)
b.ix.ownMu.Lock()
b.ix.ownFP = named
@@ -794,7 +669,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
})
t.Run("W4.6 a full store holds the merge with the reason objects full, says full once, and room takes it in", func(t *testing.T) {
b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
named := b.fp(t, testSelf)
b.ix.ownMu.Lock()
b.ix.ownFP = named
@@ -826,7 +701,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
// The same take-in into a store it fills to 85 % of its cap says filling once, and is taken in.
kept := store.used()
b2 := newTakeInBed(t)
b2.arrive(t, 2, b2.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b2.arriveHeld(t, 2, b2.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b2.ix.ownMu.Lock()
b2.ix.ownFP = b2.fp(t, testSelf)
b2.ix.ownMu.Unlock()
@@ -845,7 +720,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
t.Run("W4.12 a candidate's half dropped by hand frees its own entries, says room and takes the held merge in; the read tools list it; a recompute that moves nothing is equal", func(t *testing.T) {
bed := func(store *memObjects) *takeInBed {
b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.ix.ownMu.Lock()
b.ix.ownFP = b.fp(t, testSelf)
b.ix.ownMu.Unlock()
@@ -930,40 +805,10 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
t.Errorf("indexer_recompute = %+v (%v): want equal to the kept revision 2", got, err)
}
})
t.Run("T7 a toolchain report evaluates pending again: the revision then names the running build and is taken in", func(t *testing.T) {
b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
entries, err := ReadPending(ctx, b.pending, nil)
if err != nil {
t.Fatal(err)
}
// The build the revision names under the next toolchain, worked out on a dry indexer; this one runs it.
x := b.ix.dry().ix
for _, node := range []string{"novox", "ace"} {
key, _ := ToolchainKey(node, "go")
x.PutToolchain("mesh-build-go", key, toolchainValue(t, "go", "go1.28.0", "1"), false)
}
ready, _, _, _ := b.ix.loadNewest(ctx, WaitsOf(entries).Ready)
next, _, err := x.computeOver(ctx, ready)
if err != nil || next == "" || next == b.said[0].WaitsFor {
t.Fatalf("the build under the next toolchain: %q %v (the revision names %s)", next, err, b.said[0].WaitsFor)
}
b.ix.ownMu.Lock()
b.ix.ownFP = next
b.ix.ownMu.Unlock()
reportToolchain(t, b.ix, "mesh-build-go", "novox", "go", "go1.28.0")
if len(b.keys()) != 1 {
t.Fatalf("one node's report: pending %v, want the revision still held (the nodes disagree)", b.keys())
}
reportToolchain(t, b.ix, "mesh-build-go", "ace", "go", "go1.28.0")
if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('c') {
t.Errorf("after the toolchain report: pending %v, avahi at %s: want the revision taken in", b.keys(), b.commit(t, "avahi"))
}
})
t.Run("T12 T13 indexer_pending names the wait; a hand take-in and a drop say what they ended", func(t *testing.T) {
b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.arriveHeld(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
var answer takein.PendingAnswer
for _, tool := range PendingTools(b.pending, nil, nil, nil) {
if tool.Name == PendingTool {
@@ -975,7 +820,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
}
}
if len(answer.Waits) != 1 || answer.Waits[0].Key != takein.PendingKey(2) || !slices.Equal(answer.Waits[0].Holds, []string{testSelf, "avahi"}) ||
answer.Waits[0].Said != takein.TakeInPendingEvent || len(answer.Waits[0].Keys) != 2 {
len(answer.Waits[0].Keys) != 2 {
t.Fatalf("indexer_pending = %+v", answer)
}
// ADR 0668 §6: a later merge covers a pending one only when its commit has the pending merge's as an ancestor.
@@ -990,32 +835,47 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
}
history = "cd"
b.said = nil
bound := b.ix.Verdicts
b.ix.Verdicts = &memStore{unbound: process.ErrNotYet} // m.3 stays held while m.2 is dropped
if r := b.ix.DropPending(ctx, takein.PendingKey(2), "lead@shanks", "replaced by m.3", false); !r.Stored {
t.Fatalf("drop: %+v", r)
}
if len(b.said) < 2 || !b.said[0].Dropped || b.said[0].Key != takein.PendingKey(2) || b.said[0].By != "lead@shanks" || b.said[1].Key != takein.PendingKey(3) {
t.Fatalf("drop said %+v: want m.2 taken (dropped, by) and m.3 pending", b.said)
b.ix.Verdicts = bound
if len(b.keys()) != 1 || b.keys()[0] != takein.PendingKey(3) {
t.Fatalf("pending %v: want m.2 dropped and m.3 pending", b.keys())
}
// Issue 1966 (#650), ADR 0633 §5: the newest pending merge of its repository, which no taken merge covers, is not
// dropped; the answer names the read again. abandon drops it all the same (below).
// dropped; the answer names abandon, which drops it all the same (below).
if r := b.ix.DropPending(ctx, takein.PendingKey(3), "lead@shanks", "lost message", false); r.Stored ||
!strings.Contains(r.Answer, TakeInPendingTool) || len(b.keys()) != 1 {
!strings.Contains(r.Answer, "abandon") || len(b.keys()) != 1 {
t.Fatalf("a drop nothing covers: %+v, pending %v", r, b.keys())
}
// A pending revision is taken in by the tool, by its key (A-476, principle 4).
if _, _, err := NewPendingAsk(takein.PendingKey(3), "x", "lead@shanks", false, false, time.Unix(0, 0)); err != nil {
t.Fatalf("the ask to take a pending revision in: %v", err)
}
if r := b.ix.TakeInPendingByHand(ctx, takein.PendingKey(3), "lead@shanks", "its message was unread"); !r.Stored || len(b.keys()) != 0 || b.commit(t, "avahi") != sha('d') {
t.Fatalf("hand take-in of a revision: %+v, pending %v, avahi at %s", r, b.keys(), b.commit(t, "avahi"))
}
// A delivery group gathered is taken in by hand (A-476 b: only a group is).
b.ix.Pins = newMem()
b.ix.Revisions = newMemObjects(0) // A-448: the taken event names the catalog graph revision the take-in made
gm := b.merge(24, sha('f'), "tree-avahi-6", "")
gm.Group, gm.Head = "feat/h", sha('1')
gm.Members = []meshdeliveryclient.Head{{Repository: catalogueRepo, Number: 24, Head: sha('1')}, {Repository: "novox/mesh-sdk", Number: 25, Head: sha('2')}}
b.arrive(t, 5, gm)
b.said = nil
if r := b.ix.TakeInPendingByHand(ctx, takein.PendingKey(3), "lead@shanks", "the named indexer can never run"); !r.Stored {
if r := b.ix.TakeInPendingByHand(ctx, takein.GroupTakeInKey("feat/h"), "lead@shanks", "the group's end never comes"); !r.Stored {
t.Fatalf("hand take-in: %+v", r)
}
if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('d') || len(b.said) != 1 || b.said[0].Kind != takein.TakeInTaken || b.said[0].By != "lead@shanks" {
if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('f') || len(b.said) < 1 || b.said[0].Kind != takein.TakeInTaken || b.said[0].By != "lead@shanks" {
t.Errorf("hand take-in: pending %v, avahi at %s, said %+v", b.keys(), b.commit(t, "avahi"), b.said)
}
// §476, §493: a hand take-in's taken event carries the branch, the source revision it makes and its entries.
if len(b.said) == 1 {
if len(b.said) >= 1 {
got := b.said[0]
if got.Branch != TrunkBranch || got.Source <= 0 || len(got.Entries) != 1 || got.Entries[0].Module != "avahi" ||
got.Entries[0].Commit != sha('d') || got.Entries[0].Fingerprint == "" {
if got.Branch != TrunkBranch || got.Source <= 0 || len(got.Entries) != 2 || got.Entries[1].Module != "avahi" ||
got.Entries[1].Commit != sha('f') || got.Entries[1].Fingerprint == "" {
t.Errorf("hand take-in said branch %q source %d entries %+v: want main, a source revision and avahi at its commit", got.Branch, got.Source, got.Entries)
} else if newest, any, _ := newestTrunk(ctx, b.ix.Revisions); !any || got.Revision != newest.Revision {
t.Errorf("hand take-in said revision %d, want the revision it made, %d (A-448)", got.Revision, newest.Revision)
@@ -1025,7 +885,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
t.Errorf("the taken event is refused by its own contract: %v", err)
}
}
b.arrive(t, 4, b.merge(23, sha('e'), "tree-avahi-5", "tree-indexer-2"))
b.arriveHeld(t, 4, b.merge(23, sha('e'), "tree-avahi-5", "tree-indexer-2"))
if r := b.ix.DropPending(ctx, takein.PendingKey(4), "lead@shanks", "abandoned on the operator's word", true); !r.Stored || len(b.keys()) != 0 {
t.Errorf("an abandoned drop: %+v, pending %v", r, b.keys())
}
@@ -1062,8 +922,8 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
if _, _, found, _ := b.ix.Store.Get(ctx, key); !found {
t.Fatal("the group has no candidate")
}
b.arrive(t, 2, big(b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")))
b.arrive(t, 3, big(b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")))
b.arriveHeld(t, 2, big(b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")))
b.arriveHeld(t, 3, big(b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")))
raw, _, _, _ := b.pending.Get(ctx, takein.PendingKey(2))
if len(b.keys()) != 2 || len(raw) > 4096 || b.commit(t, "avahi") != sha('b') {
t.Fatalf("a big merge held: pending %v, value %d bytes, avahi at %s: want two small values", b.keys(), len(raw), b.commit(t, "avahi"))
@@ -1139,7 +999,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
b := newTakeInBed(t)
f := b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")
f.Words, f.Branch = strings.Repeat("x", 300*1024), "feat/g"
b.arrive(t, 2, f) // held by reference, for an indexer build that is not this one
b.arriveHeld(t, 2, f) // held by reference, with the verdicts state not on the bus
b.ix.Taken = newMem()
b.ix.Contains = func(context.Context, string, string, string) (bool, error) { return c.behind, nil }
data, _ := takein.EncodeTaken(takein.Taken{Repository: trunk.RepositoryName(catalogueRepo), Merge: sha('d')})
@@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
@@ -44,15 +43,13 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T
return []stdio.Tool{
{Name: PendingTool,
Description: "What waits for a take-in (ADR 0633): merges the indexer holds in its pending state instead of taking them in. " +
"A revision (what revision, its key its oldest merge's m.<sequence>) waits for the build of the indexer it names " +
"(waits-for): the running indexer wrote only what building and moving that indexer needs, and the new indexer takes " +
"it in at its start. A group (what group, key g.<group>) is a delivery group whose members are gathered until the " +
"A revision (what revision, its key its oldest merge's m.<sequence>) waits only while its message is unread or lost or the " +
"objects are full. A group (what group, key g.<group>) is a delivery group whose members are gathered until the " +
"forge bridge says its end; a merge in one of its members' repositories waits behind it (behind). Each names its " +
"merges, since when it waits, the modules it holds back (holds), the merges whose message the forge bridge's stream no " +
"longer holds (lost: they are not taken in; indexer_drop_pending drops them) and the last take-in event said of it. Empty waits: " +
"every merge is taken in. The watchman raises take-in-waits past 60 minutes for a revision, 30 for a group; " +
"indexer_take_in_pending and indexer_drop_pending steer a wait. own-build says which build the indexer knows it runs and own-build-since since when; own-build-unknown, when set, says the indexer cannot learn the build it " +
"runs (since when): it fixes it once at its start, so every merge waits (a take-in.pending that says so) until the module is started again.",
"indexer_take_in_pending and indexer_drop_pending steer a wait. own-build says which build the indexer knows it runs and own-build-since since when; it only names the writer, and no merge waits for it.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) {
var unread []string
@@ -63,7 +60,7 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T
return nil, err
}
answer := takein.PendingAnswer{Waits: PendingWaits(entries), Unread: unread}
// Whether this indexer knows the build it runs: when it does not, every merge waits, and this says since when and why.
// The build this indexer knows it runs, and since when.
if ownBuild != nil {
if v, _, found, err := ownBuild.Get(context.Background(), OwnBuildKey); err == nil && found {
var o OwnBuild
@@ -71,8 +68,6 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T
answer.OwnBuildSince = o.Since
if o.Known {
answer.OwnBuild = o.FP
} else {
answer.OwnBuildUnknown = o.Why
}
}
}
@@ -80,10 +75,10 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T
return answer, nil
}},
{Name: TakeInPendingTool,
Description: "Take a wait in by hand with the running indexer and its judge (ADR 0633 §5): the revision under its m. key, " +
"whatever build of the indexer it names, or a delivery group under its g. key with the members gathered so far. For " +
"when the named indexer can never run, or a group's end never comes. A hand act: who and why are said in the " +
"indexer's journal and on the take-in.taken event, which clears take-in-waits. indexer_pending names the keys.",
Description: "Take in now a pending revision (its m.<sequence> key), for a merge whose message is unread or lost or that is held " +
"for room, or a delivery group (its g.<group> key) with the members gathered so far, for when its end never comes (ADR 0633 §5). " +
"A hand act: who and why are said in the indexer's journal and on the take-in.taken event, which clears take-in-waits. " +
"indexer_pending names the keys.",
Input: map[string]any{"type": "object", "required": []string{"key", "why"}, "properties": map[string]any{
"key": map[string]any{"type": "string", "description": "the wait's key, m.<sequence> or g.<group>, as indexer_pending names it"},
"why": map[string]any{"type": "string", "description": "why it is taken in by hand, in plain words"}}},
@@ -112,33 +107,3 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T
Run: merge},
}
}
// ValidationTools is indexer_validate_again (class act, design 47 G §713): the tools bundle writes an ask in takeins and the
// indexer asks the revision's validation again; record says the hand act in the hand-act log before it is asked.
func ValidationTools(ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time, record func(what, why string) error) []stdio.Tool {
return []stdio.Tool{{Name: ValidateAgainTool,
Description: "Ask again the validation of a pending revision that could not validate (design 47 G §712, §713): the manifest " +
"validator's run on the build seat node-build-go ended could not run, exited other than 0, or answered not both cases. " +
"The revision is asked under its next try; the answer says the validation's state (the ask, its try, accepted, refused, " +
"or why it could not validate). A revision accepted, refused or whose ask is out is asked nothing: a refusal is asked " +
"again only by a later merge of its branch. A hand act, recorded through mesh-hand-acts before it is asked. " +
"indexer_pending names the keys and shows the validation whole.",
Input: map[string]any{"type": "object", "required": []string{"key", "why"}, "properties": map[string]any{
"key": map[string]any{"type": "string", "description": "the revision's wait key, m.<sequence>, as indexer_pending names it"},
"why": map[string]any{"type": "string", "description": "why it is asked again, in plain words"}}},
Run: func(args map[string]any) (any, error) {
str := func(k string) string { s, _ := args[k].(string); return s }
key, value, err := NewValidateAgainAsk(str("key"), str("why"), stdio.CallerOf(args), now())
if err != nil {
return nil, err
}
if err := record("asked the validation of the pending revision "+strings.TrimSpace(str("key"))+" again by hand", strings.TrimSpace(str("why"))); err != nil {
return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was asked: %w", err)
}
a, err := ask(key, value)
if err != nil {
return nil, err
}
return a.Result, nil
}}}
}
@@ -51,9 +51,6 @@ type TakeInAsk struct {
// Abandon, with Drop, drops a pending merge no later merge covers (ADR 0633 §5 asks a later merge; issue 1966, ADR
// 0668 §5): the hand act says why it is abandoned all the same.
Abandon bool `json:"abandon,omitempty"`
// ValidateAgain, with Pending, asks the validation of the revision under that key again (indexer_validate_again, design 47
// G §713): one that could not validate.
ValidateAgain bool `json:"validate-again,omitempty"`
// Merge asks the merge of MergeRepository at this commit to be taken in by hand (indexer_take_in_merge): its merged message
// is asked of the forge bridge and run through the take-in as a delivered one, under its stream sequence Sequence.
Merge string `json:"merge,omitempty"`
@@ -167,20 +164,6 @@ func NewMergeAsk(repository, merge string, sequence uint64, why, by string, at t
return "merge." + merge, data, err
}
// NewValidateAgainAsk is the value indexer_validate_again writes: its key in takeins, `pending.validate-again.<key>`, and the
// value. key is a revision's wait key, m.<sequence>, as indexer_pending names it.
func NewValidateAgainAsk(key, why, by string, at time.Time) (string, []byte, error) {
key = strings.TrimSpace(key)
if !strings.HasPrefix(key, "m.") {
return "", nil, errors.New("name the revision whose validation is asked again by its key, m.<sequence>, as indexer_pending names it")
}
if strings.TrimSpace(why) == "" {
return "", nil, errors.New("say why (why): a hand act is recorded through mesh-hand-acts")
}
data, err := json.Marshal(TakeInAsk{V: 1, Pending: key, ValidateAgain: true, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
return "pending.validate-again." + key, data, err
}
// NewPendingAsk is the value indexer_take_in_pending (drop false) or indexer_drop_pending (drop true) writes: its key
// in takeins, `pending.<take-in|drop>.<key>`, and the value. key is a wait's key as indexer_pending names it, for a
// drop one pending merge's; abandon drops one no later merge covers.
@@ -13,6 +13,7 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"mesh-delivery-indexer/judge"
@@ -127,24 +128,26 @@ type known struct {
// the order the bucket lists them in; modules that stand on each other have a cycle as their reason. A module is written
// only when its fingerprint or its reason changes, and only if its manifest is still the one computed from. A module with
// no kept inputs (a message that carried no manifest) is left as it is.
func (ix *Indexer) Recompute(ctx context.Context) error { return ix.recomputeWith(ctx, nil, nil) }
func (ix *Indexer) Recompute(ctx context.Context) error { return ix.recomputeWith(ctx, nil, nil, "") }
// recomputeWith is Recompute for a merge as well: the records the merge gives (overlay, by module key) stand in place
// of what the bucket holds, are computed with the others, and are written once, with their final fingerprint or reason
// (issue 1444: a merge writes each module once). A record of the overlay that has no kept inputs is written as it is. w is
// the merge's write (order.go), nil for none; it is told each module written.
func (ix *Indexer) recomputeWith(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite) error {
// the merge's write (order.go), nil for none; it is told each module written. A merge's recompute names its repository
// (A-476 c, A-476a): the modules registered for it are moved, and those of another repository only when they import a package it moved; "" moves every module (a toolchain
// report, a hand recompute).
func (ix *Indexer) recomputeWith(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite, repository string) error {
if err := ix.fillPaths(ctx); err != nil {
return err
}
if err := ix.recomputeModules(ctx, overlay, w); err != nil {
if err := ix.recomputeModules(ctx, overlay, w, repository); err != nil {
return err
}
// What the modules moved to reaches the candidates' fingerprints too (a base, the builder, a toolchain).
return ix.RecomputeCandidates(ctx)
}
func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite) error {
func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite, repository string) error {
if ix.Modules == nil {
return nil
}
@@ -153,11 +156,11 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]mesh
}
ix.recompute.Lock()
defer ix.recompute.Unlock()
return ix.recomputeLocked(ctx, overlay, w)
return ix.recomputeLocked(ctx, overlay, w, repository)
}
// recomputeLocked is recomputeModules with ix.recompute held by the caller (a hold that moves, holds.go).
func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite) error {
func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite, repository string) error {
if ix.Manifests == nil {
return ix.writeOverlay(ctx, overlay, nil, w)
}
@@ -298,6 +301,14 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc
r.tcs, r.pkgs = toolchainsUsed(toolchains), f.versions
}
}
if _, merged := overlay[module]; !merged && repository != "" && !trunk.SameRepository(k.cur.Repository, repository) &&
maps.Equal(k.held.Packages, map[string]string(r.pkgs)) {
// A module registered for another repository is moved by this repository's commit only when it imports a package the
// commit moved (A-476a, ADR 0593 §3); any other stands as held (A-476 c), and so it is for a base of this repository's.
r = result{fp: k.held.Fp2, none: k.held.None2, tcs: k.held.Toolchains, pkgs: sdkstep.PackageVersions(k.held.Packages)}
done[module] = r
return r
}
done[module] = r
k.cur.Inputs2 = &kept
return r
@@ -832,6 +832,8 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
// A-476a: a commit of mesh-sdk moves avahi, of another repository, which imports its package, and leaves mesh-tools,
// which imports nothing it moved.
a, _ := moduleOf(t, mods, "avahi")
if a.Fp2 == "" || !reflect.DeepEqual(a.Packages, map[string]string{sdk: commit}) {
t.Fatalf("after mesh-sdk's merge %.8s: fp2 %q none2 %q packages %v", commit, a.Fp2, a.None2, a.Packages)
@@ -841,12 +843,24 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
}
return a
}
// A-476 c: mesh-tools, of another repository than mesh-sdk, holds a fingerprint a recompute would move (here a stale one); a
// commit of mesh-sdk, which moved nothing it imports, leaves it as held, and only a full recompute moves it.
stale := tools
stale.Fp2 = "src2:" + strings.Repeat("5", 64)
mods.put("mesh-tools", mustEncode(t, stale))
tools = stale
first := sdkMerge(strings.Repeat("1", 40), "tree-sdk-1")
second := sdkMerge(strings.Repeat("2", 40), "tree-sdk-2")
if second.Fp2 == first.Fp2 {
t.Errorf("a mesh-sdk trunk merge kept avahi's fingerprint %q", first.Fp2)
}
n = second
if err := ix.Recompute(context.Background()); err != nil {
t.Fatal(err)
}
if o, _ := moduleOf(t, mods, "mesh-tools"); o.Fp2 == stale.Fp2 {
t.Errorf("a full recompute left mesh-tools at %q: the check above proves nothing", o.Fp2)
}
n, _ = moduleOf(t, mods, "avahi") // moved by the recompute too: its base mesh-tools moved
// Nodes that disagree keep the fingerprint already computed, with its toolchains (issue 1940, 2026-10-25 20:56: a
// stale report blanked every module and the controller's take-in refused modules already built); only a new
// fingerprint needs the agreement: a merge that moves the module's tree gives it none, naming the disagreement.
@@ -1,264 +0,0 @@
package index
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"slices"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"mesh-delivery-indexer/judge"
)
// The validation before take-in (design 47 G §712, A-213, A-232): for a pending revision of a watched branch the running
// indexer asks the seat node-build-go a `run` of its own manifest validator over two cases, change and trunk, and refuses only
// what the revision causes (ADR 0579 §2). The ask rides the check queue under the indexer's name, its kind `run` in the body.
const (
// validationProgram is the manifest validator's program, by its directory under the indexer entry's Go module: the indexer's
// own code, so no repository or module name is fixed here (ADR 0666 §5).
validationProgram = "cmd/mesh-judge"
// ValidationAsker is the asker the ask is put under (buildrun.Asker.Name, A-248): the indexer's seat name, mesh-catalog
// (design 47 G A-163, A-232).
ValidationAsker = meshcatalog.Seat
// validationCatalogSeat is the seat the indexer entry claims.
validationCatalogSeat = "mesh-catalog"
// The cases of the request, as the manifest validator names them.
validationChange = "change"
validationTrunk = judge.TrunkCase
)
// validationInputVerb is where the build seat's holder asks the input: the seat mesh-catalog's verb input (§354).
var validationInputVerb = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbInput
// validationRequest is the manifest validator's request for the ready merges of branch over the branch's current revision
// (§712): case change, every module of the revision by its entry's manifest, each one the merges carry replaced by the merged
// message's bytes (the newest merge governs) and one added for a new directory; case trunk, every module as it stands; the
// seat definitions the merges' trees carry; the packages and the predefined list as the take-in's own validator has them;
// no machines. It answers the indexer entry of the change case too, the one that claims the seat mesh-catalog.
func (ix *Indexer) validationRequest(ctx context.Context, branch string, ready []PendingEntry) (composer.Request, meshcatalog.CatalogEntry, error) {
var indexer meshcatalog.CatalogEntry
if ix.Revisions == nil {
return composer.Request{}, indexer, errors.New("the catalog graph is not kept here: no revision to validate over")
}
rev, found, err := newestOf(ctx, ix.Revisions, branch)
if err != nil {
return composer.Request{}, indexer, err
}
var entries []meshcatalog.CatalogEntry
if found {
if _, entries, err = ReadRevision(ctx, ix.Revisions, RevisionObject(rev)); err != nil {
return composer.Request{}, indexer, err
}
}
trunkCase := composer.Case{Name: validationTrunk, Manifests: map[string]composer.Manifest{}}
byPlace := map[string]string{} // <repository> <path> -> module
of := map[string]meshcatalog.CatalogEntry{}
for _, e := range entries {
raw := ix.manifestBytes(ctx, e.ManifestDigest)
if raw == "" {
b, err := json.Marshal(e.Manifest)
if err != nil {
return composer.Request{}, indexer, fmt.Errorf("the manifest of %s: %w", e.Module, err)
}
raw = string(b)
}
trunkCase.Manifests[e.Module] = composer.Manifest{Raw: json.RawMessage(raw)}
byPlace[placeKey(e.Repository, e.Path)] = e.Module
of[e.Module] = e
}
change := composer.Case{Name: validationChange, Manifests: maps.Clone(trunkCase.Manifests)}
for _, e := range ready {
if e.Covered {
continue
}
for _, dir := range slices.Sorted(maps.Keys(e.Forge.Manifests)) {
raw := e.Forge.Manifests[dir]
module, known := byPlace[placeKey(e.Forge.Repository, dir)]
if !known {
m, err := judge.ForInputs([]byte(raw))
if err != nil || m.Module == "" {
module = e.Forge.Repository + "/" + dir // a new directory whose manifest names no module: judged under its place
} else {
module = m.Module
}
byPlace[placeKey(e.Forge.Repository, dir)] = module
of[module] = meshcatalog.CatalogEntry{Module: module, Repository: e.Forge.Repository, Path: dir}
}
change.Manifests[module] = composer.Manifest{Raw: json.RawMessage(raw)}
}
}
for _, module := range slices.Sorted(maps.Keys(change.Manifests)) {
m, err := judge.ForInputs(change.Manifests[module].Raw)
if err != nil || !slices.ContainsFunc(m.Claims, func(c catalogue.Claim) bool { return c.Name == validationCatalogSeat }) {
continue
}
indexer = of[module]
if indexer.Module == "" {
indexer.Module = module
}
break
}
seats, err := ix.revisionDefinitions(ctx, ready)
if err != nil {
return composer.Request{}, indexer, err
}
req := composer.Request{V: composer.Version, Input: json.RawMessage("{}"), Cases: []composer.Case{change, trunkCase}, Seats: seats,
Predefined: ix.predefinedList(ctx)}
if known := ix.knownPackages(ctx); known != nil {
req.Packages = slices.Sorted(maps.Keys(known))
}
return req, indexer, nil
}
func placeKey(repository, path string) string {
return strings.ToLower(trunk.RepositoryName(repository)) + " " + path
}
// revisionDefinitions are the validation's seat definitions, raw (§712 `seats`, A-241): the branch's revision's seat
// definitions with the definitions the merges' trees carry laid over them, so a revision that touches no definition is still
// judged against the whole seat list. A revision's own definitions are not kept yet (A-234), so the revision's are the
// trunk's, read from the state seat-definitions as the verb seat-definitions answers them.
func (ix *Indexer) revisionDefinitions(ctx context.Context, ready []PendingEntry) ([]json.RawMessage, error) {
defs, err := ix.trunkDefinitions(ctx)
if err != nil {
return nil, err
}
merged, err := ix.revisionSeatDefinitions(ctx, ready)
if err != nil {
return nil, err
}
for seat, d := range merged {
raw, err := json.Marshal(d)
if err != nil {
return nil, fmt.Errorf("the definition of %s: %w", seat, err)
}
defs[seat] = raw
}
var out []json.RawMessage
for _, seat := range slices.Sorted(maps.Keys(defs)) {
out = append(out, defs[seat])
}
return out, nil
}
// trunkDefinitions are the trunk's seat definitions of the state seat-definitions, raw, by seat: every `<seat>.trunk` entry
// that holds a definition; none where the indexer keeps no such state.
func (ix *Indexer) trunkDefinitions(ctx context.Context) (map[string]json.RawMessage, error) {
defs := map[string]json.RawMessage{}
if ix.SeatDefinitions == nil {
return defs, nil
}
keys, err := ix.SeatDefinitions.Keys(ctx)
if err != nil {
return nil, fmt.Errorf("the %s state cannot be listed: %w", SeatDefinitionsState, err)
}
for _, key := range keys {
seat, ok := strings.CutSuffix(key, "."+SourceTrunk)
if !ok || strings.HasPrefix(key, outOfStepPrefix) {
continue
}
v, _, found, err := ix.SeatDefinitions.Get(ctx, key)
if err != nil {
return nil, fmt.Errorf("%s.%s cannot be read: %w", SeatDefinitionsState, key, err)
}
var e SeatDefinitionEntry
if !found || json.Unmarshal(v, &e) != nil || e.Digest == "" || len(e.Definition) == 0 {
continue // an entry that holds no definition is no seat of the list
}
defs[seat] = e.Definition
}
return defs, nil
}
// validationJob is the run of the manifest validator for the request (§712): the program cmd/mesh-judge, its directory under
// the Go module the go.mod at the entry's path names (read by the build, A-268), at the entry's path, built at head, the revision's head of the entry's repository; no
// heads and no arguments; the input the request's bytes, served by the seat mesh-catalog's verb input.
func validationJob(indexer meshcatalog.CatalogEntry, head, branch string, seq uint64, input []byte) (buildstep.RunJob, error) {
switch {
case indexer.Repository == "":
return buildstep.RunJob{}, fmt.Errorf("no entry of the revision claims the seat %s", validationCatalogSeat)
case head == "":
return buildstep.RunJob{}, fmt.Errorf("the revision names no head of %s, the repository of the indexer entry %s", indexer.Repository, indexer.Module)
}
in, err := buildstep.RunInputOf(validationInputVerb, input)
if err != nil {
return buildstep.RunJob{}, err
}
dir := indexer.Path
if dir == "" {
dir = "."
}
return buildstep.RunJob{
Program: buildstep.RunProgram{Repository: indexer.Repository, Commit: head, Dir: dir, Package: "./" + validationProgram},
Input: in,
Reader: fmt.Sprintf("manifest validator of %s at %d", branchName(branch), seq),
}, nil
}
func branchName(branch string) string {
if isTrunkBranch(branch) {
return TrunkBranch
}
return branch
}
// ValidationRefusal is what a validation's verdict refuses (ADR 0579 §2, §712): of each manifest the change case refuses that
// the revision adds or changes (its bytes differ from the trunk case's), each problem the trunk case lacks; and each problem
// between manifests the trunk case lacks. A problem only in unchanged manifests refuses nothing. It is empty when the revision
// is accepted. An answer that holds not both cases is no verdict: it answers an error, so the run could not validate.
func ValidationRefusal(req composer.Request, v composer.Verdict) (map[string][]string, error) {
cases := map[string]composer.CaseVerdict{}
for _, c := range v.Cases {
cases[c.Name] = c
}
change, okChange := cases[validationChange]
trunkV, okTrunk := cases[validationTrunk]
if !okChange || !okTrunk {
return nil, fmt.Errorf("the manifest validator's answer holds not both cases %s and %s", validationChange, validationTrunk)
}
raws := map[string]map[string]string{}
for _, c := range req.Cases {
raws[c.Name] = map[string]string{}
for name, m := range c.Manifests {
raws[c.Name][name] = InputDigest(m.Raw)
}
}
refused := map[string][]string{}
for _, module := range slices.Sorted(maps.Keys(change.Manifests)) {
r := change.Manifests[module]
if r.Accepted || r.Refused == "" {
continue
}
before, had := raws[validationTrunk][module]
if module != judge.Unnamed && had && before == raws[validationChange][module] {
continue // unchanged: its problems refuse nothing, and its entry keeps its verdict (ADR 0579 §1)
}
old := map[string]bool{}
for _, p := range strings.Split(trunkV.Manifests[module].Refused, "\n") {
old[p] = true
}
for _, p := range strings.Split(r.Refused, "\n") {
if p != "" && !old[p] {
refused[module] = append(refused[module], p)
}
}
}
return refused, nil
}
// RefusedWords is a refusal as the pending value's `refused` says it: each manifest and its problems, one manifest a line.
func RefusedWords(refused map[string][]string) string {
var lines []string
for _, module := range slices.Sorted(maps.Keys(refused)) {
lines = append(lines, module+": "+strings.Join(refused[module], "; "))
}
return strings.Join(lines, "\n")
}
@@ -1,130 +0,0 @@
package index
import (
"context"
"encoding/json"
"reflect"
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"mesh-delivery-indexer/judge"
)
// The validation refuses only what the revision causes (design 47 G §712, ADR 0579 §2), and its run names the indexer entry's
// repository, path and the revision's head, and no heads.
func TestValidationRefusesWhatTheRevisionCauses(t *testing.T) {
manifests := func(m map[string]string) map[string]composer.Manifest {
out := map[string]composer.Manifest{}
for k, v := range m {
out[k] = composer.Manifest{Raw: json.RawMessage(v)}
}
return out
}
req := composer.Request{Cases: []composer.Case{
{Name: "change", Manifests: manifests(map[string]string{"a": `{"module":"a","v":2}`, "b": `{"module":"b"}`})},
{Name: "trunk", Manifests: manifests(map[string]string{"a": `{"module":"a"}`, "b": `{"module":"b"}`})},
}}
verdict := func(change, trunk map[string]string) composer.Verdict {
read := func(m map[string]string) map[string]composer.ManifestRead {
out := map[string]composer.ManifestRead{"a": {Accepted: true}, "b": {Accepted: true}}
for k, v := range m {
out[k] = composer.ManifestRead{Refused: v}
}
return out
}
return composer.Verdict{Cases: []composer.CaseVerdict{{Name: "change", Manifests: read(change)}, {Name: "trunk", Manifests: read(trunk)}}}
}
for _, c := range []struct {
name string
v composer.Verdict
want, wantErr string
}{
{"a changed manifest gains a problem", verdict(map[string]string{"a": "new\nold"}, map[string]string{"a": "old"}), "a: new", ""},
{"an unchanged manifest's problem refuses nothing", verdict(map[string]string{"b": "bad"}, nil), "", ""},
{"a problem between manifests the trunk lacks", verdict(map[string]string{judge.Unnamed: "x\ny"}, map[string]string{judge.Unnamed: "y"}), judge.Unnamed + ": x", ""},
{"an answer with one case is no verdict", composer.Verdict{Cases: verdict(nil, nil).Cases[:1]}, "", "not both cases"},
} {
got, err := ValidationRefusal(req, c.v)
if c.wantErr != "" {
if err == nil || !strings.Contains(err.Error(), c.wantErr) {
t.Errorf("%s: %v, want %q", c.name, err, c.wantErr)
}
continue
}
if err != nil || RefusedWords(got) != c.want {
t.Errorf("%s: %q, %v, want %q", c.name, RefusedWords(got), err, c.want)
}
}
head := strings.Repeat("a", 40)
entry := meshcatalog.CatalogEntry{Module: "mesh-delivery-indexer", Repository: "novox/mesh-delivery", Path: "modules/mesh-delivery-indexer",
Artifacts: []meshcatalog.Artifact{{Language: "go", Kind: "bundle"}}}
job, err := validationJob(entry, head, "main", 7, []byte(`{"v":1}`))
if err != nil {
t.Fatal(err)
}
want := buildstep.RunProgram{Repository: "novox/mesh-delivery", Commit: head, Dir: "modules/mesh-delivery-indexer", Package: "./cmd/mesh-judge"}
if job.Program != want || len(job.Heads) != 0 || len(job.Args) != 0 || job.Input.Verb != "seat:mesh-catalog.input" || job.Reader != "manifest validator of main at 7" {
t.Errorf("the job %+v", job)
}
if _, err := validationJob(meshcatalog.CatalogEntry{}, head, "main", 7, nil); err == nil || !strings.Contains(err.Error(), "claims the seat mesh-catalog") {
t.Errorf("no indexer entry: %v", err)
}
}
// The validation's seats are the branch's revision's seat definitions with the merges' changed ones laid over them (A-241): a
// revision that touches no definition is judged against the whole seat list, and a merge's definition replaces its seat's.
func TestValidationSeatsAreTheRevisionsWithTheMergesOver(t *testing.T) {
ctx := context.Background()
defs := newMem()
for seat, raw := range map[string]string{"node-engine": `{"seat":"node-engine","verbs":[{"name":"old"}]}`, "git": `{"seat":"git"}`} {
v, _ := json.Marshal(SeatDefinitionEntry{Seat: seat, Source: SourceTrunk, Digest: "d-" + seat, Definition: json.RawMessage(raw)})
defs.put(SeatDefinitionKey(seat, SourceTrunk, "", 0), v)
}
v, _ := json.Marshal(SeatDefinitionEntry{Seat: "pulled", Source: SourcePull, Pull: 3, Digest: "d", Definition: json.RawMessage(`{"seat":"pulled"}`)})
defs.put(SeatDefinitionKey("pulled", SourcePull, "r", 3), v)
ix := &Indexer{Module: testSelf, SeatDefinitions: defs}
seatsOf := func(raws []json.RawMessage) map[string]string {
out := map[string]string{}
for _, raw := range raws {
var d struct {
Seat string `json:"seat"`
Verbs []struct {
Name string `json:"name"`
} `json:"verbs"`
}
if err := json.Unmarshal(raw, &d); err != nil {
t.Fatal(err)
}
out[d.Seat] = ""
if len(d.Verbs) > 0 {
out[d.Seat] = d.Verbs[0].Name
}
}
return out
}
got, err := ix.revisionDefinitions(ctx, nil)
if err != nil || !reflect.DeepEqual(seatsOf(got), map[string]string{"node-engine": "old", "git": ""}) {
t.Errorf("a revision touching no definition: %v (%v), want the trunk's whole list and no pull request's", seatsOf(got), err)
}
const contracts = "modules/seat-contracts"
f := forgemsg.Forge{Repository: "r", Merge: strings.Repeat("c", 40), Manifests: map[string]string{
contracts: `{"module":"seat-contracts","version":"1.0","build":{"artifacts":[{"name":"go-seats","kind":"package","language":"go","paths":["go"],"series":"0.1","generated-from":"."}]}}`}}
ix.Files = func(_ context.Context, _, _ string, dirs []string) (map[string]map[string]string, error) {
return map[string]map[string]string{dirs[0]: {"node-engine/seat.json": "1"}}, nil
}
ix.ReadFile = func(_ context.Context, _, _, p string) ([]byte, error) {
if p == contracts+"/seats/node-engine/seat.json" {
return []byte(`{"format":1,"seat":"node-engine","scope":"node","grain":"seat-list","decision":"d","domain":"Core","verbs":[{"name":"new"}]}`), nil
}
return nil, nil
}
got, err = ix.revisionDefinitions(ctx, []PendingEntry{{Forge: f}})
if err != nil || !reflect.DeepEqual(seatsOf(got), map[string]string{"node-engine": "new", "git": ""}) {
t.Errorf("a merge changing a definition: %v (%v), want its definition over the trunk's and the rest kept", seatsOf(got), err)
}
}
@@ -1,329 +0,0 @@
package index
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
)
// The pending revision's wait for its validation (design 47 G §713, A-232, A-247): the take-in puts the run ask of §712, writes
// `validation` on the revision's newest pending value and ends there; the build seat's step-ended of that ask, heard through the
// indexer's own durable consumer, resumes it: accepted takes the revision in as before, refused keeps it pending with `refused`,
// and an end that could not validate keeps it pending with `failed`, asked again by a later merge of the branch or by hand.
// ValidationAsks puts a run ask on the build seat (buildrun.Asker).
type ValidationAsks interface {
Put(ctx context.Context, a buildstep.StepAsk) error
}
// newestOfRevision is the index in ready of the revision's newest merge, the value that carries `validation` (A-247).
func newestOfRevision(ready []PendingEntry) int {
n := 0
for i, e := range ready {
if e.Value.Seq > ready[n].Value.Seq {
n = i
}
}
return n
}
// branchOfRevision is the watched branch the revision's merges landed on, its newest merged message's base (design 47 G §714);
// a message that names none is the trunk's. A pending value's own branch is its pull request's, never the branch validated.
func branchOfRevision(ready []PendingEntry) string {
if base := ready[newestOfRevision(ready)].Forge.Base; base != "" {
return base
}
return TrunkBranch
}
// validationGate says whether the ready revision waits for its validation (§713): it goes on to the take-in once its newest
// value's validation was accepted at that value's sequence; it waits while its ask is out, while it is refused, and while it
// could not validate; otherwise it puts a new ask, the next try, and waits. Without an asker nothing is validated.
func (ix *Indexer) validationGate(ctx context.Context, ready []PendingEntry, own string) (bool, string, error) {
if ix.ValidationAsks == nil {
return false, "", nil
}
newest := ready[newestOfRevision(ready)]
seq := newest.Value.Seq
if v := newest.Value.Validation; v != nil && v.Seq == seq {
switch {
case v.Accepted == seq:
return false, "", nil
case v.Failed != "":
return true, fmt.Sprintf("the validation of the revision at %d could not validate: %s", seq, v.Failed), nil
case newest.Value.Refused != "":
return true, fmt.Sprintf("the revision at %d is refused: %s", seq, newest.Value.Refused), nil
case v.Ask != "":
return true, fmt.Sprintf("the revision at %d waits for its validation, the ask %s", seq, v.Ask), nil
}
}
return true, "", ix.askValidation(ctx, ready, own)
}
// askValidation puts the revision's validation ask (§712) and writes it on its newest value, the try after every earlier try
// of the revision; an older value's validation goes (A-247: on the newest alone). A request or job that cannot be made could not
// validate: it is written as `failed`, said, and no ask is put.
func (ix *Indexer) askValidation(ctx context.Context, ready []PendingEntry, own string) error {
n := newestOfRevision(ready)
newest := ready[n]
try := 0
for _, e := range ready {
if v := e.Value.Validation; v != nil && v.Try > try {
try = v.Try
}
}
v := &takein.Validation{Seq: newest.Value.Seq, Try: try + 1, Asked: ix.now()}
branch := branchOfRevision(ready)
ask := buildstep.StepAsk{ID: buildstep.NewID(), Kind: buildstep.AskCheck, Language: buildstep.AskGo, Asker: ValidationAsker, At: ix.now().UTC()}
req, entry, err := ix.validationRequest(ctx, branch, ready)
var raw []byte
if err == nil {
raw, err = json.Marshal(req)
}
var job buildstep.RunJob
if err == nil {
job, err = validationJob(entry, revisionHead(ready, entry.Repository, entry.Commit), branch, v.Seq, raw)
}
if err == nil {
v.Input, err = ix.KeepInput(ctx, ask.ID, raw)
}
if err != nil {
v.Failed = "the validation could not be asked: " + err.Error()
} else {
v.Ask, ask.Run = ask.ID, &job
}
if err := ix.writeValidation(ctx, ready, n, v, own); err != nil {
return err
}
if v.Failed != "" {
ix.sayValidation(ctx, ready, v)
return nil
}
if err := ix.ValidationAsks.Put(ctx, ask); err != nil {
return fmt.Errorf("the validation ask %s of the revision at %d could not be put: %w", ask.ID, v.Seq, err)
}
ix.logf("take-in: the revision at %d is validated by the ask %s (try %d); it waits for its end", v.Seq, ask.ID, v.Try)
return nil
}
// revisionHead is the revision's head of repository: the newest merge of it the revision covers, else the commit its entry stands at.
func revisionHead(ready []PendingEntry, repository, commit string) string {
var seq uint64
head := commit
for _, e := range ready {
if e.Forge.Merge != "" && strings.EqualFold(e.Forge.Repository, repository) && e.Value.Seq >= seq {
seq, head = e.Value.Seq, e.Forge.Merge
}
}
return head
}
// writeValidation writes v on ready[n] and takes it from every other value, each by compare-and-set; a new validation clears the
// value's refusal. A value lost under it is an error: the evaluation is made again.
func (ix *Indexer) writeValidation(ctx context.Context, ready []PendingEntry, n int, v *takein.Validation, own string) error {
for i, e := range ready {
if i != n && e.Value.Validation == nil {
continue
}
p := e.Value
p.Validation, p.LastWriter = nil, own
if i == n {
p.Validation, p.Refused = v, ""
}
data, err := encodePending(p)
if err != nil {
return err
}
if err := ix.Pending.Update(ctx, e.Key, data, e.Revision); err != nil {
return fmt.Errorf("%s.%s: %w", PendingState, e.Key, err)
}
}
return nil
}
// sayValidation says the revision's wait in take-in.pending with its refusal or why it could not validate.
func (ix *Indexer) sayValidation(ctx context.Context, ready []PendingEntry, v *takein.Validation) {
t := revisionWait(takein.TakeInPending, ready, "")
t.Failed = v.Failed
t.Refused = ready[newestOfRevision(ready)].Value.Refused
ix.sayTakeIn(ctx, t)
}
// errNoValidationAnswer is an end whose answer is no verdict of both cases.
var errNoValidationAnswer = errors.New("no answer")
// HearValidationEnded takes in the end of a validation ask (§713), heard on the build seat's step-ended through the indexer's own
// durable consumer: only an end of the indexer's asks outside a pipeline; an end whose ask no pending value names now is dropped,
// written in the journal. The answer is read from the event's own output or output object (A-247).
func (ix *Indexer) HearValidationEnded(ctx context.Context, e buildstep.StepEnded) error {
if e.Asker != ValidationAsker || e.Pipeline != "" || e.Ask == "" || ix.Pending == nil || !ix.pendingOnBus(ctx) {
return nil
}
own := ix.ownBuildNow(ctx)
ix.pendingMu.Lock()
defer ix.pendingMu.Unlock()
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
if err != nil {
return err
}
ready := WaitsOf(entries).Ready
n := -1
for i, p := range ready {
if p.Value.Validation != nil && p.Value.Validation.Ask == e.Ask {
n = i
}
}
if n < 0 {
ix.logf("take-in: the end of the validation ask %s (%s on %s) is dropped: no pending revision names that ask now", e.Ask, e.Outcome, e.Node)
return nil
}
ready, lost, _, _ := ix.loadNewest(ctx, ready)
if len(lost) > 0 {
return fmt.Errorf("the revision the validation ask %s ended for holds a merge that cannot be read", e.Ask)
}
n = newestOfRevision(ready)
v := *ready[n].Value.Validation
refused, failed := ix.validationAnswer(ctx, ready, e, v)
p := ready[n].Value
switch {
case failed != "":
v.Failed = failed
case refused != "":
p.Refused = refused
default:
v.Accepted = v.Seq
}
p.Validation, p.LastWriter = &v, own
data, err := encodePending(p)
if err != nil {
return err
}
if err := ix.Pending.Update(ctx, ready[n].Key, data, ready[n].Revision); err != nil {
return fmt.Errorf("%s.%s: %w", PendingState, ready[n].Key, err)
}
if v.Accepted == 0 {
ready[n].Value = p
ix.sayValidation(ctx, ready, &v)
ix.logf("take-in: the validation ask %s of the revision at %d ended: refused %q, could not validate %q", e.Ask, v.Seq, refused, failed)
return nil
}
ix.logf("take-in: the validation ask %s accepted the revision at %d", e.Ask, v.Seq)
if own == "" {
return nil // the take-in waits for the build this program runs, as every take-in does
}
_, _, err = ix.evaluate(ctx, own, false)
return err
}
// validationAnswer reads the end's answer: the refusal of the revision, or why it could not validate (the step did not pass, the
// program exited other than 0, its answer is no verdict of both cases, or its request is no longer the one asked).
func (ix *Indexer) validationAnswer(ctx context.Context, ready []PendingEntry, e buildstep.StepEnded, v takein.Validation) (string, string) {
if e.Outcome != buildstep.OutcomePassed {
return "", strings.TrimSpace(e.Outcome + ": " + e.Said)
}
// The answer is the program's last line of standard output, inline or kept by the seat as an object (A-254); a program that
// exited other than 0 ended the step failed, read above.
out, err := e.ReadOutput(buildstep.AskGo, ix.AskVerb)
if err != nil {
return "", "the run's answer could not be read: " + err.Error()
}
var verdict composer.Verdict
if err := json.Unmarshal(out, &verdict); err != nil {
return "", fmt.Sprintf("%v: the manifest validator's output is no verdict: %v", errNoValidationAnswer, err)
}
req, _, err := ix.validationRequest(ctx, branchOfRevision(ready), ready)
var raw []byte
if err == nil {
raw, err = json.Marshal(req)
}
if err != nil {
return "", "the request cannot be made again to read the verdict: " + err.Error()
}
if runDigest(raw) != v.Input {
return "", "the revision's request is no longer the one the ask " + v.Ask + " was asked with"
}
refused, err := ValidationRefusal(req, verdict)
if err != nil {
return "", err.Error()
}
return RefusedWords(refused), ""
}
// ValidateAgainTool asks the validation of a revision that could not validate again by hand (§713).
const ValidateAgainTool = "indexer_validate_again"
// validationWords is the validation's state as the hand ask answers it.
func validationWords(v *takein.Validation, refused string) string {
switch {
case v == nil:
return "no validation is asked yet"
case v.Accepted != 0:
return fmt.Sprintf("accepted at %d (try %d, the ask %s)", v.Accepted, v.Try, v.Ask)
case v.Failed != "":
return fmt.Sprintf("could not validate at %d (try %d): %s", v.Seq, v.Try, v.Failed)
case refused != "":
return fmt.Sprintf("refused at %d (try %d): %s", v.Seq, v.Try, refused)
default:
return fmt.Sprintf("the ask %s is out since %s (try %d at %d)", v.Ask, v.Asked.UTC().Format(time.RFC3339), v.Try, v.Seq)
}
}
// ValidateAgainByHand is indexer_validate_again answered (§713): the ready revision under key (its oldest merge's m. key, as
// indexer_pending names it) whose validation could not validate is asked again, the next try; its answer says the ask's state.
// A revision accepted, refused or whose ask is out is asked nothing: a refusal is asked again only by a later merge of its branch.
func (ix *Indexer) ValidateAgainByHand(ctx context.Context, key, by, why string) TakenIn {
r := TakenIn{Module: key}
if ix.Pending == nil || ix.ValidationAsks == nil {
r.Answer = "this indexer asks no validation: nothing asked"
return r
}
own := ix.ownBuildNow(ctx)
ix.pendingMu.Lock()
defer ix.pendingMu.Unlock()
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
if err != nil {
r.Answer = err.Error()
return r
}
ready := WaitsOf(entries).Ready
if len(ready) == 0 || ready[0].Key != key {
r.Answer = fmt.Sprintf("no revision waits under %s: indexer_pending names the waits", key)
return r
}
ready, lost, reasons, _ := ix.loadNewest(ctx, ready)
if len(lost) > 0 {
r.Answer = fmt.Sprintf("nothing asked: the message of %s is not read (%s)", lost[0].Key, reasons[lost[0].Key])
return r
}
newest := ready[newestOfRevision(ready)]
v := newest.Value.Validation
if v == nil || v.Seq != newest.Value.Seq || v.Failed == "" {
r.Answer = "nothing asked: the validation is " + validationWords(v, newest.Value.Refused) + "; only one that could not validate is asked again by hand"
return r
}
ix.logf("take-in: the validation of %s is asked again by hand by %s: %s", key, by, why)
if err := ix.askValidation(ctx, ready, own); err != nil {
r.Answer = "the validation was not asked again: " + err.Error()
return r
}
after, err := ReadPending(ctx, ix.Pending, ix.logf)
if err == nil {
ready = WaitsOf(after).Ready
}
if err != nil || len(ready) == 0 {
r.Answer = "asked again; its state is not read back"
return r
}
n := ready[newestOfRevision(ready)]
r.Stored = n.Value.Validation != nil && n.Value.Validation.Failed == ""
r.Answer = fmt.Sprintf("asked again by %s (%s): the validation is %s", by, why, validationWords(n.Value.Validation, n.Value.Refused))
return r
}
@@ -1,113 +0,0 @@
package index
import (
"context"
"encoding/json"
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
)
type askRecorder struct{ asks []buildstep.StepAsk }
func (r *askRecorder) Put(_ context.Context, a buildstep.StepAsk) error {
r.asks = append(r.asks, a)
return nil
}
// Design 47 G §713, A-247: a pending revision names its ask on its newest value and resumes on that ask's end; an end of a
// replaced ask is dropped; a run that could not validate is no refusal and never passed; the start asks nothing again while
// the ask is out.
func TestAPendingRevisionWaitsForItsValidation(t *testing.T) {
ctx := context.Background()
b := newTakeInBed(t)
asks := &askRecorder{}
b.ix.ValidationAsks, b.ix.Inputs, b.ix.Revisions = asks, newMem(), newMemObjects(0)
// No entry of the revision claims the seat mesh-catalog: the validation could not be asked, so it is failed, said, and
// nothing is taken in.
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", ""))
entries := readPending(t, b)
if len(entries) != 1 || entries[0].Value.Validation == nil || !strings.Contains(entries[0].Value.Validation.Failed, "claims the seat mesh-catalog") || len(asks.asks) != 0 {
t.Fatalf("pending %+v, asks %d: want m.2 pending with failed and no ask", entries, len(asks.asks))
}
if last := b.said[len(b.said)-1]; last.Kind != takein.TakeInPending || last.Failed == "" || last.Refused != "" {
t.Errorf("said %+v, want take-in.pending naming failed, never a refusal", last)
}
if w := PendingWaits(entries); len(w) != 1 || w[0].Validation == nil || w[0].Validation.Try != 1 {
t.Errorf("indexer_pending %+v, want the validation whole", w)
}
// An ask out: the newest value names it; the start asks nothing new while it is out.
loaded, _, _, _ := b.ix.loadNewest(ctx, entries)
req, _, err := b.ix.validationRequest(ctx, TrunkBranch, loaded)
if err != nil {
t.Fatal(err)
}
raw, _ := json.Marshal(req)
setValidation := func(v takein.Validation) {
e := readPending(t, b)[0]
p := e.Value
p.Validation = &v
data, _ := encodePending(p)
if err := b.pending.Update(ctx, e.Key, data, e.Revision); err != nil {
t.Fatal(err)
}
}
setValidation(takein.Validation{Ask: "mvl0frv18d1957", Input: runDigest(raw), Seq: 2, Try: 2})
if _, _, err := b.ix.TakeInAtStart(ctx); err != nil || len(asks.asks) != 0 || len(b.keys()) != 1 {
t.Fatalf("the start: %v, asks %d, pending %v: want it waiting for its ask", err, len(asks.asks), b.keys())
}
end := func(ask, outcome, said string, answer any) buildstep.StepEnded {
out, _ := json.Marshal(answer)
return buildstep.StepEnded{Ask: ask, ID: ask, Asker: ValidationAsker, Node: "ace", Outcome: outcome, Said: said, Output: string(out)}
}
// The answer is the manifest validator's verdict itself, the program's last line (A-254).
accepted := composer.Verdict{Cases: []composer.CaseVerdict{{Name: "change"}, {Name: "trunk"}}}
// The end of a replaced ask is dropped, written in the journal.
if err := b.ix.HearValidationEnded(ctx, end("mvl0frv18d1950", buildstep.OutcomePassed, "", accepted)); err != nil || len(b.keys()) != 1 || !strings.Contains(b.logs.String(), "is dropped") {
t.Fatalf("a replaced ask's end: %v, pending %v", err, b.keys())
}
// could not run: pending with failed, never passed.
if err := b.ix.HearValidationEnded(ctx, end("mvl0frv18d1957", buildstep.OutcomeCouldNotRun, "no build node", nil)); err != nil {
t.Fatal(err)
}
if v := readPending(t, b)[0].Value.Validation; v == nil || v.Failed != "could not run: no build node" || v.Accepted != 0 {
t.Fatalf("after could not run: %+v", v)
}
// indexer_validate_again (§713): its hand act recorded first, then the next try asked; the answer says the ask's state.
var recorded []string
tools := ValidationTools(func(key string, value []byte) (TakeInAsk, error) {
a, err := DecodeTakeIn(value)
if err != nil || key != "pending.validate-again."+takein.PendingKey(2) || !a.ValidateAgain {
t.Fatalf("asked %s %+v %v", key, a, err)
}
r := b.ix.ValidateAgainByHand(ctx, a.Pending, a.By, a.Why)
a.Result = &r
return a, nil
}, b.ix.now, func(what, why string) error { recorded = append(recorded, what); return nil })
if len(tools) != 1 || tools[0].Name != ValidateAgainTool {
t.Fatalf("tools = %v", tools)
}
if _, err := tools[0].Run(map[string]any{"key": takein.PendingKey(2)}); err == nil || len(recorded) != 0 {
t.Fatalf("asked again with no why: %v, %v", err, recorded)
}
got, err := tools[0].Run(map[string]any{"key": takein.PendingKey(2), "why": "the build node is back"})
if v := readPending(t, b)[0].Value.Validation; err != nil || len(recorded) != 1 || v == nil || v.Try != 3 || v.Failed == "could not run: no build node" ||
!strings.Contains(got.(*TakenIn).Answer, "try 3") {
t.Fatalf("asked again: %v, recorded %v, validation %+v, answer %+v: want the try after 2 asked", err, recorded, v, got)
}
setValidation(takein.Validation{Ask: "mvl0frv18d1959", Input: runDigest(raw), Seq: 2, Try: 3})
if r := b.ix.ValidateAgainByHand(ctx, takein.PendingKey(2), "lead", "again"); r.Stored || !strings.Contains(r.Answer, "is out") || readPending(t, b)[0].Value.Validation.Try != 3 {
t.Fatalf("an ask out was asked again: %+v", r)
}
// Asked again (by hand or by a later merge) and accepted: the revision is taken in, and its keys go.
setValidation(takein.Validation{Ask: "mvl0frv18d1958", Input: runDigest(raw), Seq: 2, Try: 3})
if err := b.ix.HearValidationEnded(ctx, end("mvl0frv18d1958", buildstep.OutcomePassed, "", accepted)); err != nil {
t.Fatal(err)
}
if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('c') {
t.Fatalf("accepted: pending %v %+v, avahi at %s: want it taken in\n%s", b.keys(), readPending(t, b)[0].Value, b.commit(t, "avahi"), b.logs.String())
}
}
@@ -506,8 +506,8 @@ func (ix *Indexer) writeVerdict(ctx context.Context, digest string, v meshcatalo
}
// OwnBuildState is the indexer's state of the build it runs (the bucket mesh-delivery-indexer_own-build, one key, OwnBuildKey):
// written once, at the process's start by FixOwnBuild, so indexer_pending says whether this process knows its build and, when it
// does not, since when and why.
// written once, at the process's start by FixOwnBuild, so indexer_pending says which build this process knows it runs, and since
// when. The build only names the writer of what is written (A-476 b).
const (
OwnBuildStateName = "own-build"
OwnBuildKey = "state"
@@ -526,8 +526,7 @@ type OwnBuild struct {
// seat (OwnBuild; never the trunk's value of the indexer, which names a build that may be merged and not delivered), before
// this process hears a merge. It is never asked again: while the node-engine replaces the indexer the seat already names the
// successor, and an old process that learnt it later would take revisions in with its old judge. Where the start's ask is not
// answered the build stays unknown for the life of this process: every merge waits, loudly (a take-in.pending that says so, the
// own-build state, indexer_pending), and the module is started again to learn it.
// answered the build stays unknown for the life of this process, said in the own-build state and the journal; nothing waits for it.
func (ix *Indexer) FixOwnBuild(ctx context.Context) {
ix.ownMu.Lock()
if ix.ownFixed || ix.OwnBuild == nil {
@@ -543,7 +542,7 @@ func (ix *Indexer) FixOwnBuild(ctx context.Context) {
st := OwnBuild{V: 1, Since: ix.now().UTC()}
if err != nil {
st.Why = err.Error()
ix.logf("judge: the build this program runs is not known (%v), so every merge waits and a manifest with a field it does not know waits and is not refused; it is learnt only at the start, so the indexer is started again", err)
ix.logf("judge: the build this program runs is not known (%v); merges are taken in all the same, and it is learnt only at the start", err)
} else {
st.Known, st.FP = true, fp
ix.logf("judge: this program runs the indexer build %s, fixed here at its start", fp)
@@ -179,8 +179,6 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
// ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a
// commit is in another (issue 1900).
if want := []string{"seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps",
"seat:node-build-go.accept.check", // design 47 G A-231: the validation's run ask (§712) on the check queue
"seat:node-build-go.output", // design 47 G A-254: a validation's answer kept as an object
"seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) {
t.Errorf("invokes = %v, want %v", m.Invokes, want)
}
@@ -204,9 +202,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
// milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered.
index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool,
index.EntryIndexTool, index.NamedDropTool, // design 47 G §675
index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.UnpinTool, index.ConsumerTool, index.RunsTool, index.RunDropTool, // design 47 G §667, §8, §579, §64 (A-194)
index.ValidateAgainTool) // design 47 G §713
if !reflect.DeepEqual(m.Tools, want) || len(want) != 52 || !slices.Contains(m.Tools, "modules") {
index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.UnpinTool, index.ConsumerTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194)
if !reflect.DeepEqual(m.Tools, want) || len(want) != 51 || !slices.Contains(m.Tools, "modules") {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
// Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section.
+1 -4
View File
@@ -77,8 +77,7 @@
"indexer_unpin",
"indexer_consumer",
"indexer_runs",
"indexer_run_drop",
"indexer_validate_again"
"indexer_run_drop"
],
"consumes": [
"go-package-registry.published",
@@ -180,8 +179,6 @@
"seat:mesh-hand-acts.hand-act",
"seat:mesh-watchdogs.presence",
"seat:node-build-go.steps",
"seat:node-build-go.accept.check",
"seat:node-build-go.output",
"seat:node-build-node.steps",
"seat:node-build-image.steps",
"mesh-delivery-builder.builder_versions"
+11 -35
View File
@@ -1,5 +1,5 @@
// Package takein is the indexer's take-in contract (novox/hq ADR 0633 §1, §5): the events it publishes on its own stream
// `take-in` while a source revision waits for the indexer it names or a delivery group is gathered, their keys, and the
// `take-in` while a source revision waits (its message unread, lost or held for room) or a delivery group is gathered, their keys, and the
// answer of its tool indexer_pending. The indexer writes them; the watchman and the view read them. It is the indexer's
// own package: one module owns these types, so they live with it and not in mesh-sdk (the operator, 2026-11-04).
package takein
@@ -118,7 +118,7 @@ func checkTakeInKey(key string) error {
}
// TakeIn is the body of a take-in event. Kind is the event's kind and Key its key. What says what waits: a revision
// (Key its oldest merge's PendingKey, WaitsFor the indexer build it names) or a group being gathered (Key its
// (Key its oldest merge's PendingKey) or a group being gathered (Key its
// GroupTakeInKey, Members every approved member with its head, Merges the members held so far). Merges are the merges
// it holds, oldest first; Since is when the wait began. A `taken` event names the same once the wait ended; By and Why
// name a hand act (a hand take-in, or a drop, Dropped), and are carried by nothing else.
@@ -138,16 +138,9 @@ type TakeIn struct {
// Lost are the keys of merges whose message the forge bridge's stream no longer holds: they are not taken in, and the
// revision waits until indexer_drop_pending drops them (issue 1585).
Lost []string `json:"lost,omitempty"`
// Refused is why the manifest of the indexer the revision names was refused when the revision was worked out: with
// the seat definitions the revision carries (ADR 0633 §1), the manifest validator's problems. The revision waits for
// an indexer whose manifest is accepted, and says why here, as it says what it lost.
// Refused is why the revision is refused: the cycle with a build edge its catalog graph would hold (design 47 G §49). The
// revision waits, and says why here, as it says what it lost.
Refused string `json:"refused,omitempty"`
// Failed is why the revision's validation could not validate (design 47 G §713): the build seat's words, never a refusal
// and never passed; it is asked again by a later merge of the branch and by indexer_validate_again.
Failed string `json:"failed,omitempty"`
// Unknown is why the indexer holds a merge it does not take in although no indexer build is named: it cannot learn the build
// it runs itself (ADR 0633 §1), so Since is when it began not to know. A pending revision names WaitsFor or Unknown.
Unknown string `json:"own-build-unknown,omitempty"`
// Branch, Source and Entries are what a take-in took in (design 47 G §476, §493, §648): the watched branch, the source
// revision it makes of that branch, and one entry per module it wrote. Every taken event of one take-in carries the same;
// a taken event that ends a wait without a take-in (a group's end, a drop) carries none and starts no pipeline.
@@ -184,8 +177,8 @@ func (t TakeIn) check() error {
if !strings.HasPrefix(t.Key, "m.") {
return fmt.Errorf("delivery: a revision's take-in key is its oldest merge's m.<sequence>, not %q", t.Key)
}
if t.Kind == TakeInPending && t.WaitsFor == "" && t.Unknown == "" && len(t.Lost) == 0 && t.Refused == "" && t.Failed == "" {
return fmt.Errorf("delivery: a pending revision %s names the indexer build it waits for, why the indexer does not know its own, the merges it lost, its refusal or why it could not validate", t.Key)
if t.Kind == TakeInPending && len(t.Lost) == 0 && t.Refused == "" {
return fmt.Errorf("delivery: a pending revision %s names the merges it lost or its refusal", t.Key)
}
case TakeInGroup:
if t.Group == "" {
@@ -266,32 +259,15 @@ type PendingWait struct {
Unread []string `json:"unread,omitempty"`
// Refused is why the manifest of the indexer it names was refused (TakeIn.Refused).
Refused string `json:"refused,omitempty"`
// Validation is the revision's validation whole, as its newest pending value keeps it (design 47 G §713, A-247).
Validation *Validation `json:"validation,omitempty"`
}
// Validation is a pending revision's validation (design 47 G §712, §713, A-232, A-247), kept on the revision's newest pending
// value alone: the run ask's id, when it was asked, its input's digest, the sequence of the newest merge it covers and the try
// number for it, why it could not validate, and the sequence it was accepted at (never asked again once set).
type Validation struct {
Ask string `json:"ask,omitempty"`
Asked time.Time `json:"asked,omitzero"`
Input string `json:"input,omitempty"`
Seq uint64 `json:"seq"`
Try int `json:"try"`
Failed string `json:"failed,omitempty"`
Accepted uint64 `json:"accepted,omitempty"`
}
// PendingAnswer is indexer_pending's answer: every wait, oldest first, and the pending keys that do not read.
type PendingAnswer struct {
Waits []PendingWait `json:"waits"`
Unread []string `json:"unread,omitempty"`
// OwnBuildUnknown is why the running indexer does not know its own build, and since when: while it is set, every merge waits
// (a take-in.pending with the same words) and the indexer is started again to learn it. Empty when the build is known.
OwnBuildUnknown string `json:"own-build-unknown,omitempty"`
OwnBuildSince time.Time `json:"own-build-since,omitzero"`
// OwnBuild is the build the running indexer knows it runs, fixed at its start; OwnBuildSince is since when. Empty when unknown.
Waits []PendingWait `json:"waits"`
Unread []string `json:"unread,omitempty"`
OwnBuildSince time.Time `json:"own-build-since,omitzero"`
// OwnBuild is the build the running indexer knows it runs, fixed at its start; OwnBuildSince is since when. Empty when unknown:
// it names only the writer of what the indexer writes, and nothing waits for it.
OwnBuild string `json:"own-build,omitempty"`
}
@@ -21,10 +21,7 @@ func sampleTakeIns(since time.Time) []TakeIn {
Members: []meshdeliveryclient.Head{{Repository: "novox/mesh-delivery", Number: 9, Head: "ffff"}, {Repository: "novox/mesh-sdk", Number: 37, Head: "dddd"}}, Since: since},
{Kind: TakeInTaken, Key: PendingKey(12), What: TakeInRevision, Merges: m, WaitsFor: "src2:abc", Since: since,
By: "lead-47@shanks", Why: "the named indexer can never run"},
{Kind: TakeInPending, Key: PendingKey(13), What: TakeInRevision, Merges: m, Unknown: "the engine does not answer", Since: since}, // the indexer cannot learn its own build
// design 47 G §713: a revision refused by its validation, and one that could not validate.
{Kind: TakeInPending, Key: PendingKey(14), What: TakeInRevision, Merges: m, Refused: "a: names no version", Since: since},
{Kind: TakeInPending, Key: PendingKey(15), What: TakeInRevision, Merges: m, Failed: "could not run: no build node", Since: since},
{Kind: TakeInPending, Key: PendingKey(14), What: TakeInRevision, Merges: m, Refused: "a cycle with a build edge", Since: since},
}
}
@@ -108,7 +108,7 @@ func tiersToolSaysEachModulesWait(t *testing.T) {
t.Fatalf("view_tiers with waits = %+v, %v", out, err)
}
web = held.Tiers[1][0]
if last := web.Waits[len(web.Waits)-1]; last.Step != view.TakeInStep || !last.Since.Equal(heldSince) || !strings.Contains(last.Words, "src2:abc") {
if last := web.Waits[len(web.Waits)-1]; last.Step != view.TakeInStep || !last.Since.Equal(heldSince) || !strings.Contains(last.Words, "unread or lost") {
t.Errorf("web = %+v, want its last wait on the edge take-in since %s", web, heldSince)
}
if lib := held.Tiers[0][1]; lib.Module != "lib" || lib.Waits[len(lib.Waits)-1].Step != view.TakeInStep || !strings.Contains(lib.Waits[len(lib.Waits)-1].Words, "group feat") {
@@ -219,7 +219,7 @@ func (r Reader) heldBack(out *Tiers) (map[string]TierWait, []string) {
return held, nil
}
for _, w := range a.Waits {
words := "the indexer holds back " + w.Key + ", which waits for the indexer build " + w.WaitsFor
words := "the indexer holds back " + w.Key + " (its message is unread or lost, or it is held for room)"
if w.What == takein.TakeInGroup {
words = "the indexer gathers the group " + w.Group + " and its end has not come"
} else if w.Behind != "" {
@@ -132,7 +132,7 @@ func TestRunNeedsItsCredential(t *testing.T) { // 1135/AC6
}
out.findings[old.Key()] = true
w.SeedFinding(old.Key(), value)
body, err := takein.EncodeTakeIn(takein.TakeIn{Kind: takein.TakeInPending, Key: "m.00000000000000000043", What: takein.TakeInRevision, Since: time.Now(), WaitsFor: "src2:x",
body, err := takein.EncodeTakeIn(takein.TakeIn{Kind: takein.TakeInPending, Key: "m.00000000000000000043", What: takein.TakeInRevision, Since: time.Now(), Lost: []string{"m.00000000000000000043"},
Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}})
if err != nil {
t.Fatal(err)
@@ -136,27 +136,27 @@ func TestTheWatchmansFindings(t *testing.T) {
// began ten minutes ago, and the indexer still holds it.
since := c.t.Add(-10 * time.Minute)
w.SeedTakeIn(takein.PendingAnswer{Waits: []takein.PendingWait{{Key: revKey, What: takein.TakeInRevision, Keys: []string{revKey},
Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}, WaitsFor: "src2:x", Since: since}}})
Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}, Lost: []string{"m.00000000000000000042"}, Since: since}}})
w.TakeInCaughtUp()
if len(o.findings) != 0 || o.ttls[signalKey("take-in", revKey)] != TakeInRevisionBound-10*time.Minute {
t.Fatalf("within its bound: findings %v, ttl %v; want none yet and the rest of the bound", o.findings, o.ttls[signalKey("take-in", revKey)])
}
expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound-10*time.Minute)
}, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}},
{"ADR 0633 §1 (issue 1921): a merge held because the indexer cannot learn its own build is raised past the revision bound from its since, naming why", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
{"ADR 0633 §1 (issue 1921): (A-476 b) a revision held because its message is lost or unread is raised past the revision bound from its since, naming why", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
w.TakeInCaughtUp()
body, err := takein.EncodeTakeIn(takein.TakeIn{Kind: takein.TakeInPending, Key: revKey, What: takein.TakeInRevision, Since: c.t,
Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}, Unknown: "the engine does not answer"})
Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}, Lost: []string{revKey}})
if err != nil {
t.Fatal(err)
}
w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), body)
if o.ttls[signalKey("take-in", revKey)] != TakeInRevisionBound || len(o.findings) != 0 {
t.Fatalf("an unknown own build: ttls %v, findings %v", o.ttls, o.findings)
t.Fatalf("a lost message: ttls %v, findings %v", o.ttls, o.findings)
}
expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound)
for _, f := range o.findings {
if !strings.Contains(f.Summary, "cannot learn the build it runs") {
if !strings.Contains(f.Summary, "lost or unread") {
t.Errorf("finding %q does not say why", f.Summary)
}
}
@@ -573,7 +573,7 @@ func takeInBody(t *testing.T, kind, key string, since time.Time) []byte {
if key[0] == 'g' {
body.What, body.Group = takein.TakeInGroup, "feat/take-in"
} else if kind == takein.TakeInPending {
body.WaitsFor = "src2:abc"
body.Lost = []string{key}
}
b, err := takein.EncodeTakeIn(body)
if err != nil {
@@ -135,10 +135,7 @@ func (w *Watchman) takeIn(kind, key string, t takein.TakeIn) {
w.unfind(finding)
return
}
bound, what := TakeInRevisionBound, "a source revision waits for the indexer build "+t.WaitsFor
if t.Unknown != "" {
what = "a source revision is held because the indexer cannot learn the build it runs (" + t.Unknown + ")"
}
bound, what := TakeInRevisionBound, "a source revision is held (a message lost or unread, no room for it, or its refusal)"
if t.What == takein.TakeInGroup {
bound, what = TakeInGroupBound, "a delivery group is gathered and its end has not come"
}
@@ -152,7 +149,7 @@ func (w *Watchman) takeIn(kind, key string, t takein.TakeIn) {
Headline: "A merged change is held back by the indexer",
Explanation: "The indexer holds " + what + " and has not taken it in within " + bound.String() + ", so what the merge changes does " +
"not reach the machines.",
Needs: "the indexer build it waits for to run, or the take-in by hand (indexer_take_in_pending), or a drop (indexer_drop_pending)"}
Needs: "the take-in by hand (indexer_take_in_pending), or a drop (indexer_drop_pending)"}
left := bound - w.now().Sub(t.Since)
w.unfind(finding)
w.unsignal(takeInRow, key)