A-476: the take-in asks no validation, holds for no indexer build, and moves only its repository's modules

(a) validation_wait.go and validate.go go with the validation ask, its end consumer, indexer_validate_again and
Pending.Validation; (b) evaluate always takes the revision in with the build the program runs (holdFor and the build
comparison go); (c) a merge's recompute writes only the modules registered for its repository.

Builds: design 47 G A-476 (a), (b), (c)
This commit is contained in:
2026-11-17 20:20:45 +01:00
parent 0adcfbbbdb
commit 9768d0b67d
22 changed files with 147 additions and 1369 deletions
@@ -318,10 +318,6 @@ func main() {
})...)
tools = append(tools, viewTools(os.Getenv, trunk.State, time.Now)...)
tools = append(tools, index.PendingTools(keptRead{stdio.State(index.PendingState)}, keptRead{stdio.State(index.OwnBuildStateName)}, asks, time.Now)...)
tools = append(tools, index.ValidationTools(asks, time.Now, func(what, why string) error {
_, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.ValidateAgainTool, Verb: "validate-again"})
return err
})...)
tools = append(tools, index.FingerprintsTools(asks, time.Now, func(what, why string) error {
_, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.FingerprintsAgainTool, Verb: "fingerprints-again"})
return err
@@ -22,7 +22,6 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"github.com/nats-io/nats-server/v2/server"
"github.com/nats-io/nats.go"
@@ -97,11 +96,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
// The forge bridge's tool bridge_merged_message (issue 1585): answers the merged message published for a merge, by
// repository@merge, as the bridge does; a restarted indexer holds none in memory and asks it.
var bridgeMessages sync.Map
// The build seat node-build-go's stand-in (design 47 G §712, §713): it hears the indexer's validation asks and ends each with
// the manifest validator's verdict; refusing names, by the commit the job runs at, the manifest a revision's change refuses.
var validations *standIn
var refusing sync.Map
js, ctx, candidates, _, restart := startIndexerRun(t, func(ctx context.Context, builds buildLog) {
js, ctx, candidates, _, _ := startIndexerRun(t, func(ctx context.Context, builds buildLog) {
// The builder's log entries of two builds' ends (ADR 0571 clause 1), as the builder appends them.
builds.end(t, "novox/mesh-delivery", "a", "modules/a")
builds.end(t, "novox/mesh-catalog", "c", "modules/c")
@@ -128,9 +123,8 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
if _, err := tc.Put(ctx, key, value); err != nil {
t.Fatal(err)
}
// The trunk holds the indexer's entry, taken in at an earlier commit with its own manifest; the validation names no Go module (A-268).
// The trunk holds the indexer's entry, taken in at an earlier commit with its own manifest; the take-in asks no validation (A-476 a).
seedBuildModules(t, ctx, js, map[string]meshcatalog.CatalogEntry{testIndexer: trunkIndexerEntry(t, ctx, js)})
validations = standInBuildSeat(t, ctx, js, &refusing)
// The node-engine's composition on n1 (process.Conn.Call's subjects): the build of the indexer that runs there.
answer := func(m *nats.Msg) {
raw, _ := json.Marshal(map[string]any{"result": map[string]any{"runs": map[string]string{"fp": *running.Load()}}, "node": "n1"})
@@ -281,17 +275,13 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
}
return takein.TakeIn{}, false
}
held, ok := said(takein.TakeInPending, m1Key)
if !ok || held.What != takein.TakeInRevision || held.WaitsFor == "" || held.WaitsFor == first || len(held.Merges) != 1 {
t.Fatalf("take-in.pending.%s = %+v (said %v) within 20 s, want the revision waiting for another build than %s", m1Key, held, ok, first)
// A-476 a, b: the revision changing the indexer's own tree is taken in at once, with the build this program runs and no
// validation ask: it is never said pending, and the build the revision names is not waited for.
if taken, ok := said(takein.TakeInTaken, m1Key); !ok || len(taken.Merges) != 1 || taken.Merges[0].Merge != m1.Merge {
t.Fatalf("take-in.taken.%s = %+v (said %v) within 20 s, want the revision taken in at once", m1Key, taken, ok)
}
named := held.WaitsFor
// IX5 (§712): the revision was validated first, by one run ask whose job names the indexer entry's repository, path and
// the revision's head, and no heads; accepted, it is never asked again while it waits for the named indexer.
if asks := validations.asked(); len(asks) != 1 || asks[0].Run == nil || asks[0].Kind != sdkstep.AskCheck ||
asks[0].Run.Program.Repository != m1.Repository || asks[0].Run.Program.Dir != "modules/"+ix || asks[0].Run.Program.Commit != m1.Merge ||
len(asks[0].Run.Heads) != 0 || asks[0].Run.Program.Package != "./cmd/mesh-judge" {
t.Fatalf("validation asks = %+v, want one run of the indexer's manifest validator at %s", asks, m1.Merge)
if held := saidAll(takein.TakeInPending, m1Key); len(held) != 0 {
t.Errorf("the revision was said pending: %+v", held)
}
get := func(bucket, key string) ([]byte, bool) {
t.Helper()
@@ -312,11 +302,10 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
}
return e.Value(), true
}
// Only what building and moving the named indexer needs is written.
raw, ok := get(index.ModulesState, ix)
own, err := meshcatalogclient.DecodeModule(raw)
if !ok || err != nil || own.Fp2 != named || own.Commit != m1.Merge {
t.Fatalf("modules.%s = %+v (%v), want the named build %s at the merge", ix, own, err, named)
if !ok || err != nil || own.Commit != m1.Merge {
t.Fatalf("modules.%s = %+v (%v), want the indexer at the merge", ix, own, err)
}
if _, ok := get(index.Bucket(testIndexer, index.ManifestsState), own.Manifest); !ok {
t.Errorf("manifests.%s is not written", own.Manifest)
@@ -325,16 +314,11 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
t.Errorf("the verdict on the indexer's manifest %s is not written", own.Manifest)
}
repoKey, _ := trunk.RepositoryKey(m1.Repository)
if _, ok := get(index.ModulesState, "a"); ok {
t.Errorf("modules.a is written while the revision is pending")
if raw, ok := get(index.ModulesState, "a"); !ok || !strings.Contains(string(raw), m1.Merge) {
t.Errorf("modules.a = %s, want it at the merge %s", raw, m1.Merge)
}
if _, ok := get(index.Bucket(testIndexer, index.TakenState), repoKey); ok {
t.Errorf("taken.%s is written while the revision is pending", repoKey)
}
if e, err := candidates.Get(ctx, "feat/x"); err != nil {
t.Errorf("the candidate feat/x is gone while the merge of #12 is pending: %v", err)
} else if c, err := index.DecodeCandidate(e.Value()); err != nil || len(c.Pulls) != 2 {
t.Errorf("the candidate feat/x = %+v while the merge of #12 is pending, want its two pull requests", c)
if _, ok := get(index.Bucket(testIndexer, index.TakenState), repoKey); !ok {
t.Errorf("taken.%s is not written", repoKey)
}
// A group's first member arrives meanwhile; its end is still to come (the knowledge base's remark: it stays across the restart).
@@ -365,34 +349,13 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
}
var w []takein.PendingWait
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if w = waits(); len(w) == 2 && w[0].WaitsFor == named && w[1].Said != "" {
if w = waits(); len(w) == 1 && w[0].Said != "" {
break
}
}
if len(w) != 2 || w[0].Key != m1Key || w[0].WaitsFor != named || !slices.Contains(w[0].Holds, "a") || !slices.Contains(w[0].Holds, ix) ||
w[1].Key != groupKey || w[1].What != takein.TakeInGroup || len(w[1].Merges) != 1 || w[1].Said != takein.TakeInPendingEvent {
t.Fatalf("indexer_pending = %+v, want the revision %s waiting for %s holding a and the indexer, then the group gathered", w, m1Key, named)
}
// The named indexer starts (the node-engine now runs that build): it takes the revision in, and leaves the group.
running.Store(&named)
restart()
if taken, ok := said(takein.TakeInTaken, m1Key); !ok || len(taken.Merges) != 1 || taken.Merges[0].Merge != m1.Merge {
t.Fatalf("take-in.taken.%s = %+v (said %v) within 20 s of the start of %s", m1Key, taken, ok, named)
}
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if w = waits(); len(w) == 1 {
break
}
}
if raw, ok := get(index.ModulesState, "a"); !ok || !strings.Contains(string(raw), m1.Merge) {
t.Errorf("modules.a = %s after the take-in at the start, want it at the merge %s", raw, m1.Merge)
}
if _, ok := get(index.Bucket(testIndexer, index.TakenState), repoKey); !ok {
t.Errorf("taken.%s is not written by the take-in at the start", repoKey)
}
if len(w) != 1 || w[0].Key != groupKey || len(w[0].Keys) != 1 || w[0].Keys[0] != g1Key {
t.Fatalf("indexer_pending after the start = %+v, want only the group gathered, its first member %s", w, g1Key)
if len(w) != 1 || w[0].Key != groupKey || w[0].What != takein.TakeInGroup || len(w[0].Merges) != 1 || len(w[0].Keys) != 1 || w[0].Keys[0] != g1Key ||
w[0].Said != takein.TakeInPendingEvent {
t.Fatalf("indexer_pending = %+v, want only the group gathered, its first member %s", w, g1Key)
}
if _, ok := get(index.ModulesState, "c"); ok {
t.Errorf("modules.c is written while its group is gathered")
@@ -425,23 +388,6 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
t.Errorf("the group's first member %s was said pending as a revision of its own", g1Key)
}
// IX5 (§712, §713): a merge whose changed manifest gains a problem the trunk case lacks stays pending, refused naming it,
// said in take-in.pending; nothing of it is taken in.
m3 := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/y", Number: 13, Head: h1,
Merge: "3333333333333333333333333333333333333333", Root: "r0", Trees: map[string]string{"modules/a": "tree-a-3"},
Manifests: map[string]string{"modules/a": `{"module":"a","version":"2.0"}`}}
refusing.Store(m3.Merge, "a")
m3Key := takein.PendingKey(merged(m3))
if refused, ok := said(takein.TakeInPending, m3Key); !ok || !strings.Contains(refused.Refused, "a: the stand-in's problem") || refused.Failed != "" {
t.Fatalf("take-in.pending.%s = %+v (said %v) within 20 s, want the revision refused naming a's problem", m3Key, refused, ok)
}
if w = waits(); len(w) != 1 || w[0].Key != m3Key || w[0].Validation == nil || w[0].Validation.Accepted != 0 || len(validations.asked()) != 3 {
t.Fatalf("indexer_pending = %+v, asks %d: want the refused revision pending with its validation, asked once after m1 and the group", w, len(validations.asked()))
}
if raw, ok := get(index.ModulesState, "a"); !ok || strings.Contains(string(raw), m3.Merge) {
t.Errorf("modules.a = %s: the refused revision was taken in", raw)
}
}
// trunkIndexerEntry keeps the indexer's own manifest by its digest, accepted, and answers its trunk entry at an earlier commit.
@@ -475,63 +421,6 @@ func trunkIndexerEntry(t *testing.T, ctx context.Context, js jetstream.JetStream
return e
}
// standIn is the build seat node-build-go as the indexer's validation meets it: the asks it heard, oldest first.
type standIn struct {
mu sync.Mutex
asks []sdkstep.StepAsk
}
func (s *standIn) asked() []sdkstep.StepAsk {
s.mu.Lock()
defer s.mu.Unlock()
return slices.Clone(s.asks)
}
// standInBuildSeat keeps the seat's check queue as a stream and hears the indexer's asks on it (accept.check.indexer): each is
// ended at once with step-ended on the seat's family, its answer inline (A-254), the program's last line: the manifest validator's verdict over both cases,
// the change case refusing the manifest refusing names for the job's commit.
func standInBuildSeat(t *testing.T, ctx context.Context, js jetstream.JetStream, refusing *sync.Map) *standIn {
t.Helper()
subject, err := sdkstep.AskSubjectBy(sdkstep.AskGo, sdkstep.AskCheck, index.ValidationAsker)
if err != nil || subject != "mesh.seat.node-build-go.accept.check.mesh-catalog" {
t.Fatalf("the validation's subject %q %v, want the indexer's seat name its asker (A-163, A-232)", subject, err)
}
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "SEAT_NODE_BUILD_GO_CHECK", Subjects: []string{"mesh.seat.node-build-go.accept.check.>"}}); err != nil {
t.Fatal(err)
}
// The indexer's state inputs and its objects, where the validation's request is kept for the build seat's holder (§354).
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, index.InputsState), History: 1}); err != nil {
t.Fatal(err)
}
if _, err := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: catalogue.BusName(testIndexer, catalogue.StateObjectsName(index.InputsState))}); err != nil {
t.Fatal(err)
}
s := &standIn{}
if _, err := js.Conn().Subscribe(subject, func(m *nats.Msg) {
a, err := sdkstep.DecodeStepAsk(m.Data)
if err != nil || a.Run == nil {
t.Errorf("the stand-in build seat heard what is no run ask: %v", err)
return
}
s.mu.Lock()
s.asks = append(s.asks, a)
s.mu.Unlock()
change := composer.CaseVerdict{Name: "change", Manifests: map[string]composer.ManifestRead{}}
if module, ok := refusing.Load(a.Run.Program.Commit); ok {
change.Manifests[module.(string)] = composer.ManifestRead{Refused: "the stand-in's problem"}
}
verdict, _ := json.Marshal(composer.Verdict{V: 1, Cases: []composer.CaseVerdict{change, {Name: judge.TrunkCase, Manifests: map[string]composer.ManifestRead{}}}})
end, _ := json.Marshal(sdkstep.StepEnded{V: 1, ID: a.ID, Ask: a.ID, Asker: index.ValidationAsker, Node: "n1", Outcome: sdkstep.OutcomePassed,
Output: string(verdict), At: time.Now().UTC()})
if _, err := js.Publish(ctx, nodebuildgo.SubjectStepEnded("n1"), end); err != nil {
t.Errorf("the stand-in build seat could not say step-ended: %v", err)
}
}); err != nil {
t.Fatal(err)
}
return s
}
// jsSource is a bucket as a views.Source.
type jsSource struct {
ctx context.Context
@@ -479,11 +479,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear;
// until then every merge is taken in at once, as before.
Pending: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.PendingState), "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)},
SayTakeIn: sayTakeIn(js, cred),
// The validation's run ask (design 47 G §713, A-247, A-248), put under the indexer's own asker.
ValidationAsks: buildrun.Asker{Name: index.ValidationAsker, Publisher: buildrun.JetStreamAsks{JS: js}},
// The build seat's verb output, for a validation's answer kept as an object (A-254).
AskVerb: calls.CallFile}
SayTakeIn: sayTakeIn(js, cred)}
// The registrations and the changes that wait for their warrant (design 47 G §2, §667): their states are bound when they
// appear; a warrant is heard on the router's decided event for this module, below.
registry := newRegistry(js, conn, cred, calls.Call,
@@ -206,6 +206,15 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T)
if pub.Commit != m2 || len(pub.Packages) != 3 {
t.Fatalf("packages.%s = %+v within 20 s, want the merge %s with a digest of go, announce and buildstep", key, pub, m2)
}
// A-476 c: mesh-sdk's merge moved no module registered for another repository: importer stands as it was.
if now := module("importer", func(meshcatalog.ModulesByModuleValue) bool { return true }); now.Fp2 != importer.Fp2 || now.Packages[sdkGo] != m1 {
t.Errorf("mesh-sdk's merge moved importer of novox/mesh-catalog: %+v, want it unmoved at %s", now.Packages, m1)
}
// The next merge of its own repository moves it, built against the package published meanwhile.
merged("novox/mesh-catalog", 302, "8888888888888888888888888888888888888888", forgemsg.Forge{Root: "r0",
Trees: map[string]string{"modules/importer": "t1", "modules/plain": "t2"},
Manifests: map[string]string{"modules/importer": named("importer"), "modules/plain": named("plain")},
Locks: map[string]map[string]string{"modules/importer": {"go.sum": goSumHash}, "modules/plain": {"go.sum": goSumHash}}})
moved := module("importer", func(m meshcatalog.ModulesByModuleValue) bool { return m.Fp2 != importer.Fp2 })
if moved.Packages[sdkGo] != m2 {
t.Errorf("importer moved to %+v, want it built against %s", moved.Packages, m2)
@@ -28,7 +28,6 @@ type takerIn interface {
NamedDrop(ctx context.Context, fingerprint, member, by, why string) index.TakenIn
Unpin(ctx context.Context, revision, record, by, why string) index.TakenIn
DropRunByHand(ctx context.Context, run, by, why string) index.TakenIn
ValidateAgainByHand(ctx context.Context, key, by, why string) index.TakenIn
RegistryAct(ctx context.Context, a index.TakeInAsk) index.TakenIn
}
@@ -75,9 +74,6 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream
} else if a.Merge != "" {
asked, r = "hand take-in of a merge", ix.TakeInMerge(ctx, a.MergeRepository, a.Merge, a.Sequence, a.By, a.Why)
a.Module = r.Module
} else if a.Pending != "" && a.ValidateAgain {
asked, r = "validation asked again", ix.ValidateAgainByHand(ctx, a.Pending, a.By, a.Why)
a.Module = a.Pending
} else if a.Pending != "" && a.Drop {
asked, r = "drop of a pending merge", ix.DropPending(ctx, a.Pending, a.By, a.Why, a.Abandon)
a.Module = a.Pending
@@ -7,8 +7,6 @@ import (
"sync/atomic"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
@@ -155,15 +153,6 @@ func heardStepEnded(ctx context.Context, ix *index.Indexer, m jetstream.Msg, log
_ = m.Ack()
return
}
// The end of a validation ask resumes its pending revision (design 47 G §713); its own fault is given again by the bus.
var end buildstep.StepEnded
if err := json.Unmarshal(m.Data(), &end); err == nil {
if err := ix.HearValidationEnded(ctx, end); err != nil {
logf("take-in: the end of the validation ask %s is not taken in, so the bus gives it again: %v", end.Ask, err)
leftForBus(m, "the end of the validation ask "+end.Ask, err, logf)
return
}
}
if err := ix.HearInputEnded(ctx, e); err != nil {
logf("inputs: the input of the ask %s could not be deleted, so the bus gives it again: %v", e.Ask, err)
leftForBus(m, "the step-ended of the ask "+e.Ask, err, logf)
@@ -6,7 +6,6 @@ import (
"context"
"errors"
"fmt"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/stateobjects"
"reflect"
"slices"
@@ -234,12 +233,6 @@ type Indexer struct {
Pending Store
// SayTakeIn publishes a take-in event (take-in.pending, take-in.taken; ADR 0633 §5); nil says none.
SayTakeIn func(ctx context.Context, t takein.TakeIn) error
// ValidationAsks puts the validation's run ask (design 47 G §713, A-247: buildrun.Asker, named as the indexer's asker,
// A-248); nil validates nothing, and a revision is taken in as before.
ValidationAsks ValidationAsks
// AskVerb asks a seat verb as the runtime calls it: the build seat's verb output, for a run's answer the seat keeps as an
// object (design 47 G A-254, buildstep.StepEnded.ReadOutput); nil reads none, and such an answer could not validate.
AskVerb buildstep.AskVerb
// Runs is the state runs: the runs on request the indexer holds a candidate for (runcandidates.go, A-174); nil keeps none.
Runs Store
runMu sync.Mutex // one run event at a time (runcandidates.go)
@@ -60,7 +60,7 @@ func (ix *Indexer) FingerprintsAgain(ctx context.Context, module string) TakenIn
ix.freshMu.Unlock()
// The path of a merge's recompute: the modules, then the candidates their fingerprints reach (recomputeWith), then the
// pending merges a changed fingerprint settles (PendingAgain, as after a toolchain report).
err = ix.recomputeWith(ctx, nil, nil)
err = ix.recomputeWith(ctx, nil, nil, "")
ix.freshMu.Lock()
ix.fresh = nil
ix.freshMu.Unlock()
@@ -144,7 +144,7 @@ func (ix *Indexer) applyMoving(ctx context.Context, f forgemsg.Forge, drop bool,
ix.logf("%s at %s: its packages are not published: %v", f.Repository, f.Merge, err)
}
// What this merge moved reaches the modules standing on it (a base's tree, build-agent's fingerprint), in the graph's order.
return w, ix.recomputeWith(ctx, overlay, w)
return w, ix.recomputeWith(ctx, overlay, w, f.Repository)
}
// baseFP answers the new-form fingerprint of a base module as modules.<base> holds it, or why it has none.
@@ -13,7 +13,6 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
@@ -23,8 +22,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-indexer/judge"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
)
// The take-in step (novox/hq ADR 0633): a source revision is taken in only by the indexer it names. A merged message is
@@ -98,8 +95,6 @@ type Pending struct {
// Held is why a ready merge is not taken in although its revision names this build: objects full (HeldObjectsFull),
// until room is said (RoomAgain).
Held string `json:"held,omitempty"`
// Validation is the revision's validation (design 47 G §713, A-247), on the revision's newest value alone.
Validation *takein.Validation `json:"validation,omitempty"`
}
// PendingEntry is one key of the pending state as read: its key, revision, value and the merged message decoded. For a
@@ -786,31 +781,37 @@ func (ix *Indexer) evaluate(ctx context.Context, own string, start bool) (Applie
if len(lost) > 0 {
return Applied{}, ix.waitingLine(w), ix.holdLost(ctx, w.Ready, lost, reasons, unread)
}
if wait, words, err := ix.validationGate(ctx, ready, own); wait || err != nil {
return Applied{}, words, err
}
names, dry, err := ix.computeOver(ctx, ready)
dry, err := ix.computeOver(ctx, ready)
if err != nil {
return Applied{}, "", err
}
if words, refused, err := ix.refuseCycle(ctx, ready, dry); refused || err != nil {
return Applied{}, words, err
}
if names == own {
a, err := ix.takeInWhole(ctx, ready, own, "")
if errors.Is(err, errClaimLost) {
return a, "claimed by another take-in", nil
}
if errors.Is(err, errVerdictsNotYet) {
return a, "the verdicts state is not on the bus yet: the revision waits for it, taken in when it binds", nil
}
if errors.Is(err, ErrObjectsFull) {
ix.holdFull(ctx, ready)
return a, ix.waitingLine(w) + "; held: " + HeldObjectsFull, nil
}
return a, ix.waitingLine(w), err
// The indexer takes in with the build it runs, whatever build of the indexer the revision names (A-476 b).
a, err := ix.takeInWhole(ctx, ready, own, "")
if errors.Is(err, errClaimLost) {
return a, "claimed by another take-in", nil
}
return Applied{}, ix.waitingLine(w), ix.holdFor(ctx, ready, names, own, dry)
if errors.Is(err, errVerdictsNotYet) {
return a, "the verdicts state is not on the bus yet: the revision waits for it, taken in when it binds", nil
}
if errors.Is(err, ErrObjectsFull) {
ix.holdFull(ctx, ready)
return a, ix.waitingLine(w) + "; held: " + HeldObjectsFull, nil
}
return a, ix.waitingLine(w), err
}
// newestOfRevision is the index in ready of the revision's newest merge.
func newestOfRevision(ready []PendingEntry) int {
n := 0
for i, e := range ready {
if e.Value.Seq > ready[n].Value.Seq {
n = i
}
}
return n
}
// refuseCycle refuses the revision whose catalog graph would hold a cycle with a build edge the take-in refuses (design 47 G
@@ -951,7 +952,6 @@ func (ix *Indexer) dry() *dryRun {
x := &Indexer{Module: ix.Module, Manifests: ix.Manifests, Packages: ix.Packages, JudgeID: ix.JudgeID, Files: ix.Files, GoSum: ix.GoSum,
GoMod: ix.GoMod, Now: ix.Now, Builds: ix.Builds, Named: ix.Named, Trunk: ix.Trunk, BranchHead: ix.BranchHead, Read: ix.Read, Contains: ix.Contains,
ReadOpen: nil, OwnBuild: nil, Message: ix.Message, SayRefused: nil, SayAccepted: nil, SayTakeIn: nil, Pending: nil,
ValidationAsks: nil, AskVerb: nil, // a dry run asks no validation (§713)
// The registry milestone's fields: what a take-in would write on the bus stays out of a dry run, so the states of the
// requirements, findings, seat definitions, holds and their bounds are not overlaid, nothing is said, and no hold is
// weighed (Holds and Counted nil hold nothing); ReadFile only reads.
@@ -991,16 +991,15 @@ func (ix *Indexer) dry() *dryRun {
}
// computeOver applies the ready merges, in merge order, to a dry indexer, as one overlay: each later merge's trees
// govern (ADR 0633 §3). It answers the fingerprint the indexer's own module has once they are applied (none is "",
// with its reason), and the dry run, whose overlays hold what the take-in would write. A merge that can never be read
// govern (ADR 0633 §3). It answers the dry run, whose overlays hold what the take-in would write. A merge that can never be read
// is left out, as its take-in will drop it.
func (ix *Indexer) computeOver(ctx context.Context, ready []PendingEntry) (string, *dryRun, error) {
func (ix *Indexer) computeOver(ctx context.Context, ready []PendingEntry) (*dryRun, error) {
d := ix.dry()
// The manifests are judged against the seat definitions the revision carries (seatsof.go, ADR 0633 §1), so the indexer
// it names is not refused for a verb or an event the built-in list lacks.
seats, err := ix.revisionSeats(ctx, ready)
if err != nil {
return "", d, fmt.Errorf("reading the seat definitions of %s: %w", ready[0].Key, err)
return d, fmt.Errorf("reading the seat definitions of %s: %w", ready[0].Key, err)
}
var applyErr error
judge.WithSeats(seats, func() {
@@ -1018,24 +1017,9 @@ func (ix *Indexer) computeOver(ctx context.Context, ready []PendingEntry) (strin
}
})
if applyErr != nil {
return "", d, applyErr
return d, applyErr
}
key, _ := meshcatalogclient.ModulesKey(ix.Module)
value, _, found, err := d.ix.Modules.Get(ctx, key)
if err != nil {
return "", d, err
}
if !found {
return NoBuild + "the indexer holds no record of itself", d, nil
}
m, err := meshcatalogclient.DecodeModule(value)
if err != nil {
return "", d, nil
}
if m.Fp2 == "" {
return NoBuild + m.None2, d, nil
}
return m.Fp2, d, nil
return d, nil
}
// claim marks the entries as taken in by this take-in, by compare-and-set on the revisions read. A lost one (another
@@ -1298,83 +1282,6 @@ func saidWaits(entries []PendingEntry) []takein.TakeIn {
return out
}
// holdFor keeps the ready merges pending for the indexer build they name and writes only what building and moving
// that indexer needs (ADR 0633 §1): the indexer's own module entry, its key of the module graph and the verdict on its
// manifest, from the dry run; the manifests and the packages are written by the dry run itself. The revision is said
// pending under its oldest merge's key, and a wait it was said under before by another key is said taken.
func (ix *Indexer) holdFor(ctx context.Context, ready []PendingEntry, names, own string, d *dryRun) error {
key, _ := meshcatalogclient.ModulesKey(ix.Module)
if err := copyKey(ctx, d.modules, ix.Modules, key); err != nil {
return fmt.Errorf("the indexer's own module entry: %w", err)
}
if gk, err := graph.Key(ix.Module); err == nil {
if err := copyKey(ctx, d.graph, ix.Graph, gk); err != nil && !errors.Is(err, process.ErrNotYet) {
return fmt.Errorf("the indexer's key of the module graph: %w", err)
}
}
// Why the manifest of the indexer the revision names was refused, when it was (judged with the seat definitions the
// revision carries): said with the wait, as a merge the forge bridge could not give is (holdLost).
refused := ""
if value, _, found, err := d.ix.Modules.Get(ctx, key); err == nil && found {
if m, err := meshcatalogclient.DecodeModule(value); err == nil && m.Manifest != "" {
if vk, err := meshcatalogclient.VerdictsKey(m.Manifest); err == nil {
if err := copyKey(ctx, d.verdicts, ix.Verdicts, vk); err != nil && !errors.Is(err, process.ErrNotYet) {
return fmt.Errorf("the verdict on the indexer's manifest: %w", err)
}
if vv, _, ok, err := d.ix.Verdicts.Get(ctx, vk); err == nil && ok {
if v, err := meshcatalogclient.DecodeVerdict(vv); err == nil && meshcatalogclient.VerdictRefuses(v) {
refused = strings.Join(v.Problems, "; ")
}
}
}
}
}
// The indexer's own entry the held revision names is kept as the trunk revision after the newest, every other entry as
// it was (milestone 47/G W4.7: a merged revision naming a newer indexer replaces only the indexer's entry).
if err := ix.KeepView(ctx); err != nil {
return fmt.Errorf("the revision with the indexer's own entry: %w", err)
}
t := revisionWait(takein.TakeInPending, ready, names)
t.Refused = refused
var ended, unmarked []PendingEntry
said, changed := false, false
for _, e := range ready {
switch {
case e.Value.Said == t.Key && e.Value.SaidKind == takein.TakeInPending:
said = true
changed = changed || e.Value.WaitsFor != names || e.Value.Refused != refused
if e.Value.WaitsFor != names || e.Value.Refused != refused {
unmarked = append(unmarked, e)
}
case e.Value.Said != "" && e.Value.SaidKind == takein.TakeInPending:
ended = append(ended, e)
unmarked = append(unmarked, e)
default:
unmarked = append(unmarked, e)
}
}
for _, old := range saidWaits(ended) {
ix.sayTakeIn(ctx, old)
}
if said && !changed {
// Nothing to say on the bus, but the start decided nothing and a reader of the journal sees why: the build learnt, the build the wait names.
ix.logf("take-in: %d merges, oldest %s, still name the indexer build %s, this program runs %s: they wait, as already said", len(ready), t.Key, names, own)
}
if !said || changed {
ix.logf("take-in: %d merges, oldest %s, name the indexer build %s, not %s, which runs: they wait for it", len(ready), t.Key, names, own)
if refused != "" {
ix.logf("take-in: the manifest of the indexer they name is refused, judged with the seat definitions they carry, so no such indexer runs: %s", refused)
}
if !ix.sayTakeIn(ctx, t) {
ix.markSaid(ctx, unmarked, nil, func(p *Pending) { p.WaitsFor, p.LastWriter, p.Refused = names, own, refused })
return nil
}
unmarked = ready
}
ix.markSaid(ctx, unmarked, &t, func(p *Pending) { p.WaitsFor, p.LastWriter, p.Refused = names, own, refused })
return nil
}
// copyKey writes what the overlay holds of key to the store, when it wrote or deleted it.
func copyKey(ctx context.Context, from *overlayStore, to Store, key string) error {
if from == nil || to == nil {
@@ -1439,9 +1346,6 @@ func waitOf(entries []PendingEntry, what, behind string) takein.PendingWait {
if e.Value.Refused != "" {
pw.Refused = e.Value.Refused
}
if e.Value.Validation != nil {
pw.Validation = e.Value.Validation
}
if e.Value.Said != "" && e.Value.SaidAt.After(pw.SaidAt) {
pw.Said, pw.SaidAt = "take-in."+e.Value.SaidKind, e.Value.SaidAt
}
@@ -106,6 +106,16 @@ func (b *takeInBed) merge(number int, c, avahi, indexer string) forgemsg.Forge {
return f
}
// arriveHeld arrives f while the verdicts state is not on the bus, so the take-in holds it (ADR 0668 §4): the way a merge is
// left pending now that no merge waits for the indexer build it names (A-476 b).
func (b *takeInBed) arriveHeld(t *testing.T, seq uint64, f forgemsg.Forge) {
t.Helper()
bound := b.ix.Verdicts
b.ix.Verdicts = &memStore{unbound: process.ErrNotYet}
defer func() { b.ix.Verdicts = bound }()
b.arrive(t, seq, f)
}
func (b *takeInBed) arrive(t *testing.T, seq uint64, f forgemsg.Forge) {
t.Helper()
b.publish(t, f)
@@ -224,16 +234,6 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
if want := []string{"pending " + takein.PendingKey(7), "taken " + takein.PendingKey(7)}; !slices.Equal(b.events(), want) || len(b.keys()) != 0 {
t.Fatalf("said %v, pending %v: want %v and nothing pending", b.events(), b.keys(), want)
}
// Two merges said pending under the oldest's key (waiting for a named indexer), then taken in together: one taken, under
// that key, and no entry's key said twice.
b.said = nil
b.arrive(t, 8, b.merge(31, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.arrive(t, 9, b.merge(32, sha('d'), "tree-avahi-4", "tree-indexer-2"))
b.said = nil
b.arrive(t, 10, b.merge(33, sha('e'), "tree-avahi-5", b.base.Trees["modules/"+testSelf]))
if want := []string{"taken " + takein.PendingKey(8), "taken " + takein.PendingKey(9), "taken " + takein.PendingKey(10)}; !slices.Equal(b.events(), want) || len(b.keys()) != 0 {
t.Errorf("said %v, pending %v: want %v, the wait's key (m.8, said pending) once and each other entry's own", b.events(), b.keys(), want)
}
})
t.Run("T7 and T3 a merge alone, an indexer change, a merge behind it and a revert", func(t *testing.T) {
b := newTakeInBed(t)
@@ -259,42 +259,28 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
t.Fatalf("alone: avahi %s (was %s), pending %v, said %v: want taken in, said taken under its key (1966/AC195-1)", avahi1, avahi0, b.keys(), b.events())
}
b.said = nil
avahi1, verdicts := b.fp(t, "avahi"), len(b.ix.Verdicts.(*memStore).vals)
// T4: a change to the indexer writes only its entry (with its graph key and verdict); the rest is pending.
avahi1 := b.fp(t, "avahi")
// A-476 b: a merge naming another indexer build is taken in at once, with the build this program runs: the indexer's entry
// and avahi are written, nothing is held and said pending.
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
named := b.fp(t, testSelf)
if named == own || b.fp(t, "avahi") != avahi1 || b.commit(t, "avahi") != sha('b') {
t.Fatalf("indexer change: indexer %s (own %s), avahi %s at %s: want only the indexer written", named, own, b.fp(t, "avahi"), b.commit(t, "avahi"))
if named == own || b.fp(t, "avahi") == avahi1 || b.commit(t, "avahi") != sha('c') || len(b.keys()) != 0 {
t.Fatalf("indexer change: indexer %s (own %s), avahi %s at %s, pending %v: want the whole revision taken in at once", named, own, b.fp(t, "avahi"), b.commit(t, "avahi"), b.keys())
}
if g := graphOf(t, b.ix, testSelf); g.FP != named {
t.Errorf("graph of the indexer names %s, want %s", g.FP, named)
}
if n := len(b.ix.Verdicts.(*memStore).vals); n > verdicts+1 {
t.Errorf("verdicts %d, want at most %d: only the new indexer manifest's", n, verdicts+1)
if want := []string{"taken " + takein.PendingKey(2)}; !slices.Equal(b.events(), want) {
t.Errorf("said %v, want %v: nothing said pending", b.events(), want)
}
if want := []string{takein.PendingKey(2)}; !reflect.DeepEqual(b.keys(), want) || !slices.Equal(b.events(), []string{"pending " + takein.PendingKey(2)}) || b.said[0].WaitsFor != named {
t.Fatalf("pending %v, said %+v: want m.2 pending for %s", b.keys(), b.said, named)
}
// T5: the merge delivered again writes no second key and says nothing again.
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
if len(b.keys()) != 1 || len(b.said) != 1 {
t.Errorf("redelivered: pending %v, said %v", b.keys(), b.events())
}
// A later merge joins: the newest governs and still names the new indexer.
// A later merge is taken in at once too, and a revert to the first indexer.
b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
if len(b.keys()) != 2 || b.commit(t, "avahi") != sha('b') || len(b.said) != 1 {
t.Fatalf("behind: pending %v, avahi at %s, said %v: want both pending and nothing said again", b.keys(), b.commit(t, "avahi"), b.events())
}
// A revert brings the indexer back to the running build: everything is taken in, oldest first.
b.arrive(t, 4, b.merge(23, sha('e'), "tree-avahi-5", b.base.Trees["modules/"+testSelf]))
if len(b.keys()) != 0 || b.fp(t, testSelf) != own || b.commit(t, "avahi") != sha('e') {
t.Fatalf("revert: pending %v, indexer %s, avahi at %s: want all taken in", b.keys(), b.fp(t, testSelf), b.commit(t, "avahi"))
}
if want := []string{"pending " + takein.PendingKey(2), "taken " + takein.PendingKey(2), "taken " + takein.PendingKey(3), "taken " + takein.PendingKey(4)}; !slices.Equal(b.events(), want) {
t.Errorf("said %v, want %v", b.events(), want)
}
})
t.Run("IX22 a merged revision naming a newer indexer is the previous catalog graph revision with only the indexer's entry replaced", func(t *testing.T) { // design 47 G §273, §237
t.Run("IX22 a merged revision naming a newer indexer is a catalog graph revision with its entry and the merge's others replaced", func(t *testing.T) { // design 47 G §273, §237
b := newTakeInBed(t)
store := newMemObjects(0)
b.ix.Revisions = store
@@ -305,14 +291,14 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
if err != nil || before.Revision != 1 {
t.Fatalf("the view's revision %+v (%v), want revision 1", before, err)
}
// The merge changes avahi and the indexer: only the indexer is written, the revision is made at once.
// The merge changes avahi and the indexer: both are taken in at once (A-476 b), in a revision after the first.
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
after, _, err := newestTrunk(ctx, store)
if err != nil || after.Revision != 2 {
t.Fatalf("after the merge naming a newer indexer the newest revision is %+v (%v), want revision 2", after, err)
if err != nil || after.Revision <= before.Revision {
t.Fatalf("after the merge naming a newer indexer the newest revision is %+v (%v), want one after revision %d", after, err, before.Revision)
}
if changed := changedEntries(before.Entries, after.Entries); !slices.Equal(changed, []string{testSelf}) {
t.Errorf("the revision differs from the previous in %v, want the indexer's entry alone", changed)
if changed := changedEntries(before.Entries, after.Entries); !slices.Equal(changed, []string{"avahi", testSelf}) && !slices.Equal(changed, []string{testSelf, "avahi"}) {
t.Errorf("the revision differs from the previous in %v, want avahi and the indexer", changed)
}
})
t.Run("P1 a revision that adds a Go module its own indexer uses is computed against that module's packages at its merge", func(t *testing.T) {
@@ -353,71 +339,13 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
t.Fatal(err)
}
b.arrive(t, 2, f)
if len(b.said) != 1 || !strings.HasPrefix(b.said[0].WaitsFor, "src2:") {
t.Errorf("said %+v, want m.2 waiting for the indexer build its merge's packages give, not for a version no take-in has published", b.said)
if len(b.said) != 1 || b.said[0].Kind != takein.TakeInTaken || b.said[0].WaitsFor != "" || len(b.keys()) != 0 {
t.Errorf("said %+v, pending %v, want m.2 taken in at once", b.said, b.keys())
}
if _, ok := b.ix.publishedAll(ctx)[sibling]; !ok {
t.Errorf("the packages the revision publishes at its merge lack %s", sibling)
}
})
t.Run("K1 the indexer a revision names is judged with the seat definitions the revision carries", func(t *testing.T) {
// A successor's manifest invokes a verb of the node-engine's seat that the seat list lacks and the revision's
// definitions add (ADR 0633 §1, §6): refused without them, with its reason said on the wait and in indexer_pending;
// accepted with them, and the revision waits for the build it names.
const verb, contracts = "revision-verb", "modules/seat-contracts"
// The third row's definer is the Delivery domain's contracts module defining a Core seat: not taken (ADR 0636 §5).
for _, row := range []struct {
name string
defined bool
refused bool
module string
}{{"definitions the revision does not carry", false, true, ""}, {"definitions the revision carries", true, false, "seat-contracts"},
{"definitions of a seat outside the carrier's domain", true, true, "mesh-delivery-contracts"}} {
b := newTakeInBed(t)
f := b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")
f.Manifests = map[string]string{}
for dir, raw := range b.base.Manifests {
f.Manifests[dir] = raw
}
dir := "modules/" + testSelf
f.Manifests[dir] = strings.Replace(f.Manifests[dir], `"module": "`+testSelf+`"`,
`"module": "`+testSelf+`", "invokes": ["seat:node-engine.`+verb+`"]`, 1)
if row.defined {
f.Trees[contracts] = "tree-seat-contracts"
f.Locks = map[string]map[string]string{}
for d, l := range b.base.Locks {
f.Locks[d] = l
}
f.Locks[contracts] = map[string]string{}
f.Manifests[contracts] = `{"module":"` + row.module + `","version":"1.0","build":{"artifacts":[{"name":"go-seats","kind":"package","language":"go","paths":["go"],"series":"0.1","generated-from":"."}]}}`
b.ix.Files = func(_ context.Context, repository, commit string, dirs []string) (map[string]map[string]string, error) {
if repository != f.Repository || commit != f.Merge || !slices.Equal(dirs, []string{contracts + "/seats"}) {
t.Errorf("listed %v of %s at %s", dirs, repository, commit)
}
return map[string]map[string]string{dirs[0]: {"node-engine/seat.json": "1", "node-engine/fixtures/a.json": "2"}}, nil
}
b.ix.ReadFile = func(_ context.Context, _, _, p string) ([]byte, error) {
switch p {
case contracts + "/seats/node-engine/seat.json":
return []byte(`{"format":1,"seat":"node-engine","scope":"node","grain":"seat-list","decision":"d","domain":"Core","verbs":[{"name":"` + verb + `"}]}`), nil
}
return nil, nil
}
}
b.arrive(t, 2, f)
waits := PendingWaits(readPending(t, b))
refused := ""
if len(b.said) > 0 {
refused = b.said[0].Refused
}
if got := strings.Contains(refused, `serves no verb "`+verb+`"`); got != row.refused || (len(waits) == 1 && waits[0].Refused != refused) {
t.Errorf("%s: said %q, indexer_pending %+v: want a refusal naming the verb %v", row.name, refused, waits, row.refused)
}
if _, ok := catalogue.SeatNamed("node-engine"); !ok || judgeSeatsHave("node-engine", verb) {
t.Errorf("%s: the seats are not put back after the dry run", row.name)
}
}
})
t.Run("A-326 a revision whose catalog graph holds a cycle with a build edge no build seat breaks is refused, naming it", func(t *testing.T) {
// avahi's artifact needs the indexer's (kind 2), and the indexer is built on avahi (kind 3): avahi holds no build seat, so
// no running version breaks the cycle, and the revision is refused at take-in naming every artifact on it (design 47 G §49).
@@ -443,15 +371,15 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
t.Errorf("indexer_pending %+v, want the refusal naming the cycle", waits)
}
})
t.Run("T10 the named indexer takes the revision in at its start; a gathered group stays", func(t *testing.T) {
t.Run("T10 a start takes what waits in once its build is known; a gathered group stays", func(t *testing.T) {
b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
g := b.merge(30, sha('f'), "", "")
g.Repository, g.Trees, g.Manifests, g.Locks, g.Group = "novox/mesh-sdk", map[string]string{}, nil, nil, "feat/x"
g.Members = []meshdeliveryclient.Head{{Repository: "novox/mesh-sdk", Number: 30, Head: g.Head}, {Repository: "novox/mesh-host", Number: 4, Head: sha('9')}}
b.arrive(t, 5, g)
named := b.fp(t, testSelf)
// The new indexer starts; its build is not known (the engine does not answer at its start): a hand ask is answered
b.arriveHeld(t, 5, g)
own, _ := b.ix.OwnBuild(ctx)
// The indexer starts; its build is not known (the engine does not answer at its start): a hand ask is answered
// meanwhile (R5), and the start waits for nothing. It is never asked again, so it stays unknown until it is started again.
mk := func(ask func(context.Context) (string, error)) *Indexer {
n := &Indexer{Module: testSelf, Store: b.ix.Store, Modules: b.ix.Modules, Manifests: b.ix.Manifests, Graph: b.ix.Graph, Verdicts: b.ix.Verdicts,
@@ -462,18 +390,6 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
n.FixOwnBuild(ctx)
return n
}
// A start that decides nothing says so (issue 1921): the build it learnt, the build the held wait names, also when the
// wait was said before; and indexer_pending names the known build and since when.
oldBuild, _ := b.ix.OwnBuild(ctx)
oldStart := mk(func(context.Context) (string, error) { return oldBuild, nil })
b.logs.Reset()
if _, words, err := oldStart.TakeInAtStart(ctx); err != nil || len(b.keys()) != 2 || !strings.Contains(b.logs.String(), "still name the indexer build") ||
!strings.Contains(b.logs.String(), "at the start, this program runs build") {
t.Fatalf("a start that holds: %q %v, journal %q: want the build learnt and the build the wait names said", words, err, b.logs.String())
}
if answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{}); err != nil || answer.(takein.PendingAnswer).OwnBuild != oldBuild || answer.(takein.PendingAnswer).OwnBuildSince.IsZero() {
t.Errorf("indexer_pending = %+v %v, want the known build and since", answer, err)
}
asked := 0
next := mk(func(context.Context) (string, error) { asked++; return "", errors.New("not yet") })
if r := next.TakeInPendingByHand(ctx, "m.nothing", "lead@shanks", "a test"); r.Stored || !strings.Contains(r.Answer, "nothing waits") {
@@ -483,12 +399,15 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
t.Fatalf("a start with its build unknown: %q %v, pending %v: want it to answer at once and take nothing in", words, err, b.keys())
}
// The engine answers now, and the same process does not ask it again: it learnt nothing it did not know at its start.
next.OwnBuild = func(context.Context) (string, error) { asked++; return named, nil }
next.OwnBuild = func(context.Context) (string, error) { asked++; return own, nil }
if _, words, err := next.TakeInAtStart(ctx); err != nil || asked != 1 || len(b.keys()) != 2 || !strings.Contains(words, "not known") {
t.Fatalf("the window of a replacement: %q %v, asked %d, pending %v: want the build fixed at the start, never learnt later", words, err, asked, b.keys())
}
if answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{}); err != nil || answer.(takein.PendingAnswer).OwnBuild != own || answer.(takein.PendingAnswer).OwnBuildSince.IsZero() {
t.Errorf("indexer_pending = %+v %v, want the known build and since", answer, err)
}
// The module started again fixes the build at its start and takes the revision in.
next = mk(func(context.Context) (string, error) { return named, nil })
next = mk(func(context.Context) (string, error) { return own, nil })
if _, _, err := next.TakeInAtStart(ctx); err != nil {
t.Fatal(err)
}
@@ -544,7 +463,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
if _, words, err := b.ix.EndGroup(ctx, end); err != nil || !strings.Contains(words, "taken in already") {
t.Errorf("an end again: %q %v", words, err)
}
// T11 (the lead's row): a group whose revision changes the indexer: the group's wait is taken, then the revision pending.
// T11: a group whose revision changes the indexer: the group's wait is taken and the revision taken in at once (A-476 b).
b.said = nil
g := member(catalogueRepo, 40, sha('c'))
g.Group, g.Trees = "feat/two", b.merge(0, "", "", "tree-indexer-7").Trees
@@ -553,7 +472,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
if _, _, err := b.ix.EndGroup(ctx, forgemsg.ForgeGroupEnd{Group: "feat/two", Members: []forgemsg.GroupMerge{{Repository: catalogueRepo, Number: 40, Merge: sha('c')}}}); err != nil {
t.Fatal(err)
}
if want := []string{"pending g.feat/two", "taken g.feat/two", "pending " + takein.PendingKey(9)}; !slices.Equal(b.events(), want) {
if want := []string{"pending g.feat/two", "taken g.feat/two", "taken " + takein.PendingKey(9)}; !slices.Equal(b.events(), want) {
t.Errorf("a group naming another indexer: said %v, want %v", b.events(), want)
}
})
@@ -593,11 +512,8 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
}
held := func() *takeInBed {
b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
b.ix.ownMu.Lock()
b.ix.ownFP = b.fp(t, testSelf)
b.ix.ownMu.Unlock()
b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.arriveHeld(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
return b
}
// A probe that fails keeps the group's wait and stops nothing else: an unrelated merge is still taken in.
@@ -719,7 +635,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
})
t.Run("T5 two take-ins of one pending revision: the one that loses the claim writes nothing", func(t *testing.T) {
b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
named := b.fp(t, testSelf)
b.ix.ownMu.Lock()
b.ix.ownFP = named
@@ -774,8 +690,8 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
{"a take-in that fails", func(b *takeInBed) { b.mods.failWrite = once(func(string) bool { return true }) }},
} {
b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.arriveHeld(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
named := b.fp(t, testSelf)
b.ix.ownMu.Lock()
b.ix.ownFP = named
@@ -794,7 +710,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
})
t.Run("W4.6 a full store holds the merge with the reason objects full, says full once, and room takes it in", func(t *testing.T) {
b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
named := b.fp(t, testSelf)
b.ix.ownMu.Lock()
b.ix.ownFP = named
@@ -826,7 +742,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
// The same take-in into a store it fills to 85 % of its cap says filling once, and is taken in.
kept := store.used()
b2 := newTakeInBed(t)
b2.arrive(t, 2, b2.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b2.arriveHeld(t, 2, b2.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b2.ix.ownMu.Lock()
b2.ix.ownFP = b2.fp(t, testSelf)
b2.ix.ownMu.Unlock()
@@ -845,7 +761,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
t.Run("W4.12 a candidate's half dropped by hand frees its own entries, says room and takes the held merge in; the read tools list it; a recompute that moves nothing is equal", func(t *testing.T) {
bed := func(store *memObjects) *takeInBed {
b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.ix.ownMu.Lock()
b.ix.ownFP = b.fp(t, testSelf)
b.ix.ownMu.Unlock()
@@ -930,40 +846,10 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
t.Errorf("indexer_recompute = %+v (%v): want equal to the kept revision 2", got, err)
}
})
t.Run("T7 a toolchain report evaluates pending again: the revision then names the running build and is taken in", func(t *testing.T) {
b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
entries, err := ReadPending(ctx, b.pending, nil)
if err != nil {
t.Fatal(err)
}
// The build the revision names under the next toolchain, worked out on a dry indexer; this one runs it.
x := b.ix.dry().ix
for _, node := range []string{"novox", "ace"} {
key, _ := ToolchainKey(node, "go")
x.PutToolchain("mesh-build-go", key, toolchainValue(t, "go", "go1.28.0", "1"), false)
}
ready, _, _, _ := b.ix.loadNewest(ctx, WaitsOf(entries).Ready)
next, _, err := x.computeOver(ctx, ready)
if err != nil || next == "" || next == b.said[0].WaitsFor {
t.Fatalf("the build under the next toolchain: %q %v (the revision names %s)", next, err, b.said[0].WaitsFor)
}
b.ix.ownMu.Lock()
b.ix.ownFP = next
b.ix.ownMu.Unlock()
reportToolchain(t, b.ix, "mesh-build-go", "novox", "go", "go1.28.0")
if len(b.keys()) != 1 {
t.Fatalf("one node's report: pending %v, want the revision still held (the nodes disagree)", b.keys())
}
reportToolchain(t, b.ix, "mesh-build-go", "ace", "go", "go1.28.0")
if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('c') {
t.Errorf("after the toolchain report: pending %v, avahi at %s: want the revision taken in", b.keys(), b.commit(t, "avahi"))
}
})
t.Run("T12 T13 indexer_pending names the wait; a hand take-in and a drop say what they ended", func(t *testing.T) {
b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.arriveHeld(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
var answer takein.PendingAnswer
for _, tool := range PendingTools(b.pending, nil, nil, nil) {
if tool.Name == PendingTool {
@@ -975,7 +861,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
}
}
if len(answer.Waits) != 1 || answer.Waits[0].Key != takein.PendingKey(2) || !slices.Equal(answer.Waits[0].Holds, []string{testSelf, "avahi"}) ||
answer.Waits[0].Said != takein.TakeInPendingEvent || len(answer.Waits[0].Keys) != 2 {
len(answer.Waits[0].Keys) != 2 {
t.Fatalf("indexer_pending = %+v", answer)
}
// ADR 0668 §6: a later merge covers a pending one only when its commit has the pending merge's as an ancestor.
@@ -990,11 +876,14 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
}
history = "cd"
b.said = nil
bound := b.ix.Verdicts
b.ix.Verdicts = &memStore{unbound: process.ErrNotYet} // m.3 stays held while m.2 is dropped
if r := b.ix.DropPending(ctx, takein.PendingKey(2), "lead@shanks", "replaced by m.3", false); !r.Stored {
t.Fatalf("drop: %+v", r)
}
if len(b.said) < 2 || !b.said[0].Dropped || b.said[0].Key != takein.PendingKey(2) || b.said[0].By != "lead@shanks" || b.said[1].Key != takein.PendingKey(3) {
t.Fatalf("drop said %+v: want m.2 taken (dropped, by) and m.3 pending", b.said)
b.ix.Verdicts = bound
if len(b.keys()) != 1 || b.keys()[0] != takein.PendingKey(3) {
t.Fatalf("pending %v: want m.2 dropped and m.3 pending", b.keys())
}
// Issue 1966 (#650), ADR 0633 §5: the newest pending merge of its repository, which no taken merge covers, is not
// dropped; the answer names the read again. abandon drops it all the same (below).
@@ -1014,8 +903,8 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
// §476, §493: a hand take-in's taken event carries the branch, the source revision it makes and its entries.
if len(b.said) == 1 {
got := b.said[0]
if got.Branch != TrunkBranch || got.Source <= 0 || len(got.Entries) != 1 || got.Entries[0].Module != "avahi" ||
got.Entries[0].Commit != sha('d') || got.Entries[0].Fingerprint == "" {
if got.Branch != TrunkBranch || got.Source <= 0 || len(got.Entries) != 2 || got.Entries[1].Module != "avahi" ||
got.Entries[1].Commit != sha('d') || got.Entries[1].Fingerprint == "" {
t.Errorf("hand take-in said branch %q source %d entries %+v: want main, a source revision and avahi at its commit", got.Branch, got.Source, got.Entries)
} else if newest, any, _ := newestTrunk(ctx, b.ix.Revisions); !any || got.Revision != newest.Revision {
t.Errorf("hand take-in said revision %d, want the revision it made, %d (A-448)", got.Revision, newest.Revision)
@@ -1025,7 +914,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
t.Errorf("the taken event is refused by its own contract: %v", err)
}
}
b.arrive(t, 4, b.merge(23, sha('e'), "tree-avahi-5", "tree-indexer-2"))
b.arriveHeld(t, 4, b.merge(23, sha('e'), "tree-avahi-5", "tree-indexer-2"))
if r := b.ix.DropPending(ctx, takein.PendingKey(4), "lead@shanks", "abandoned on the operator's word", true); !r.Stored || len(b.keys()) != 0 {
t.Errorf("an abandoned drop: %+v, pending %v", r, b.keys())
}
@@ -1062,8 +951,8 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
if _, _, found, _ := b.ix.Store.Get(ctx, key); !found {
t.Fatal("the group has no candidate")
}
b.arrive(t, 2, big(b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")))
b.arrive(t, 3, big(b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")))
b.arriveHeld(t, 2, big(b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")))
b.arriveHeld(t, 3, big(b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")))
raw, _, _, _ := b.pending.Get(ctx, takein.PendingKey(2))
if len(b.keys()) != 2 || len(raw) > 4096 || b.commit(t, "avahi") != sha('b') {
t.Fatalf("a big merge held: pending %v, value %d bytes, avahi at %s: want two small values", b.keys(), len(raw), b.commit(t, "avahi"))
@@ -1139,7 +1028,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
b := newTakeInBed(t)
f := b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")
f.Words, f.Branch = strings.Repeat("x", 300*1024), "feat/g"
b.arrive(t, 2, f) // held by reference, for an indexer build that is not this one
b.arriveHeld(t, 2, f) // held by reference, with the verdicts state not on the bus
b.ix.Taken = newMem()
b.ix.Contains = func(context.Context, string, string, string) (bool, error) { return c.behind, nil }
data, _ := takein.EncodeTaken(takein.Taken{Repository: trunk.RepositoryName(catalogueRepo), Merge: sha('d')})
@@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
@@ -112,33 +111,3 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T
Run: merge},
}
}
// ValidationTools is indexer_validate_again (class act, design 47 G §713): the tools bundle writes an ask in takeins and the
// indexer asks the revision's validation again; record says the hand act in the hand-act log before it is asked.
func ValidationTools(ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time, record func(what, why string) error) []stdio.Tool {
return []stdio.Tool{{Name: ValidateAgainTool,
Description: "Ask again the validation of a pending revision that could not validate (design 47 G §712, §713): the manifest " +
"validator's run on the build seat node-build-go ended could not run, exited other than 0, or answered not both cases. " +
"The revision is asked under its next try; the answer says the validation's state (the ask, its try, accepted, refused, " +
"or why it could not validate). A revision accepted, refused or whose ask is out is asked nothing: a refusal is asked " +
"again only by a later merge of its branch. A hand act, recorded through mesh-hand-acts before it is asked. " +
"indexer_pending names the keys and shows the validation whole.",
Input: map[string]any{"type": "object", "required": []string{"key", "why"}, "properties": map[string]any{
"key": map[string]any{"type": "string", "description": "the revision's wait key, m.<sequence>, as indexer_pending names it"},
"why": map[string]any{"type": "string", "description": "why it is asked again, in plain words"}}},
Run: func(args map[string]any) (any, error) {
str := func(k string) string { s, _ := args[k].(string); return s }
key, value, err := NewValidateAgainAsk(str("key"), str("why"), stdio.CallerOf(args), now())
if err != nil {
return nil, err
}
if err := record("asked the validation of the pending revision "+strings.TrimSpace(str("key"))+" again by hand", strings.TrimSpace(str("why"))); err != nil {
return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was asked: %w", err)
}
a, err := ask(key, value)
if err != nil {
return nil, err
}
return a.Result, nil
}}}
}
@@ -51,9 +51,6 @@ type TakeInAsk struct {
// Abandon, with Drop, drops a pending merge no later merge covers (ADR 0633 §5 asks a later merge; issue 1966, ADR
// 0668 §5): the hand act says why it is abandoned all the same.
Abandon bool `json:"abandon,omitempty"`
// ValidateAgain, with Pending, asks the validation of the revision under that key again (indexer_validate_again, design 47
// G §713): one that could not validate.
ValidateAgain bool `json:"validate-again,omitempty"`
// Merge asks the merge of MergeRepository at this commit to be taken in by hand (indexer_take_in_merge): its merged message
// is asked of the forge bridge and run through the take-in as a delivered one, under its stream sequence Sequence.
Merge string `json:"merge,omitempty"`
@@ -167,20 +164,6 @@ func NewMergeAsk(repository, merge string, sequence uint64, why, by string, at t
return "merge." + merge, data, err
}
// NewValidateAgainAsk is the value indexer_validate_again writes: its key in takeins, `pending.validate-again.<key>`, and the
// value. key is a revision's wait key, m.<sequence>, as indexer_pending names it.
func NewValidateAgainAsk(key, why, by string, at time.Time) (string, []byte, error) {
key = strings.TrimSpace(key)
if !strings.HasPrefix(key, "m.") {
return "", nil, errors.New("name the revision whose validation is asked again by its key, m.<sequence>, as indexer_pending names it")
}
if strings.TrimSpace(why) == "" {
return "", nil, errors.New("say why (why): a hand act is recorded through mesh-hand-acts")
}
data, err := json.Marshal(TakeInAsk{V: 1, Pending: key, ValidateAgain: true, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
return "pending.validate-again." + key, data, err
}
// NewPendingAsk is the value indexer_take_in_pending (drop false) or indexer_drop_pending (drop true) writes: its key
// in takeins, `pending.<take-in|drop>.<key>`, and the value. key is a wait's key as indexer_pending names it, for a
// drop one pending merge's; abandon drops one no later merge covers.
@@ -13,6 +13,7 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"mesh-delivery-indexer/judge"
@@ -127,24 +128,26 @@ type known struct {
// the order the bucket lists them in; modules that stand on each other have a cycle as their reason. A module is written
// only when its fingerprint or its reason changes, and only if its manifest is still the one computed from. A module with
// no kept inputs (a message that carried no manifest) is left as it is.
func (ix *Indexer) Recompute(ctx context.Context) error { return ix.recomputeWith(ctx, nil, nil) }
func (ix *Indexer) Recompute(ctx context.Context) error { return ix.recomputeWith(ctx, nil, nil, "") }
// recomputeWith is Recompute for a merge as well: the records the merge gives (overlay, by module key) stand in place
// of what the bucket holds, are computed with the others, and are written once, with their final fingerprint or reason
// (issue 1444: a merge writes each module once). A record of the overlay that has no kept inputs is written as it is. w is
// the merge's write (order.go), nil for none; it is told each module written.
func (ix *Indexer) recomputeWith(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite) error {
// the merge's write (order.go), nil for none; it is told each module written. A merge's recompute names its repository
// (A-476 c): only the modules registered for it are moved, none of another repository; "" moves every module (a toolchain
// report, a hand recompute).
func (ix *Indexer) recomputeWith(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite, repository string) error {
if err := ix.fillPaths(ctx); err != nil {
return err
}
if err := ix.recomputeModules(ctx, overlay, w); err != nil {
if err := ix.recomputeModules(ctx, overlay, w, repository); err != nil {
return err
}
// What the modules moved to reaches the candidates' fingerprints too (a base, the builder, a toolchain).
return ix.RecomputeCandidates(ctx)
}
func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite) error {
func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite, repository string) error {
if ix.Modules == nil {
return nil
}
@@ -153,11 +156,11 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]mesh
}
ix.recompute.Lock()
defer ix.recompute.Unlock()
return ix.recomputeLocked(ctx, overlay, w)
return ix.recomputeLocked(ctx, overlay, w, repository)
}
// recomputeLocked is recomputeModules with ix.recompute held by the caller (a hold that moves, holds.go).
func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite) error {
func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite, repository string) error {
if ix.Manifests == nil {
return ix.writeOverlay(ctx, overlay, nil, w)
}
@@ -307,6 +310,12 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc
for _, key := range slices.Sorted(maps.Keys(all)) {
k := all[key]
r := of(key, nil)
if _, merged := overlay[key]; !merged && repository != "" && !trunk.SameRepository(k.cur.Repository, repository) {
// A module registered for another repository is not moved by this repository's commit (A-476 c): it stands as held.
r = result{fp: k.held.Fp2, none: k.held.None2, tcs: k.held.Toolchains, pkgs: sdkstep.PackageVersions(k.held.Packages)}
edges[key] = graphEdges{module: key, cur: k.cur, manifest: k.manifest, fp: r.fp, none: r.none, tcs: r.tcs, pkgs: r.pkgs}
continue
}
edges[key] = graphEdges{module: key, cur: k.cur, manifest: k.manifest, fp: r.fp, none: r.none, tcs: r.tcs, pkgs: r.pkgs}
if _, merged := overlay[key]; merged {
final := overlay[key]
@@ -829,9 +829,18 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
t.Helper()
f := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-sdk", Branch: "feat/x", Number: 3, Head: h1,
Merge: commit, Root: tree, Trees: map[string]string{"": tree}, Manifests: map[string]string{"": `{"module":"mesh-sdk","version":"1.0"}`}}
prev, _ := moduleOf(t, mods, "avahi")
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
// A-476 c: a commit of mesh-sdk moves no module registered for another repository; the next recompute (a hand one here)
// moves avahi, which imports its package.
if after, _ := moduleOf(t, mods, "avahi"); after.Fp2 != prev.Fp2 || !reflect.DeepEqual(after.Packages, prev.Packages) {
t.Fatalf("a commit of mesh-sdk moved avahi, registered for another repository: fp2 %q (was %q) packages %v (were %v)", after.Fp2, prev.Fp2, after.Packages, prev.Packages)
}
if err := ix.Recompute(context.Background()); err != nil {
t.Fatal(err)
}
a, _ := moduleOf(t, mods, "avahi")
if a.Fp2 == "" || !reflect.DeepEqual(a.Packages, map[string]string{sdk: commit}) {
t.Fatalf("after mesh-sdk's merge %.8s: fp2 %q none2 %q packages %v", commit, a.Fp2, a.None2, a.Packages)
@@ -1,264 +0,0 @@
package index
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"slices"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"mesh-delivery-indexer/judge"
)
// The validation before take-in (design 47 G §712, A-213, A-232): for a pending revision of a watched branch the running
// indexer asks the seat node-build-go a `run` of its own manifest validator over two cases, change and trunk, and refuses only
// what the revision causes (ADR 0579 §2). The ask rides the check queue under the indexer's name, its kind `run` in the body.
const (
// validationProgram is the manifest validator's program, by its directory under the indexer entry's Go module: the indexer's
// own code, so no repository or module name is fixed here (ADR 0666 §5).
validationProgram = "cmd/mesh-judge"
// ValidationAsker is the asker the ask is put under (buildrun.Asker.Name, A-248): the indexer's seat name, mesh-catalog
// (design 47 G A-163, A-232).
ValidationAsker = meshcatalog.Seat
// validationCatalogSeat is the seat the indexer entry claims.
validationCatalogSeat = "mesh-catalog"
// The cases of the request, as the manifest validator names them.
validationChange = "change"
validationTrunk = judge.TrunkCase
)
// validationInputVerb is where the build seat's holder asks the input: the seat mesh-catalog's verb input (§354).
var validationInputVerb = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbInput
// validationRequest is the manifest validator's request for the ready merges of branch over the branch's current revision
// (§712): case change, every module of the revision by its entry's manifest, each one the merges carry replaced by the merged
// message's bytes (the newest merge governs) and one added for a new directory; case trunk, every module as it stands; the
// seat definitions the merges' trees carry; the packages and the predefined list as the take-in's own validator has them;
// no machines. It answers the indexer entry of the change case too, the one that claims the seat mesh-catalog.
func (ix *Indexer) validationRequest(ctx context.Context, branch string, ready []PendingEntry) (composer.Request, meshcatalog.CatalogEntry, error) {
var indexer meshcatalog.CatalogEntry
if ix.Revisions == nil {
return composer.Request{}, indexer, errors.New("the catalog graph is not kept here: no revision to validate over")
}
rev, found, err := newestOf(ctx, ix.Revisions, branch)
if err != nil {
return composer.Request{}, indexer, err
}
var entries []meshcatalog.CatalogEntry
if found {
if _, entries, err = ReadRevision(ctx, ix.Revisions, RevisionObject(rev)); err != nil {
return composer.Request{}, indexer, err
}
}
trunkCase := composer.Case{Name: validationTrunk, Manifests: map[string]composer.Manifest{}}
byPlace := map[string]string{} // <repository> <path> -> module
of := map[string]meshcatalog.CatalogEntry{}
for _, e := range entries {
raw := ix.manifestBytes(ctx, e.ManifestDigest)
if raw == "" {
b, err := json.Marshal(e.Manifest)
if err != nil {
return composer.Request{}, indexer, fmt.Errorf("the manifest of %s: %w", e.Module, err)
}
raw = string(b)
}
trunkCase.Manifests[e.Module] = composer.Manifest{Raw: json.RawMessage(raw)}
byPlace[placeKey(e.Repository, e.Path)] = e.Module
of[e.Module] = e
}
change := composer.Case{Name: validationChange, Manifests: maps.Clone(trunkCase.Manifests)}
for _, e := range ready {
if e.Covered {
continue
}
for _, dir := range slices.Sorted(maps.Keys(e.Forge.Manifests)) {
raw := e.Forge.Manifests[dir]
module, known := byPlace[placeKey(e.Forge.Repository, dir)]
if !known {
m, err := judge.ForInputs([]byte(raw))
if err != nil || m.Module == "" {
module = e.Forge.Repository + "/" + dir // a new directory whose manifest names no module: judged under its place
} else {
module = m.Module
}
byPlace[placeKey(e.Forge.Repository, dir)] = module
of[module] = meshcatalog.CatalogEntry{Module: module, Repository: e.Forge.Repository, Path: dir}
}
change.Manifests[module] = composer.Manifest{Raw: json.RawMessage(raw)}
}
}
for _, module := range slices.Sorted(maps.Keys(change.Manifests)) {
m, err := judge.ForInputs(change.Manifests[module].Raw)
if err != nil || !slices.ContainsFunc(m.Claims, func(c catalogue.Claim) bool { return c.Name == validationCatalogSeat }) {
continue
}
indexer = of[module]
if indexer.Module == "" {
indexer.Module = module
}
break
}
seats, err := ix.revisionDefinitions(ctx, ready)
if err != nil {
return composer.Request{}, indexer, err
}
req := composer.Request{V: composer.Version, Input: json.RawMessage("{}"), Cases: []composer.Case{change, trunkCase}, Seats: seats,
Predefined: ix.predefinedList(ctx)}
if known := ix.knownPackages(ctx); known != nil {
req.Packages = slices.Sorted(maps.Keys(known))
}
return req, indexer, nil
}
func placeKey(repository, path string) string {
return strings.ToLower(trunk.RepositoryName(repository)) + " " + path
}
// revisionDefinitions are the validation's seat definitions, raw (§712 `seats`, A-241): the branch's revision's seat
// definitions with the definitions the merges' trees carry laid over them, so a revision that touches no definition is still
// judged against the whole seat list. A revision's own definitions are not kept yet (A-234), so the revision's are the
// trunk's, read from the state seat-definitions as the verb seat-definitions answers them.
func (ix *Indexer) revisionDefinitions(ctx context.Context, ready []PendingEntry) ([]json.RawMessage, error) {
defs, err := ix.trunkDefinitions(ctx)
if err != nil {
return nil, err
}
merged, err := ix.revisionSeatDefinitions(ctx, ready)
if err != nil {
return nil, err
}
for seat, d := range merged {
raw, err := json.Marshal(d)
if err != nil {
return nil, fmt.Errorf("the definition of %s: %w", seat, err)
}
defs[seat] = raw
}
var out []json.RawMessage
for _, seat := range slices.Sorted(maps.Keys(defs)) {
out = append(out, defs[seat])
}
return out, nil
}
// trunkDefinitions are the trunk's seat definitions of the state seat-definitions, raw, by seat: every `<seat>.trunk` entry
// that holds a definition; none where the indexer keeps no such state.
func (ix *Indexer) trunkDefinitions(ctx context.Context) (map[string]json.RawMessage, error) {
defs := map[string]json.RawMessage{}
if ix.SeatDefinitions == nil {
return defs, nil
}
keys, err := ix.SeatDefinitions.Keys(ctx)
if err != nil {
return nil, fmt.Errorf("the %s state cannot be listed: %w", SeatDefinitionsState, err)
}
for _, key := range keys {
seat, ok := strings.CutSuffix(key, "."+SourceTrunk)
if !ok || strings.HasPrefix(key, outOfStepPrefix) {
continue
}
v, _, found, err := ix.SeatDefinitions.Get(ctx, key)
if err != nil {
return nil, fmt.Errorf("%s.%s cannot be read: %w", SeatDefinitionsState, key, err)
}
var e SeatDefinitionEntry
if !found || json.Unmarshal(v, &e) != nil || e.Digest == "" || len(e.Definition) == 0 {
continue // an entry that holds no definition is no seat of the list
}
defs[seat] = e.Definition
}
return defs, nil
}
// validationJob is the run of the manifest validator for the request (§712): the program cmd/mesh-judge, its directory under
// the Go module the go.mod at the entry's path names (read by the build, A-268), at the entry's path, built at head, the revision's head of the entry's repository; no
// heads and no arguments; the input the request's bytes, served by the seat mesh-catalog's verb input.
func validationJob(indexer meshcatalog.CatalogEntry, head, branch string, seq uint64, input []byte) (buildstep.RunJob, error) {
switch {
case indexer.Repository == "":
return buildstep.RunJob{}, fmt.Errorf("no entry of the revision claims the seat %s", validationCatalogSeat)
case head == "":
return buildstep.RunJob{}, fmt.Errorf("the revision names no head of %s, the repository of the indexer entry %s", indexer.Repository, indexer.Module)
}
in, err := buildstep.RunInputOf(validationInputVerb, input)
if err != nil {
return buildstep.RunJob{}, err
}
dir := indexer.Path
if dir == "" {
dir = "."
}
return buildstep.RunJob{
Program: buildstep.RunProgram{Repository: indexer.Repository, Commit: head, Dir: dir, Package: "./" + validationProgram},
Input: in,
Reader: fmt.Sprintf("manifest validator of %s at %d", branchName(branch), seq),
}, nil
}
func branchName(branch string) string {
if isTrunkBranch(branch) {
return TrunkBranch
}
return branch
}
// ValidationRefusal is what a validation's verdict refuses (ADR 0579 §2, §712): of each manifest the change case refuses that
// the revision adds or changes (its bytes differ from the trunk case's), each problem the trunk case lacks; and each problem
// between manifests the trunk case lacks. A problem only in unchanged manifests refuses nothing. It is empty when the revision
// is accepted. An answer that holds not both cases is no verdict: it answers an error, so the run could not validate.
func ValidationRefusal(req composer.Request, v composer.Verdict) (map[string][]string, error) {
cases := map[string]composer.CaseVerdict{}
for _, c := range v.Cases {
cases[c.Name] = c
}
change, okChange := cases[validationChange]
trunkV, okTrunk := cases[validationTrunk]
if !okChange || !okTrunk {
return nil, fmt.Errorf("the manifest validator's answer holds not both cases %s and %s", validationChange, validationTrunk)
}
raws := map[string]map[string]string{}
for _, c := range req.Cases {
raws[c.Name] = map[string]string{}
for name, m := range c.Manifests {
raws[c.Name][name] = InputDigest(m.Raw)
}
}
refused := map[string][]string{}
for _, module := range slices.Sorted(maps.Keys(change.Manifests)) {
r := change.Manifests[module]
if r.Accepted || r.Refused == "" {
continue
}
before, had := raws[validationTrunk][module]
if module != judge.Unnamed && had && before == raws[validationChange][module] {
continue // unchanged: its problems refuse nothing, and its entry keeps its verdict (ADR 0579 §1)
}
old := map[string]bool{}
for _, p := range strings.Split(trunkV.Manifests[module].Refused, "\n") {
old[p] = true
}
for _, p := range strings.Split(r.Refused, "\n") {
if p != "" && !old[p] {
refused[module] = append(refused[module], p)
}
}
}
return refused, nil
}
// RefusedWords is a refusal as the pending value's `refused` says it: each manifest and its problems, one manifest a line.
func RefusedWords(refused map[string][]string) string {
var lines []string
for _, module := range slices.Sorted(maps.Keys(refused)) {
lines = append(lines, module+": "+strings.Join(refused[module], "; "))
}
return strings.Join(lines, "\n")
}
@@ -1,130 +0,0 @@
package index
import (
"context"
"encoding/json"
"reflect"
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"mesh-delivery-indexer/judge"
)
// The validation refuses only what the revision causes (design 47 G §712, ADR 0579 §2), and its run names the indexer entry's
// repository, path and the revision's head, and no heads.
func TestValidationRefusesWhatTheRevisionCauses(t *testing.T) {
manifests := func(m map[string]string) map[string]composer.Manifest {
out := map[string]composer.Manifest{}
for k, v := range m {
out[k] = composer.Manifest{Raw: json.RawMessage(v)}
}
return out
}
req := composer.Request{Cases: []composer.Case{
{Name: "change", Manifests: manifests(map[string]string{"a": `{"module":"a","v":2}`, "b": `{"module":"b"}`})},
{Name: "trunk", Manifests: manifests(map[string]string{"a": `{"module":"a"}`, "b": `{"module":"b"}`})},
}}
verdict := func(change, trunk map[string]string) composer.Verdict {
read := func(m map[string]string) map[string]composer.ManifestRead {
out := map[string]composer.ManifestRead{"a": {Accepted: true}, "b": {Accepted: true}}
for k, v := range m {
out[k] = composer.ManifestRead{Refused: v}
}
return out
}
return composer.Verdict{Cases: []composer.CaseVerdict{{Name: "change", Manifests: read(change)}, {Name: "trunk", Manifests: read(trunk)}}}
}
for _, c := range []struct {
name string
v composer.Verdict
want, wantErr string
}{
{"a changed manifest gains a problem", verdict(map[string]string{"a": "new\nold"}, map[string]string{"a": "old"}), "a: new", ""},
{"an unchanged manifest's problem refuses nothing", verdict(map[string]string{"b": "bad"}, nil), "", ""},
{"a problem between manifests the trunk lacks", verdict(map[string]string{judge.Unnamed: "x\ny"}, map[string]string{judge.Unnamed: "y"}), judge.Unnamed + ": x", ""},
{"an answer with one case is no verdict", composer.Verdict{Cases: verdict(nil, nil).Cases[:1]}, "", "not both cases"},
} {
got, err := ValidationRefusal(req, c.v)
if c.wantErr != "" {
if err == nil || !strings.Contains(err.Error(), c.wantErr) {
t.Errorf("%s: %v, want %q", c.name, err, c.wantErr)
}
continue
}
if err != nil || RefusedWords(got) != c.want {
t.Errorf("%s: %q, %v, want %q", c.name, RefusedWords(got), err, c.want)
}
}
head := strings.Repeat("a", 40)
entry := meshcatalog.CatalogEntry{Module: "mesh-delivery-indexer", Repository: "novox/mesh-delivery", Path: "modules/mesh-delivery-indexer",
Artifacts: []meshcatalog.Artifact{{Language: "go", Kind: "bundle"}}}
job, err := validationJob(entry, head, "main", 7, []byte(`{"v":1}`))
if err != nil {
t.Fatal(err)
}
want := buildstep.RunProgram{Repository: "novox/mesh-delivery", Commit: head, Dir: "modules/mesh-delivery-indexer", Package: "./cmd/mesh-judge"}
if job.Program != want || len(job.Heads) != 0 || len(job.Args) != 0 || job.Input.Verb != "seat:mesh-catalog.input" || job.Reader != "manifest validator of main at 7" {
t.Errorf("the job %+v", job)
}
if _, err := validationJob(meshcatalog.CatalogEntry{}, head, "main", 7, nil); err == nil || !strings.Contains(err.Error(), "claims the seat mesh-catalog") {
t.Errorf("no indexer entry: %v", err)
}
}
// The validation's seats are the branch's revision's seat definitions with the merges' changed ones laid over them (A-241): a
// revision that touches no definition is judged against the whole seat list, and a merge's definition replaces its seat's.
func TestValidationSeatsAreTheRevisionsWithTheMergesOver(t *testing.T) {
ctx := context.Background()
defs := newMem()
for seat, raw := range map[string]string{"node-engine": `{"seat":"node-engine","verbs":[{"name":"old"}]}`, "git": `{"seat":"git"}`} {
v, _ := json.Marshal(SeatDefinitionEntry{Seat: seat, Source: SourceTrunk, Digest: "d-" + seat, Definition: json.RawMessage(raw)})
defs.put(SeatDefinitionKey(seat, SourceTrunk, "", 0), v)
}
v, _ := json.Marshal(SeatDefinitionEntry{Seat: "pulled", Source: SourcePull, Pull: 3, Digest: "d", Definition: json.RawMessage(`{"seat":"pulled"}`)})
defs.put(SeatDefinitionKey("pulled", SourcePull, "r", 3), v)
ix := &Indexer{Module: testSelf, SeatDefinitions: defs}
seatsOf := func(raws []json.RawMessage) map[string]string {
out := map[string]string{}
for _, raw := range raws {
var d struct {
Seat string `json:"seat"`
Verbs []struct {
Name string `json:"name"`
} `json:"verbs"`
}
if err := json.Unmarshal(raw, &d); err != nil {
t.Fatal(err)
}
out[d.Seat] = ""
if len(d.Verbs) > 0 {
out[d.Seat] = d.Verbs[0].Name
}
}
return out
}
got, err := ix.revisionDefinitions(ctx, nil)
if err != nil || !reflect.DeepEqual(seatsOf(got), map[string]string{"node-engine": "old", "git": ""}) {
t.Errorf("a revision touching no definition: %v (%v), want the trunk's whole list and no pull request's", seatsOf(got), err)
}
const contracts = "modules/seat-contracts"
f := forgemsg.Forge{Repository: "r", Merge: strings.Repeat("c", 40), Manifests: map[string]string{
contracts: `{"module":"seat-contracts","version":"1.0","build":{"artifacts":[{"name":"go-seats","kind":"package","language":"go","paths":["go"],"series":"0.1","generated-from":"."}]}}`}}
ix.Files = func(_ context.Context, _, _ string, dirs []string) (map[string]map[string]string, error) {
return map[string]map[string]string{dirs[0]: {"node-engine/seat.json": "1"}}, nil
}
ix.ReadFile = func(_ context.Context, _, _, p string) ([]byte, error) {
if p == contracts+"/seats/node-engine/seat.json" {
return []byte(`{"format":1,"seat":"node-engine","scope":"node","grain":"seat-list","decision":"d","domain":"Core","verbs":[{"name":"new"}]}`), nil
}
return nil, nil
}
got, err = ix.revisionDefinitions(ctx, []PendingEntry{{Forge: f}})
if err != nil || !reflect.DeepEqual(seatsOf(got), map[string]string{"node-engine": "new", "git": ""}) {
t.Errorf("a merge changing a definition: %v (%v), want its definition over the trunk's and the rest kept", seatsOf(got), err)
}
}
@@ -1,329 +0,0 @@
package index
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
)
// The pending revision's wait for its validation (design 47 G §713, A-232, A-247): the take-in puts the run ask of §712, writes
// `validation` on the revision's newest pending value and ends there; the build seat's step-ended of that ask, heard through the
// indexer's own durable consumer, resumes it: accepted takes the revision in as before, refused keeps it pending with `refused`,
// and an end that could not validate keeps it pending with `failed`, asked again by a later merge of the branch or by hand.
// ValidationAsks puts a run ask on the build seat (buildrun.Asker).
type ValidationAsks interface {
Put(ctx context.Context, a buildstep.StepAsk) error
}
// newestOfRevision is the index in ready of the revision's newest merge, the value that carries `validation` (A-247).
func newestOfRevision(ready []PendingEntry) int {
n := 0
for i, e := range ready {
if e.Value.Seq > ready[n].Value.Seq {
n = i
}
}
return n
}
// branchOfRevision is the watched branch the revision's merges landed on, its newest merged message's base (design 47 G §714);
// a message that names none is the trunk's. A pending value's own branch is its pull request's, never the branch validated.
func branchOfRevision(ready []PendingEntry) string {
if base := ready[newestOfRevision(ready)].Forge.Base; base != "" {
return base
}
return TrunkBranch
}
// validationGate says whether the ready revision waits for its validation (§713): it goes on to the take-in once its newest
// value's validation was accepted at that value's sequence; it waits while its ask is out, while it is refused, and while it
// could not validate; otherwise it puts a new ask, the next try, and waits. Without an asker nothing is validated.
func (ix *Indexer) validationGate(ctx context.Context, ready []PendingEntry, own string) (bool, string, error) {
if ix.ValidationAsks == nil {
return false, "", nil
}
newest := ready[newestOfRevision(ready)]
seq := newest.Value.Seq
if v := newest.Value.Validation; v != nil && v.Seq == seq {
switch {
case v.Accepted == seq:
return false, "", nil
case v.Failed != "":
return true, fmt.Sprintf("the validation of the revision at %d could not validate: %s", seq, v.Failed), nil
case newest.Value.Refused != "":
return true, fmt.Sprintf("the revision at %d is refused: %s", seq, newest.Value.Refused), nil
case v.Ask != "":
return true, fmt.Sprintf("the revision at %d waits for its validation, the ask %s", seq, v.Ask), nil
}
}
return true, "", ix.askValidation(ctx, ready, own)
}
// askValidation puts the revision's validation ask (§712) and writes it on its newest value, the try after every earlier try
// of the revision; an older value's validation goes (A-247: on the newest alone). A request or job that cannot be made could not
// validate: it is written as `failed`, said, and no ask is put.
func (ix *Indexer) askValidation(ctx context.Context, ready []PendingEntry, own string) error {
n := newestOfRevision(ready)
newest := ready[n]
try := 0
for _, e := range ready {
if v := e.Value.Validation; v != nil && v.Try > try {
try = v.Try
}
}
v := &takein.Validation{Seq: newest.Value.Seq, Try: try + 1, Asked: ix.now()}
branch := branchOfRevision(ready)
ask := buildstep.StepAsk{ID: buildstep.NewID(), Kind: buildstep.AskCheck, Language: buildstep.AskGo, Asker: ValidationAsker, At: ix.now().UTC()}
req, entry, err := ix.validationRequest(ctx, branch, ready)
var raw []byte
if err == nil {
raw, err = json.Marshal(req)
}
var job buildstep.RunJob
if err == nil {
job, err = validationJob(entry, revisionHead(ready, entry.Repository, entry.Commit), branch, v.Seq, raw)
}
if err == nil {
v.Input, err = ix.KeepInput(ctx, ask.ID, raw)
}
if err != nil {
v.Failed = "the validation could not be asked: " + err.Error()
} else {
v.Ask, ask.Run = ask.ID, &job
}
if err := ix.writeValidation(ctx, ready, n, v, own); err != nil {
return err
}
if v.Failed != "" {
ix.sayValidation(ctx, ready, v)
return nil
}
if err := ix.ValidationAsks.Put(ctx, ask); err != nil {
return fmt.Errorf("the validation ask %s of the revision at %d could not be put: %w", ask.ID, v.Seq, err)
}
ix.logf("take-in: the revision at %d is validated by the ask %s (try %d); it waits for its end", v.Seq, ask.ID, v.Try)
return nil
}
// revisionHead is the revision's head of repository: the newest merge of it the revision covers, else the commit its entry stands at.
func revisionHead(ready []PendingEntry, repository, commit string) string {
var seq uint64
head := commit
for _, e := range ready {
if e.Forge.Merge != "" && strings.EqualFold(e.Forge.Repository, repository) && e.Value.Seq >= seq {
seq, head = e.Value.Seq, e.Forge.Merge
}
}
return head
}
// writeValidation writes v on ready[n] and takes it from every other value, each by compare-and-set; a new validation clears the
// value's refusal. A value lost under it is an error: the evaluation is made again.
func (ix *Indexer) writeValidation(ctx context.Context, ready []PendingEntry, n int, v *takein.Validation, own string) error {
for i, e := range ready {
if i != n && e.Value.Validation == nil {
continue
}
p := e.Value
p.Validation, p.LastWriter = nil, own
if i == n {
p.Validation, p.Refused = v, ""
}
data, err := encodePending(p)
if err != nil {
return err
}
if err := ix.Pending.Update(ctx, e.Key, data, e.Revision); err != nil {
return fmt.Errorf("%s.%s: %w", PendingState, e.Key, err)
}
}
return nil
}
// sayValidation says the revision's wait in take-in.pending with its refusal or why it could not validate.
func (ix *Indexer) sayValidation(ctx context.Context, ready []PendingEntry, v *takein.Validation) {
t := revisionWait(takein.TakeInPending, ready, "")
t.Failed = v.Failed
t.Refused = ready[newestOfRevision(ready)].Value.Refused
ix.sayTakeIn(ctx, t)
}
// errNoValidationAnswer is an end whose answer is no verdict of both cases.
var errNoValidationAnswer = errors.New("no answer")
// HearValidationEnded takes in the end of a validation ask (§713), heard on the build seat's step-ended through the indexer's own
// durable consumer: only an end of the indexer's asks outside a pipeline; an end whose ask no pending value names now is dropped,
// written in the journal. The answer is read from the event's own output or output object (A-247).
func (ix *Indexer) HearValidationEnded(ctx context.Context, e buildstep.StepEnded) error {
if e.Asker != ValidationAsker || e.Pipeline != "" || e.Ask == "" || ix.Pending == nil || !ix.pendingOnBus(ctx) {
return nil
}
own := ix.ownBuildNow(ctx)
ix.pendingMu.Lock()
defer ix.pendingMu.Unlock()
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
if err != nil {
return err
}
ready := WaitsOf(entries).Ready
n := -1
for i, p := range ready {
if p.Value.Validation != nil && p.Value.Validation.Ask == e.Ask {
n = i
}
}
if n < 0 {
ix.logf("take-in: the end of the validation ask %s (%s on %s) is dropped: no pending revision names that ask now", e.Ask, e.Outcome, e.Node)
return nil
}
ready, lost, _, _ := ix.loadNewest(ctx, ready)
if len(lost) > 0 {
return fmt.Errorf("the revision the validation ask %s ended for holds a merge that cannot be read", e.Ask)
}
n = newestOfRevision(ready)
v := *ready[n].Value.Validation
refused, failed := ix.validationAnswer(ctx, ready, e, v)
p := ready[n].Value
switch {
case failed != "":
v.Failed = failed
case refused != "":
p.Refused = refused
default:
v.Accepted = v.Seq
}
p.Validation, p.LastWriter = &v, own
data, err := encodePending(p)
if err != nil {
return err
}
if err := ix.Pending.Update(ctx, ready[n].Key, data, ready[n].Revision); err != nil {
return fmt.Errorf("%s.%s: %w", PendingState, ready[n].Key, err)
}
if v.Accepted == 0 {
ready[n].Value = p
ix.sayValidation(ctx, ready, &v)
ix.logf("take-in: the validation ask %s of the revision at %d ended: refused %q, could not validate %q", e.Ask, v.Seq, refused, failed)
return nil
}
ix.logf("take-in: the validation ask %s accepted the revision at %d", e.Ask, v.Seq)
if own == "" {
return nil // the take-in waits for the build this program runs, as every take-in does
}
_, _, err = ix.evaluate(ctx, own, false)
return err
}
// validationAnswer reads the end's answer: the refusal of the revision, or why it could not validate (the step did not pass, the
// program exited other than 0, its answer is no verdict of both cases, or its request is no longer the one asked).
func (ix *Indexer) validationAnswer(ctx context.Context, ready []PendingEntry, e buildstep.StepEnded, v takein.Validation) (string, string) {
if e.Outcome != buildstep.OutcomePassed {
return "", strings.TrimSpace(e.Outcome + ": " + e.Said)
}
// The answer is the program's last line of standard output, inline or kept by the seat as an object (A-254); a program that
// exited other than 0 ended the step failed, read above.
out, err := e.ReadOutput(buildstep.AskGo, ix.AskVerb)
if err != nil {
return "", "the run's answer could not be read: " + err.Error()
}
var verdict composer.Verdict
if err := json.Unmarshal(out, &verdict); err != nil {
return "", fmt.Sprintf("%v: the manifest validator's output is no verdict: %v", errNoValidationAnswer, err)
}
req, _, err := ix.validationRequest(ctx, branchOfRevision(ready), ready)
var raw []byte
if err == nil {
raw, err = json.Marshal(req)
}
if err != nil {
return "", "the request cannot be made again to read the verdict: " + err.Error()
}
if runDigest(raw) != v.Input {
return "", "the revision's request is no longer the one the ask " + v.Ask + " was asked with"
}
refused, err := ValidationRefusal(req, verdict)
if err != nil {
return "", err.Error()
}
return RefusedWords(refused), ""
}
// ValidateAgainTool asks the validation of a revision that could not validate again by hand (§713).
const ValidateAgainTool = "indexer_validate_again"
// validationWords is the validation's state as the hand ask answers it.
func validationWords(v *takein.Validation, refused string) string {
switch {
case v == nil:
return "no validation is asked yet"
case v.Accepted != 0:
return fmt.Sprintf("accepted at %d (try %d, the ask %s)", v.Accepted, v.Try, v.Ask)
case v.Failed != "":
return fmt.Sprintf("could not validate at %d (try %d): %s", v.Seq, v.Try, v.Failed)
case refused != "":
return fmt.Sprintf("refused at %d (try %d): %s", v.Seq, v.Try, refused)
default:
return fmt.Sprintf("the ask %s is out since %s (try %d at %d)", v.Ask, v.Asked.UTC().Format(time.RFC3339), v.Try, v.Seq)
}
}
// ValidateAgainByHand is indexer_validate_again answered (§713): the ready revision under key (its oldest merge's m. key, as
// indexer_pending names it) whose validation could not validate is asked again, the next try; its answer says the ask's state.
// A revision accepted, refused or whose ask is out is asked nothing: a refusal is asked again only by a later merge of its branch.
func (ix *Indexer) ValidateAgainByHand(ctx context.Context, key, by, why string) TakenIn {
r := TakenIn{Module: key}
if ix.Pending == nil || ix.ValidationAsks == nil {
r.Answer = "this indexer asks no validation: nothing asked"
return r
}
own := ix.ownBuildNow(ctx)
ix.pendingMu.Lock()
defer ix.pendingMu.Unlock()
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
if err != nil {
r.Answer = err.Error()
return r
}
ready := WaitsOf(entries).Ready
if len(ready) == 0 || ready[0].Key != key {
r.Answer = fmt.Sprintf("no revision waits under %s: indexer_pending names the waits", key)
return r
}
ready, lost, reasons, _ := ix.loadNewest(ctx, ready)
if len(lost) > 0 {
r.Answer = fmt.Sprintf("nothing asked: the message of %s is not read (%s)", lost[0].Key, reasons[lost[0].Key])
return r
}
newest := ready[newestOfRevision(ready)]
v := newest.Value.Validation
if v == nil || v.Seq != newest.Value.Seq || v.Failed == "" {
r.Answer = "nothing asked: the validation is " + validationWords(v, newest.Value.Refused) + "; only one that could not validate is asked again by hand"
return r
}
ix.logf("take-in: the validation of %s is asked again by hand by %s: %s", key, by, why)
if err := ix.askValidation(ctx, ready, own); err != nil {
r.Answer = "the validation was not asked again: " + err.Error()
return r
}
after, err := ReadPending(ctx, ix.Pending, ix.logf)
if err == nil {
ready = WaitsOf(after).Ready
}
if err != nil || len(ready) == 0 {
r.Answer = "asked again; its state is not read back"
return r
}
n := ready[newestOfRevision(ready)]
r.Stored = n.Value.Validation != nil && n.Value.Validation.Failed == ""
r.Answer = fmt.Sprintf("asked again by %s (%s): the validation is %s", by, why, validationWords(n.Value.Validation, n.Value.Refused))
return r
}
@@ -1,113 +0,0 @@
package index
import (
"context"
"encoding/json"
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
)
type askRecorder struct{ asks []buildstep.StepAsk }
func (r *askRecorder) Put(_ context.Context, a buildstep.StepAsk) error {
r.asks = append(r.asks, a)
return nil
}
// Design 47 G §713, A-247: a pending revision names its ask on its newest value and resumes on that ask's end; an end of a
// replaced ask is dropped; a run that could not validate is no refusal and never passed; the start asks nothing again while
// the ask is out.
func TestAPendingRevisionWaitsForItsValidation(t *testing.T) {
ctx := context.Background()
b := newTakeInBed(t)
asks := &askRecorder{}
b.ix.ValidationAsks, b.ix.Inputs, b.ix.Revisions = asks, newMem(), newMemObjects(0)
// No entry of the revision claims the seat mesh-catalog: the validation could not be asked, so it is failed, said, and
// nothing is taken in.
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", ""))
entries := readPending(t, b)
if len(entries) != 1 || entries[0].Value.Validation == nil || !strings.Contains(entries[0].Value.Validation.Failed, "claims the seat mesh-catalog") || len(asks.asks) != 0 {
t.Fatalf("pending %+v, asks %d: want m.2 pending with failed and no ask", entries, len(asks.asks))
}
if last := b.said[len(b.said)-1]; last.Kind != takein.TakeInPending || last.Failed == "" || last.Refused != "" {
t.Errorf("said %+v, want take-in.pending naming failed, never a refusal", last)
}
if w := PendingWaits(entries); len(w) != 1 || w[0].Validation == nil || w[0].Validation.Try != 1 {
t.Errorf("indexer_pending %+v, want the validation whole", w)
}
// An ask out: the newest value names it; the start asks nothing new while it is out.
loaded, _, _, _ := b.ix.loadNewest(ctx, entries)
req, _, err := b.ix.validationRequest(ctx, TrunkBranch, loaded)
if err != nil {
t.Fatal(err)
}
raw, _ := json.Marshal(req)
setValidation := func(v takein.Validation) {
e := readPending(t, b)[0]
p := e.Value
p.Validation = &v
data, _ := encodePending(p)
if err := b.pending.Update(ctx, e.Key, data, e.Revision); err != nil {
t.Fatal(err)
}
}
setValidation(takein.Validation{Ask: "mvl0frv18d1957", Input: runDigest(raw), Seq: 2, Try: 2})
if _, _, err := b.ix.TakeInAtStart(ctx); err != nil || len(asks.asks) != 0 || len(b.keys()) != 1 {
t.Fatalf("the start: %v, asks %d, pending %v: want it waiting for its ask", err, len(asks.asks), b.keys())
}
end := func(ask, outcome, said string, answer any) buildstep.StepEnded {
out, _ := json.Marshal(answer)
return buildstep.StepEnded{Ask: ask, ID: ask, Asker: ValidationAsker, Node: "ace", Outcome: outcome, Said: said, Output: string(out)}
}
// The answer is the manifest validator's verdict itself, the program's last line (A-254).
accepted := composer.Verdict{Cases: []composer.CaseVerdict{{Name: "change"}, {Name: "trunk"}}}
// The end of a replaced ask is dropped, written in the journal.
if err := b.ix.HearValidationEnded(ctx, end("mvl0frv18d1950", buildstep.OutcomePassed, "", accepted)); err != nil || len(b.keys()) != 1 || !strings.Contains(b.logs.String(), "is dropped") {
t.Fatalf("a replaced ask's end: %v, pending %v", err, b.keys())
}
// could not run: pending with failed, never passed.
if err := b.ix.HearValidationEnded(ctx, end("mvl0frv18d1957", buildstep.OutcomeCouldNotRun, "no build node", nil)); err != nil {
t.Fatal(err)
}
if v := readPending(t, b)[0].Value.Validation; v == nil || v.Failed != "could not run: no build node" || v.Accepted != 0 {
t.Fatalf("after could not run: %+v", v)
}
// indexer_validate_again (§713): its hand act recorded first, then the next try asked; the answer says the ask's state.
var recorded []string
tools := ValidationTools(func(key string, value []byte) (TakeInAsk, error) {
a, err := DecodeTakeIn(value)
if err != nil || key != "pending.validate-again."+takein.PendingKey(2) || !a.ValidateAgain {
t.Fatalf("asked %s %+v %v", key, a, err)
}
r := b.ix.ValidateAgainByHand(ctx, a.Pending, a.By, a.Why)
a.Result = &r
return a, nil
}, b.ix.now, func(what, why string) error { recorded = append(recorded, what); return nil })
if len(tools) != 1 || tools[0].Name != ValidateAgainTool {
t.Fatalf("tools = %v", tools)
}
if _, err := tools[0].Run(map[string]any{"key": takein.PendingKey(2)}); err == nil || len(recorded) != 0 {
t.Fatalf("asked again with no why: %v, %v", err, recorded)
}
got, err := tools[0].Run(map[string]any{"key": takein.PendingKey(2), "why": "the build node is back"})
if v := readPending(t, b)[0].Value.Validation; err != nil || len(recorded) != 1 || v == nil || v.Try != 3 || v.Failed == "could not run: no build node" ||
!strings.Contains(got.(*TakenIn).Answer, "try 3") {
t.Fatalf("asked again: %v, recorded %v, validation %+v, answer %+v: want the try after 2 asked", err, recorded, v, got)
}
setValidation(takein.Validation{Ask: "mvl0frv18d1959", Input: runDigest(raw), Seq: 2, Try: 3})
if r := b.ix.ValidateAgainByHand(ctx, takein.PendingKey(2), "lead", "again"); r.Stored || !strings.Contains(r.Answer, "is out") || readPending(t, b)[0].Value.Validation.Try != 3 {
t.Fatalf("an ask out was asked again: %+v", r)
}
// Asked again (by hand or by a later merge) and accepted: the revision is taken in, and its keys go.
setValidation(takein.Validation{Ask: "mvl0frv18d1958", Input: runDigest(raw), Seq: 2, Try: 3})
if err := b.ix.HearValidationEnded(ctx, end("mvl0frv18d1958", buildstep.OutcomePassed, "", accepted)); err != nil {
t.Fatal(err)
}
if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('c') {
t.Fatalf("accepted: pending %v %+v, avahi at %s: want it taken in\n%s", b.keys(), readPending(t, b)[0].Value, b.commit(t, "avahi"), b.logs.String())
}
}
@@ -204,9 +204,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
// milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered.
index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool,
index.EntryIndexTool, index.NamedDropTool, // design 47 G §675
index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.UnpinTool, index.ConsumerTool, index.RunsTool, index.RunDropTool, // design 47 G §667, §8, §579, §64 (A-194)
index.ValidateAgainTool) // design 47 G §713
if !reflect.DeepEqual(m.Tools, want) || len(want) != 52 || !slices.Contains(m.Tools, "modules") {
index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.UnpinTool, index.ConsumerTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194)
if !reflect.DeepEqual(m.Tools, want) || len(want) != 51 || !slices.Contains(m.Tools, "modules") {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
// Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section.
+1 -2
View File
@@ -77,8 +77,7 @@
"indexer_unpin",
"indexer_consumer",
"indexer_runs",
"indexer_run_drop",
"indexer_validate_again"
"indexer_run_drop"
],
"consumes": [
"go-package-registry.published",
@@ -266,21 +266,6 @@ type PendingWait struct {
Unread []string `json:"unread,omitempty"`
// Refused is why the manifest of the indexer it names was refused (TakeIn.Refused).
Refused string `json:"refused,omitempty"`
// Validation is the revision's validation whole, as its newest pending value keeps it (design 47 G §713, A-247).
Validation *Validation `json:"validation,omitempty"`
}
// Validation is a pending revision's validation (design 47 G §712, §713, A-232, A-247), kept on the revision's newest pending
// value alone: the run ask's id, when it was asked, its input's digest, the sequence of the newest merge it covers and the try
// number for it, why it could not validate, and the sequence it was accepted at (never asked again once set).
type Validation struct {
Ask string `json:"ask,omitempty"`
Asked time.Time `json:"asked,omitzero"`
Input string `json:"input,omitempty"`
Seq uint64 `json:"seq"`
Try int `json:"try"`
Failed string `json:"failed,omitempty"`
Accepted uint64 `json:"accepted,omitempty"`
}
// PendingAnswer is indexer_pending's answer: every wait, oldest first, and the pending keys that do not read.