Compare commits

..
Author SHA1 Message Date
jschoubben a5b763f8a2 Merge remote-tracking branch 'origin/main' into feat/try-speed 2026-10-25 03:17:48 +01:00
jschoubben 50a2e4c4e2 mesh-delivery-checker: let a try asked by a bare commit take a mesh-sdk pin that is the head of a mesh-sdk branch, naming it, as a try asked by branch does (issue 1921, issue 1624, try speed S12) 2026-10-25 02:03:26 +02:00
jschoubben 4169faf3f0 mesh-delivery-checker: report a lint finding as the lint step's failure (ADR 0542 §2); pin mesh-sdk's try speed-up head with main in it in the checker and the indexer (issue 1921) 2026-10-25 01:31:53 +02:00
jschoubben 103eec8855 Merge remote-tracking branch 'origin/main' into feat/try-speed 2026-10-25 01:29:44 +02:00
jschoubben 8a62f23afe mesh-delivery-indexer: pin mesh-sdk at the try speed-up's head, beside the checker (issue 1921) 2026-10-25 01:27:18 +02:00
jschoubben 84395420ee mesh-delivery-checker: take an ask's longest level off its time for the clone, as the levels now run side by side, and pin mesh-sdk's side-by-side levels (issue 1921, try speed S10) 2026-10-25 01:24:37 +02:00
jschoubben 9740842af7 mesh-delivery-checker: stop a check whose heads its group no longer holds and end it failed, superseded; judge a group's open members by the verdict it had once others merged, so their merge is not refused (issue 1921, issue 1791, try speed S9) 2026-10-25 01:20:29 +02:00
jschoubben 588cfd54bb mesh-delivery-indexer: keep a merged member in its group's candidate until every member merged, so the heads the Approve covers hold and no new check starts for the members left (issue 1921, issue 1791, try speed S9) 2026-10-25 01:18:20 +02:00
jschoubben 4ccd78c9ec mesh-delivery-checker: put every member's check ask of a group at once and judge the answers in member order, so a group's members run side by side (issue 1921, try speed S8) 2026-10-25 01:15:28 +02:00
jschoubben 6fe7b7dd6c Name every failed test in a check's verdict and a try's failed list (issue 1921)
The checker stopped reading a level's test results at the first that did
not pass and kept only its first failure, so a check named one fault per
round. It now reads and counts every result and writes every failed test
with its first line into the verdict's Failures and the try's failed
list (at most 50, then how many more); Failed keeps the first one's
words whole (issue 1901). Pins mesh-sdk fix/1921-every-failure.
2026-10-24 23:44:46 +02:00
141 changed files with 1342 additions and 13751 deletions
@@ -19,7 +19,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/announce"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/provider"
@@ -126,8 +125,8 @@ func (b *bed) startStoppable(t *testing.T) (<-chan error, func()) {
return done, stopped
}
// 1297/B2b: started whole on its credential, the assigner says on the bus every state its manifest declares, each readable,
// having opened every bucket it names. The states are read from module.json, so a state added there is expected here too.
// 1297/B2b: started whole on its credential, the assigner says on the bus the four states it keeps, each readable,
// having opened every bucket it names.
func TestTheAssignerStartsAndSaysWhatItKeeps(t *testing.T) {
b := startBed(t, "")
done := b.start(t)
@@ -151,21 +150,9 @@ func TestTheAssignerStartsAndSaysWhatItKeeps(t *testing.T) {
}
break
}
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
if err != nil {
t.Fatal(err)
}
m, err := catalogue.ParseManifest(raw)
if err != nil {
t.Fatal(err)
}
if said.Module != "mesh-delivery-assigner" || said.Node != "n1" || len(said.States) != len(m.State) {
t.Fatalf("the assigner said %+v, want the %d states its manifest declares", said, len(m.State))
}
for _, s := range m.State {
if !said.Readable(s.Name) {
t.Errorf("the assigner said %+v: its state %s is not readable", said, s.Name)
}
if said.Module != "mesh-delivery-assigner" || said.Node != "n1" || len(said.States) != 4 || !said.Readable("asks") ||
!said.Readable("wants") || !said.Readable("differences") || !said.Readable("compositions") {
t.Fatalf("the assigner said %+v, want its four states, each readable", said)
}
}
@@ -182,42 +169,6 @@ func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
t.Errorf("without %s the assigner started", missing)
}
}
// ADR 0546: without the puts' bucket it starts all the same, and once the bucket is made (by the controller at its start,
// ADR 0201) a pass opens it, so a put is taken further without a restart.
was := PutsReopenEvery
PutsReopenEvery = 0
defer func() { PutsReopenEvery = was }()
b := startBed(t, delivery.AssignerPutsBucket)
done := b.start(t)
select {
case err := <-done:
t.Fatalf("without %s the assigner stopped: %v", delivery.AssignerPutsBucket, err)
case <-time.After(2 * time.Second):
}
puts, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: delivery.AssignerPutsBucket, History: 5})
if err != nil {
t.Fatal(err)
}
pkey, _ := delivery.PutKey("n1", "web")
asked := delivery.Put{FP: delivery.SourcePrefix2 + strings.Repeat("b", 64), Why: "a try", At: time.Now()}.Said(time.Now(), delivery.PutAsked, "asked")
raw, _ := delivery.EncodePut(asked)
if _, err := puts.Put(b.ctx, pkey, raw); err != nil {
t.Fatal(err)
}
for deadline := time.Now().Add(15 * time.Second); ; time.Sleep(200 * time.Millisecond) {
// A change of a bucket it watches starts a pass, which opens the puts' bucket.
if _, err := b.buckets[delivery.PresenceBucket].Put(b.ctx, "n1", []byte(time.Now().String())); err != nil {
t.Fatal(err)
}
if e, err := puts.Get(b.ctx, pkey); err == nil {
if got, err := delivery.DecodePut(e.Value()); err == nil && got.Step == delivery.PutRefused && strings.Contains(got.Words, "not assigned") {
break
}
}
if time.Now().After(deadline) {
t.Fatal("the put was not taken further once its bucket was made")
}
}
}
// putJSON writes a value into one of the bed's buckets.
@@ -330,12 +281,9 @@ func (b *bed) get(t *testing.T, bucket, key string) []byte {
return nil
}
// 1297/B4 on the bus, 1906/D5: the controller sends ace, g14 and shanks builds while the canary ace does not run F healthy yet.
// Started whole, the assigner wants F on the canary and writes no difference at all: there is no send to compare with (ADR 0539
// §8). 1945/AC19, AC21: shanks, which runs the older fingerprint (composed and granted) with no want, gets its first want at that
// fingerprint, never at F; g14, which runs a build of the old form, gets none, so web is on neither list there and the assigner
// raises the condition naming g14, web and the canary rule; once ace runs F healthy, g14's first want names F and the condition is
// cleared.
// 1297/B4 on the bus, 1906/D5: the controller sends ace and g14 builds while the canary ace does not run F healthy yet.
// Started whole, the assigner wants F on the canary, holds g14 back with no want, and writes no difference at all: there
// is no send to compare with (ADR 0539 §8).
func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
b := startBed(t, "")
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
@@ -348,21 +296,19 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
}
bkey, _ := delivery.BuilderBuildKey("web", fNew)
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
for _, fp := range []string{fNew, fOld} {
gkey, _ := delivery.ControllerGrantKey("web", fp)
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant)
}
runs := map[string]string{"ace": fOld, "g14": "build-9", "shanks": fOld}
for _, node := range []string{"ace", "g14", "shanks"} {
b.compose(t, node, "web", fNew, 'c', delivery.JudgedNeither)
gkey, _ := delivery.ControllerGrantKey("web", fNew)
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant)
b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedNeither)
b.compose(t, "g14", "web", fNew, 'c', delivery.JudgedNeither)
for _, node := range []string{"ace", "g14"} {
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
b.putJSON(t, delivery.PresenceBucket, node, on)
have, _ := delivery.EncodeHave(delivery.Have{FP: runs[node], Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
have, _ := delivery.EncodeHave(delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, node+".web", have)
b.putJSON(t, provider.AssignmentsBucket, node+".web", []byte(`{"node":"`+node+`","module":"web","build":"b-`+node+`"}`))
}
b.compose(t, "shanks", "web", fOld, 'a', delivery.JudgedNeither)
b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-8"}`))
b.putJSON(t, provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-9"}`))
b.start(t)
wkey, _ := delivery.WantKey("ace", "web")
@@ -370,41 +316,12 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
if err != nil || want.FP != fNew {
t.Fatalf("the canary ace's want = %+v, %v", want, err)
}
if w := b.wantNow(t, "shanks", "web", "shanks' first want at what it runs", func(w delivery.Want) bool { return w.FP != "" }); w.FP != fOld || w.Composed != "sha256:"+strings.Repeat("a", 64) || w.Newest != fNew {
t.Errorf("shanks' want = %+v, want the fingerprint it runs naming its composition, held back from F", w)
}
var neither asks.Condition
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && neither.Key == ""; time.Sleep(100 * time.Millisecond) {
for k, c := range b.conditionsSaid(t) {
if strings.Contains(k, ".neither.") && c.State == asks.StateRaised {
neither = c
}
}
}
if neither.Subject.Machine != "g14" || neither.Subject.ID != "web" || neither.Severity != asks.SeverityUrgent || !strings.Contains(neither.Explanation, "canary") {
t.Errorf("the condition of web on neither list on g14 = %+v", neither)
}
time.Sleep(500 * time.Millisecond)
if keys, err := b.buckets[delivery.AssignerDifferencesBucket].Keys(b.ctx); err == nil && len(keys) != 0 {
t.Errorf("differences were written: %v", keys)
}
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, "want.g14.web"); err == nil {
t.Errorf("g14, which runs an old-form build and is held back by the canary, was given a want: %s", e.Value())
}
// 1945/AC21: once the canary ace runs F healthy, g14's first want names F by the rules as usual, and web leaves neither list
// there by itself: the condition is cleared.
healthyF, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, "ace.web", healthyF)
if w := b.wantNow(t, "g14", "web", "g14's first want at F once the canary runs it healthy", func(w delivery.Want) bool { return w.FP != "" }); w.FP != fNew || w.Composed != "sha256:"+strings.Repeat("c", 64) || w.Newest != "" {
t.Errorf("g14's want = %+v, want F with its composition", w)
}
cleared := false
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && !cleared; time.Sleep(100 * time.Millisecond) {
c, ok := b.conditionsSaid(t)[neither.Key]
cleared = ok && c.State == asks.StateCleared
}
if !cleared {
t.Errorf("the condition of web on neither list on g14 was not cleared once g14 got its want")
t.Errorf("g14, held back by the canary with no want before, was given one: %s", e.Value())
}
}
@@ -424,14 +341,12 @@ func TestAHeldFingerprintIsAskedAbout(t *testing.T) {
b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-1"}`))
b.start(t)
hkey, _ := delivery.HeldKey("web", fNew)
// 1906/H3: the held record names the machine and keeps the health, when and the node's words.
if h, err := delivery.DecodeHeld(b.get(t, delivery.AssignerWantsBucket, hkey)); err != nil || h.Node != "ace" ||
!strings.HasPrefix(h.Words, "health failed, phase running, seen at ") || !strings.HasSuffix(h.Words, ": connection refused") {
if h, err := delivery.DecodeHeld(b.get(t, delivery.AssignerWantsBucket, hkey)); err != nil || h.Node != "ace" {
t.Fatalf("held = %+v, %v", h, err)
}
akey, _ := delivery.HoldAskKey("web", fNew)
a, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
if err != nil || a.Asking <= 0 || a.Ask.ID != delivery.AskingID(akey, a.Asking) || len(a.Ask.Options) != 2 {
if err != nil || a.Ask.ID != delivery.AskingID(akey, 0) || len(a.Ask.Options) != 2 {
t.Fatalf("the ask = %+v, %v", a, err)
}
}
@@ -498,7 +413,7 @@ func TestAFailedDataModuleIsAskedToRestore(t *testing.T) {
key := ""
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && key == ""; time.Sleep(100 * time.Millisecond) {
for k, c := range b.conditionsSaid(t) {
if c.State == asks.StateRaised && c.Subject.ID == "mail" && strings.Contains(k, ".held.") {
if c.State == asks.StateRaised && c.Subject.ID == "mail" {
if c.Subject.Machine != "ace" || !strings.Contains(c.Explanation, "connection refused") {
t.Fatalf("the condition = %+v, want it to name ace and the check's words", c)
}
@@ -665,15 +580,8 @@ func (b *bed) wantNow(t *testing.T, node, module, what string, ok func(delivery.
return last
}
// answer says the teller wrote the operator's answer to the ask that stands at key (its warrant names that asking's id).
func (b *bed) answer(t *testing.T, key, option string) {
t.Helper()
a, err := delivery.DecodeAssignerAsk(key, b.get(t, delivery.AssignerAsksBucket, key))
if err != nil {
t.Fatal(err)
}
b.answerTo(t, key, option, a.Ask.ID)
}
// answer says the teller wrote the operator's answer to the ask at key.
func (b *bed) answer(t *testing.T, key, option string) { b.answerTo(t, key, option, "r1") }
// answerTo is answer to the ask with this id (the router's warrant names it).
func (b *bed) answerTo(t *testing.T, key, option, askID string) {
@@ -717,8 +625,8 @@ func TestAReleaseAnswerWritesTheWantAgain(t *testing.T) {
}
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
cleared := false
for k, c := range b.conditionsSaid(t) {
cleared = cleared || (c.Subject.ID == "web" && c.State == asks.StateCleared && strings.Contains(k, ".held."))
for _, c := range b.conditionsSaid(t) {
cleared = cleared || (c.Subject.ID == "web" && c.State == asks.StateCleared)
}
if cleared {
return
@@ -865,81 +773,3 @@ func TestAUserListRevisionOfTheBusIsWrittenAtOnce(t *testing.T) {
t.Errorf("the user-list revision was asked")
}
}
// 1906/H2, whole on the bus: given a fingerprint held after one failed check, when every node that runs the module then runs
// it healthy, the assigner lifts the hold by itself: the held record and its ask are taken away, the held condition is
// cleared saying why, and the want stays at the fingerprint.
func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
b := startBed(t, "")
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
b.start(t)
hkey, _ := delivery.HeldKey("web", fNew)
akey, _ := delivery.HoldAskKey("web", fNew)
b.get(t, delivery.AssignerWantsBucket, hkey)
b.get(t, delivery.AssignerAsksBucket, akey)
healthy, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy)
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
if b.hasKey(delivery.AssignerWantsBucket, hkey) || b.hasKey(delivery.AssignerAsksBucket, akey) {
continue
}
for k, c := range b.conditionsSaid(t) {
if c.Subject.ID == "web" && c.State == asks.StateCleared && strings.Contains(k, ".held.") && strings.Contains(c.Resolved, "lifted its hold by itself") {
if w := b.wantNow(t, "ace", "web", "the want at the lifted fingerprint", func(w delivery.Want) bool { return w.FP == fNew }); w.Rule != 0 {
t.Errorf("the want = %+v", w)
}
return
}
}
}
t.Fatalf("the hold was not lifted: held %v, ask %v", b.hasKey(delivery.AssignerWantsBucket, hkey), b.hasKey(delivery.AssignerAsksBucket, akey))
}
// 1906/H2, whole on the bus: a hold lifted by itself and written again after the assigner was started again is asked under a
// new id, so the teller raises it; the answer to the first hold does not carry over (1906/H4).
func TestAHoldWrittenAgainAfterALiftAndARestartIsAskedAgain(t *testing.T) {
b := startBed(t, "")
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
healthy, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
_, stop := b.startStoppable(t)
hkey, _ := delivery.HeldKey("web", fNew)
akey, _ := delivery.HoldAskKey("web", fNew)
first, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
if err != nil {
t.Fatal(err)
}
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy)
for deadline := time.Now().Add(15 * time.Second); b.hasKey(delivery.AssignerWantsBucket, hkey); time.Sleep(100 * time.Millisecond) {
if time.Now().After(deadline) {
t.Fatal("the hold was not lifted")
}
}
stop()
time.Sleep(1100 * time.Millisecond) // the next hold is written in another second
b.answerTo(t, akey, "keep", first.Ask.ID) // an answer to the first hold, which the later hold must not take
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
b.start(t)
again, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
if err != nil {
t.Fatal(err)
}
if again.Ask.ID == first.Ask.ID {
t.Fatalf("the hold written again is asked under the first id %s", first.Ask.ID)
}
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy)
for deadline := time.Now().Add(15 * time.Second); b.hasKey(delivery.AssignerWantsBucket, hkey); time.Sleep(100 * time.Millisecond) {
if time.Now().After(deadline) {
t.Fatal("the stale \"keep\" of the first hold kept the second one held")
}
}
}
@@ -16,7 +16,6 @@ import (
"os/signal"
"strings"
"syscall"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
@@ -30,15 +29,6 @@ import (
)
func main() {
if os.Getenv("MESH_BROKER_FILE") == "" {
// Started by the tool runner, with no credential of its own: the put's tools (ADR 0546), served through the runtime,
// since a process never serves (ADR 0436). The process the manifest runs names its credential.
if err := serveTools(); err != nil {
fmt.Fprintln(os.Stderr, "mesh-delivery-assigner:", err)
os.Exit(1)
}
return
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
if err := run(ctx, os.Getenv, os.Stderr); err != nil {
@@ -69,37 +59,29 @@ var readBuckets = []string{
const goBuildModule = "mesh-build-go"
// ownBuckets are the assigner's own states, as its manifest declares them.
var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerCompositionsBucket, delivery.AssignerDifferencesBucket,
delivery.AssignerPutsBucket, delivery.AssignerWantsBucket}
var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerCompositionsBucket, delivery.AssignerDifferencesBucket, delivery.AssignerWantsBucket}
// keptStates is what the assigner keeps and who may read it (ADR 0495 §2-§3): its wants by the watchman and the view
// (and the node agent from release 6), its differences by the view, its compositions by the checker, its asks by the view
// and the teller, its puts by whoever follows one (ADR 0546).
// and the teller.
var keptStates = []announce.State{
{Name: "asks", Readable: true},
{Name: "compositions", Readable: true},
{Name: "differences", Readable: true},
{Name: "puts", Readable: true},
{Name: "wants", Readable: true},
}
// buckets is every bucket the assigner opened, by name.
type buckets map[string]jetstream.KeyValue
// open opens every bucket the assigner reads and writes; a bucket it cannot open stops the start, named with its grant (issue 1945),
// since a pass without it would write wants from half the mesh. The puts' bucket is the exception: whatever keeps it from opening (it
// does not exist yet, the credential is not granted it yet, the bus does not answer in time), the assigner writes wants
// without it and takes no put further until it starts again (ADR 0546).
func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any)) (buckets, error) {
// open opens every bucket the assigner reads and writes; a bucket it cannot open stops the start, named, since a pass
// without it would write wants from half the mesh.
func open(ctx context.Context, js jetstream.JetStream) (buckets, error) {
out := buckets{}
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
kv, err := js.KeyValue(ctx, name)
if name == delivery.AssignerPutsBucket && err != nil {
logf("the bucket %s cannot be opened (%v): no put is taken further until the assigner starts again", name, err)
continue
}
if err != nil {
return nil, process.EssentialErr("assigner", name, "write a want from the whole mesh", err)
return nil, fmt.Errorf("the bucket %s cannot be opened: %w", name, err)
}
out[name] = kv
}
@@ -136,7 +118,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil {
return err
}
kvs, err := open(ctx, js, logger.Printf)
kvs, err := open(ctx, js)
if err != nil {
return err
}
@@ -151,14 +133,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
logger.Printf("what the assigner keeps could not be said on the bus, so its readers are not granted until it is: %v", err)
}
logger.Printf("started on the account %s, reading %d buckets", account.User, len(kvs))
return follow(ctx, js, kvs, sayVia(ctx, js), emitting{said}, logger)
return follow(ctx, js, kvs, sayVia(ctx, js), logger)
}
// emitting is the process connection as the puts' caller: its calls, and its events with no options.
type emitting struct{ *process.Conn }
func (e emitting) Emit(eventType string, body any) error { return e.Conn.Emit(eventType, body) }
// watched are the buckets whose every value a pass reads; the manifests are read by digest when a pass needs one, and
// watched only for keys, so a manifest that arrives wakes a composition that waits for it.
var watched = []string{
@@ -179,7 +156,7 @@ var watched = []string{
// follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after
// each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends.
func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(asks.Condition) error, bus caller, logger *log.Logger) error {
func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(asks.Condition) error, logger *log.Logger) error {
w := newWorld()
dirty := make(chan struct{}, 1)
wake := func() {
@@ -188,15 +165,13 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
default:
}
}
errs := make(chan error, len(watched)+2)
settled := make(chan struct{}, len(watched)+2)
errs := make(chan error, len(watched)+1)
settled := make(chan struct{}, len(watched)+1)
watch := func(name string, opts ...jetstream.WatchOpt) error {
kw, err := kvs[name].WatchAll(ctx, opts...)
if err != nil {
return fmt.Errorf("the bucket %s cannot be watched: %w", name, err)
}
// The puts' watch never stops the wants (ADR 0546): when it ends, the puts are no longer heard, said in the journal.
optional := name == delivery.AssignerPutsBucket
go func() {
defer kw.Stop()
for {
@@ -206,17 +181,11 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
return
case e, ok := <-kw.Updates():
if !ok {
if optional {
logger.Printf("the watch of %s ended: no put is taken further until the assigner starts again", name)
return
}
errs <- fmt.Errorf("the watch of %s ended", name)
return
}
if e == nil {
if !optional { // the puts never gate the first pass, so never the wants (ADR 0546)
settled <- struct{}{}
}
settled <- struct{}{}
continue
}
var value []byte
@@ -230,39 +199,6 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
}()
return nil
}
// The puts' watch is not waited for before the first pass: a watch of them that ends before it handed what stands only
// stops the puts, never the wants.
if kvs[delivery.AssignerPutsBucket] != nil {
if err := watch(delivery.AssignerPutsBucket); err != nil {
logger.Printf("%v: no put is taken further until it is opened again", err)
delete(kvs, delivery.AssignerPutsBucket)
}
}
// The puts' bucket is made by the controller from the manifest's states at its start (ADR 0201), and the assigner's
// credential reads it once the bus's user list carries the grant: neither waits for the assigner. So while it is not open,
// a pass started by a change opens it again, at most once a minute, and from then on the puts are watched and taken
// further, without a restart.
var triedPuts time.Time
reopenPuts := func(p *passer) {
if kvs[delivery.AssignerPutsBucket] != nil || time.Since(triedPuts) < PutsReopenEvery {
return
}
triedPuts = time.Now()
kv, err := js.KeyValue(ctx, delivery.AssignerPutsBucket)
if err != nil {
return
}
kvs[delivery.AssignerPutsBucket] = kv
if err := watch(delivery.AssignerPutsBucket); err != nil {
logger.Printf("%v: no put is taken further until it is opened again", err)
delete(kvs, delivery.AssignerPutsBucket)
return
}
if bus != nil {
p.acts = newPutActs(bus, wake, logger.Printf)
}
logger.Printf("the bucket %s is open: puts are taken further", delivery.AssignerPutsBucket)
}
for _, name := range watched {
var opts []jetstream.WatchOpt
if name == delivery.HaveBucket {
@@ -286,9 +222,6 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
}
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, said: map[string]bool{}, answeredAt: map[string]uint64{}}
p.asks = newComposeAsks(ctx, buildrun.Asker{Publisher: buildrun.JetStreamAsks{JS: js}, Waiter: buildrun.JetStreamAsks{JS: js}}, wake, logger.Printf)
if bus != nil && kvs[delivery.AssignerPutsBucket] != nil {
p.acts = newPutActs(bus, wake, logger.Printf)
}
wake()
for {
select {
@@ -298,15 +231,11 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
return err
case <-settled:
case <-dirty:
reopenPuts(p)
p.pass(ctx)
}
}
}
// PutsReopenEvery is how often, at most, a pass opens the puts' bucket again while it is not open.
var PutsReopenEvery = time.Minute
// sayVia publishes a condition on the operator-channel seat's one subject for conditions, under the assigner's own name
// (ADR 0437 clause 9).
func sayVia(ctx context.Context, js jetstream.JetStream) func(asks.Condition) error {
@@ -2,23 +2,15 @@ package main
import (
"context"
"errors"
"fmt"
"io"
"log"
"os"
"path/filepath"
"reflect"
"slices"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// 1297/B2a: the buckets the program opens are exactly the manifest's reads and states, as the broker names them
@@ -56,53 +48,6 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
if got, want := slices.Sorted(slices.Values(kept)), slices.Sorted(slices.Values(ownBuckets)); !reflect.DeepEqual(got, want) {
t.Errorf("the program announces %v, keeps %v", got, want)
}
// ADR 0546: whatever keeps the puts' bucket from opening (not there, not granted, no answer in time), the assigner opens
// the rest and starts, writing wants; any other bucket it cannot open stops the start, named.
for _, c := range []struct {
missing string
err error
starts bool
}{
{delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true},
{delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true},
{delivery.AssignerPutsBucket, context.DeadlineExceeded, true},
{delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false},
} {
var logged string
kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err}, func(f string, a ...any) { logged = fmt.Sprintf(f, a...) })
switch {
case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, "no put is taken further")):
t.Errorf("%s refused with %v: the start answered %v, %d buckets, logged %q", c.missing, c.err, err, len(kvs), logged)
case !c.starts && (err == nil || !strings.Contains(err.Error(), "grant of "+c.missing)): // 1945: named with its grant
t.Errorf("%s refused with %v: the start answered %v", c.missing, c.err, err)
}
}
// And a watch of the puts that ends before it handed what stands (its subscription closed, its grant withdrawn) does not
// keep the first pass from being made: the wants are written all the same.
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
wrote := make(chan string, 8)
kvs := buckets{}
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
kvs[name] = &watchKV{memKV: &memKV{values: map[string][]byte{}}}
}
ckey, _ := delivery.ComposedNodeKey("ace")
cd, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("d", 64), Judgement: delivery.JudgedUnchanged})
if err != nil {
t.Fatal(err)
}
kvs[delivery.ControllerComposedBucket].(*watchKV).entries = []kvEntry{{key: ckey, value: cd}}
kvs[delivery.AssignerPutsBucket].(*watchKV).closed = true
kvs[delivery.AssignerWantsBucket].(*watchKV).wrote = wrote
go follow(ctx, nil, kvs, func(asks.Condition) error { return nil }, nil, log.New(io.Discard, "", 0))
select {
case key := <-wrote:
if key != "want.ace.node" {
t.Errorf("the first pass wrote %s", key)
}
case <-ctx.Done():
t.Error("a puts watch that ended before its first marker kept the first pass from being made: no want was written")
}
}
// 1297/B2a: without a credential the program does not start, and says why.
@@ -112,61 +57,3 @@ func TestNoCredential(t *testing.T) {
t.Errorf("run without a credential = %v", err)
}
}
// refusingJS opens every bucket but one, which it refuses with err.
type refusingJS struct {
jetstream.JetStream
missing string
err error
}
func (r refusingJS) KeyValue(_ context.Context, name string) (jetstream.KeyValue, error) {
if name == r.missing {
return nil, r.err
}
return &memKV{values: map[string][]byte{}}, nil
}
// watchKV is a memKV whose watch hands its entries and then the marker that all stands was handed, or ends at once when
// closed; a put on it is said on wrote.
type watchKV struct {
*memKV
entries []kvEntry
closed bool
wrote chan string
}
func (w *watchKV) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
ch := make(chan jetstream.KeyValueEntry, len(w.entries)+1)
if w.closed {
close(ch)
return keyWatcher{ch}, nil
}
for _, e := range w.entries {
ch <- e
}
ch <- nil
return keyWatcher{ch}, nil
}
func (w *watchKV) Put(ctx context.Context, key string, value []byte) (uint64, error) {
if w.wrote != nil {
w.wrote <- key
}
return 1, nil
}
type keyWatcher struct{ ch chan jetstream.KeyValueEntry }
func (k keyWatcher) Updates() <-chan jetstream.KeyValueEntry { return k.ch }
func (k keyWatcher) Stop() error { return nil }
type kvEntry struct {
jetstream.KeyValueEntry
key string
value []byte
}
func (e kvEntry) Key() string { return e.key }
func (e kvEntry) Value() []byte { return e.value }
func (e kvEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut }
@@ -3,8 +3,6 @@ package main
import (
"context"
"errors"
"fmt"
"slices"
"strings"
"time"
@@ -30,29 +28,11 @@ type passer struct {
// answeredAt is, by the id of a backup ask, how many values the have had when its answer was written into the want.
answeredAt map[string]uint64
asks *composeAsks
// now is the clock; nil is time.Now.
now func() time.Time
// neither is, by node and module, each module on neither list whose condition this process raised (issue 1945). Nil until the
// first pass after a start, which clears what a previous process may have raised (startClears).
neither map[string]neitherOne
// acts are the puts' acts (ADR 0546); nil: the puts are not taken further (no bus to call, or no puts bucket).
acts *putActs
// noWant is, by node and module, the reason last said why the node gets no want for the module (issue 1945): a reason is
// said in the journal when it appears or changes, and its clearing when the node has a want again; one line per change.
noWant map[string]string
}
func (p *passer) clock() time.Time {
if p.now != nil {
return p.now()
}
return time.Now()
}
func (p *passer) pass(ctx context.Context) {
p.compositions(ctx)
p.wants(ctx)
p.puts(ctx)
p.nodeWants(ctx)
p.backups(ctx)
}
@@ -98,8 +78,6 @@ func (p *passer) wants(ctx context.Context) {
}
}
wanted := p.w.wantsOf()
puts := p.putsByModule()
p.staleAsks(ctx, trunk)
names := map[string]bool{}
for name := range trunk {
names[name] = true
@@ -107,23 +85,11 @@ func (p *passer) wants(ctx context.Context) {
for name := range wanted {
names[name] = true
}
neither := map[string]neitherOne{}
// The first pass after a start remembers nothing a previous process said: a neither or held condition it raised would stay
// raised for ever once the module has its want or is no longer held. That pass clears, once, the neither condition of every
// assigned module not on neither list and the held condition of every module not held; a clear of a condition that is not open is
// quiet at the router.
first := p.neither == nil
stale := map[string]neitherOne{}
defer func() { p.sayNeither(neither, stale) }()
for name := range names {
fp := trunk[name].FP2
name := name
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name],
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name],
Composed: func(node, fp string) (delivery.ComposedDigest, bool) { return p.w.composed(node, name, fp) },
GrantedAt: func(fp string) bool {
key, err := delivery.ControllerGrantKey(name, fp)
return err == nil && p.w.has(delivery.ControllerGrantsBucket, key)
},
Restored: func(node string) bool {
key, err := delivery.RestoreAskKey(node, name, fp)
return err == nil && p.answeredWith(key, assign.OptionRestore)
@@ -137,13 +103,7 @@ func (p *passer) wants(ctx context.Context) {
grant, _ := delivery.ControllerGrantKey(name, fp)
hold, _ := delivery.HoldAskKey(name, fp)
m.Released = p.answeredWith(hold, "release")
m.Kept = p.answeredWith(hold, "keep")
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(delivery.ControllerGrantsBucket, grant)
if raw := p.w.value(delivery.ControllerGrantsBucket, grant); raw != nil {
if g, err := delivery.DecodeControllerGrant(raw); err == nil {
m.GrantedOn = func(node string) bool { return assign.GrantNamesNode(g, node, name) }
}
}
if m.Released {
p.clearHeld(name, fp)
}
@@ -153,14 +113,6 @@ func (p *passer) wants(ctx context.Context) {
last[node] = p.w.lastHealthy(node, name, fp)
}
out := assign.Wants(m, nodes, wanted[name], last)
gatherNeither(neither, name, out)
p.sayNoWant(name, m.Nodes, wanted[name], out)
if first {
startClears(stale, m, out)
if fp != "" && !m.Held && out.Held == nil && !out.Lifted {
p.clearHeld(name, fp)
}
}
if out.Held != nil {
key, err := delivery.HeldKey(name, fp)
if err == nil {
@@ -168,11 +120,8 @@ func (p *passer) wants(ctx context.Context) {
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeHeld(held) })
}
}
if out.Lifted {
p.lift(ctx, name, fp)
}
if out.Held != nil || (m.Held && !out.Lifted) {
p.holdAsk(ctx, name, fp, out.Held)
if out.Held != nil || m.Held {
p.holdAsk(ctx, name, fp)
p.heldCondition(ctx, name, fp, out.Held, len(out.Restore) > 0)
}
for _, node := range out.Restore {
@@ -193,110 +142,6 @@ func (p *passer) wants(ctx context.Context) {
}
}
// sayNoWant says in the journal why a node of module gets no want after this pass, from the rule the outcome names for it
// (issue 1945: a new assignment whose trunk build was not done was held back with nothing said), once per reason, and says
// when the node has a want again. A node with a live want, written before and kept or written in this pass, has one.
func (p *passer) sayNoWant(module string, nodes []string, before map[string]delivery.Want, out assign.Outcome) {
if p.noWant == nil {
p.noWant = map[string]string{}
}
assigned := map[string]bool{}
for _, node := range nodes {
assigned[node] = true
key := node + "." + module
w, written := out.Wants[node]
if !written {
w, written = before[node]
}
reason := ""
if h, held := out.Rules[node]; held && (!written || w.Removed) {
reason = fmt.Sprintf("rule %d: %s", h.Rule, h.Words)
}
last, said := p.noWant[key]
switch {
case reason != "" && reason != last:
p.logf("%s on %s gets no want: %s", module, node, reason)
p.noWant[key] = reason
case reason == "" && said && written && !w.Removed:
p.logf("%s on %s has a want", module, node)
delete(p.noWant, key)
}
}
for key := range p.noWant {
if node, m, _ := strings.Cut(key, "."); m == module && !assigned[node] {
delete(p.noWant, key) // no longer assigned there: its removed want says so
}
}
}
// neitherOne is a module assigned to a node that holds no want for it after a pass, and the rule holding its first want back.
type neitherOne struct {
node, module string
holding assign.Holding
}
// gatherNeither adds the nodes on which module is on neither list in its outcome, keyed by node and module.
func gatherNeither(into map[string]neitherOne, module string, out assign.Outcome) {
for node, h := range out.Neither {
into[node+"."+module] = neitherOne{node: node, module: module, holding: h}
}
}
// startClears adds, for the first pass after a start, every node of m that is not on neither list after the pass: a neither
// condition a previous process raised for it is cleared. That covers a node with a want and one with none that is not on neither
// list, such as a held module (issue 1945: the hold is said on its own, so a neither raised for it before the restart would stay up
// for ever) or a move waiting for its record ask. Clearing a condition that is not open is quiet at the router.
func startClears(into map[string]neitherOne, m assign.Module, out assign.Outcome) {
for _, node := range m.Nodes {
if _, on := out.Neither[node]; !on {
into[node+"."+m.Name] = neitherOne{node: node, module: m.Name}
}
}
}
// sayNeither raises, once, the condition of every module on neither list in this pass and writes it to the journal, and clears,
// once, the condition of each one said before that has a want again (issue 1945). A say that fails is made again at the next pass.
// stale is, on the first pass after a start, the modules whose condition a previous process may have raised and that have a want:
// each is cleared once.
func (p *passer) sayNeither(now, stale map[string]neitherOne) {
if p.neither == nil {
p.neither = map[string]neitherOne{}
for key, one := range stale {
if _, on := now[key]; !on {
p.neither[key] = one
}
}
}
for key, one := range now {
if _, said := p.neither[key]; said {
continue
}
c := assign.NeitherCondition(one.node, one.module, one.holding, true)
if p.say != nil {
if err := p.say(c); err != nil {
p.logf("the condition of %s on neither list on %s could not be said, and is said at the next pass: %v", one.module, one.node, err)
continue
}
}
p.logf("%s is assigned to %s and on neither list: no want, and not in the node's own composition; rule %d holds its first want back: %s",
one.module, one.node, one.holding.Rule, one.holding.Words)
p.neither[key] = one
}
for key, one := range p.neither {
if _, still := now[key]; still {
continue
}
if p.say != nil {
if err := p.say(assign.NeitherCondition(one.node, one.module, assign.Holding{}, false)); err != nil {
p.logf("the condition of %s on neither list on %s could not be cleared, and is cleared at the next pass: %v", one.module, one.node, err)
continue
}
}
p.logf("%s on %s has a want again", one.module, one.node)
delete(p.neither, key)
}
}
// compositions writes the composition check's verdict of every group the indexer holds a candidate for, under the
// checker's key of that check: waits, with why, while the indexer holds what it composes from only in part.
func (p *passer) compositions(ctx context.Context) {
@@ -365,154 +210,15 @@ func (p *passer) put(ctx context.Context, bucket, key string, encode func() ([]b
// holdAsk writes, once, the ask about a held fingerprint (ADR 0528 clause 6, ADR 0437 clause 10): the first ask that needs
// no proof of who answered, which the view may answer and the teller fills. In release 5 an answer changes nothing on a
// node (wants are only compared, ADR 0429 §7), and the options say so.
func (p *passer) holdAsk(ctx context.Context, module, fp string, fresh *delivery.Held) {
func (p *passer) holdAsk(ctx context.Context, module, fp string) {
key, err := delivery.HoldAskKey(module, fp)
if err != nil || p.w.has(delivery.AssignerAsksBucket, key) {
return
}
// The asking is the hold's time in seconds: a hold of the same fingerprint written again after a lift is another asking, under
// an id of its own, also after a restart, since the router drops a second ask under an id it keeps (issue 1906).
now := p.clock()
asking := int(now.Unix())
h := p.heldHold(module, fp, fresh)
a := assign.HoldAsk(key, module, fp, h.Node, h.Words, asking, now)
a := assign.HoldAsk(key, module, fp, time.Now())
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
}
// lift takes away a hold the assigner lifted by itself, since every node that runs the module runs the held fingerprint healthy
// (issue 1906): the held record and its ask (the teller then takes the ask back from the operator), and the held condition is
// cleared saying why, once. A hold of the same fingerprint written later is said and asked again.
func (p *passer) lift(ctx context.Context, module, fp string) {
held, err := delivery.HeldKey(module, fp)
if err != nil {
return
}
ask, err := delivery.HoldAskKey(module, fp)
if err != nil {
return
}
c := assign.LiftedCondition(module, fp)
if !p.said[c.Key+".lifted"] {
if p.say != nil {
if err := p.say(c); err != nil {
// The hold stays written, so the next pass lifts it again and says so then.
p.logf("the condition of the lifted hold of %s could not be cleared, and is cleared at the next pass: %v", module, err)
return
}
}
p.logf("%s", assign.LiftedWords(module, fp))
p.said[c.Key+".lifted"] = true
delete(p.said, c.Key+".raised") // a hold written again is said again
}
p.remove(ctx, delivery.AssignerWantsBucket, held)
p.remove(ctx, delivery.AssignerAsksBucket, ask)
// The restore asks of that fingerprint, on every node, go with the hold: every node runs it healthy, so no node waits to be
// set back from it (issue 1906, note 28).
for key := range p.w.read(delivery.AssignerAsksBucket) {
if k, err := delivery.ParseAnyAskKey(key); err == nil && k.Kind == delivery.AskKindRestore && k.Module == module && k.FP == fp {
p.remove(ctx, delivery.AssignerAsksBucket, key)
}
}
}
// staleAsk says whether the hold, restore or record ask at key (read as the SDK reads it) is about something the trunk has left
// behind: a module no longer on the trunk, a fingerprint that is no longer the module's trunk fingerprint, or a composition the
// controller no longer composes for the node at that fingerprint. A backup ask and an ask of another kind are never stale here.
// A module of the trunk whose fingerprint is not read yet, and a composition not read yet, are not stale: nothing known is
// nothing taken away.
func (p *passer) staleAsk(key string, trunk map[string]delivery.Module) bool {
k, err := delivery.ParseAnyAskKey(key)
if err != nil {
return false
}
switch k.Kind {
case delivery.AskKindHold, delivery.AskKindRestore:
m, on := trunk[k.Module]
return !on || (m.FP2 != "" && m.FP2 != k.FP)
case delivery.AskKindRecord:
m, on := trunk[k.Module]
if !on {
return true
}
// The assigner asks about a composition at more than the trunk fingerprint: a node held back by the rules is asked about the
// revised composition of the fingerprint it runs (ADR 0545), and a node's first want at the older fingerprint it runs. So a
// record ask is stale only when its digest is the composition of none of the fingerprints the node can still want (the
// trunk's, its want's, its have's), and only when every one of them is read.
for _, fp := range p.fingerprintsOf(k.Node, k.Module, m.FP2) {
d, ok := p.w.composed(k.Node, k.Module, fp)
if !ok || d.Digest == k.Composed {
return false
}
}
return len(p.fingerprintsOf(k.Node, k.Module, m.FP2)) > 0
}
return false
}
// fingerprintsOf are the fingerprints a node may still want of module: the trunk's, the one its want names and the one its have
// names, each once.
func (p *passer) fingerprintsOf(node, module, trunkFP string) []string {
var out []string
add := func(fp string) {
if fp != "" && !slices.Contains(out, fp) {
out = append(out, fp)
}
}
add(trunkFP)
if key, err := delivery.WantKey(node, module); err == nil {
if w, err := delivery.DecodeWant(p.w.value(delivery.AssignerWantsBucket, key)); err == nil {
add(w.FP)
}
}
if key, err := delivery.HaveKey(node, module); err == nil {
if h, err := delivery.DecodeHave(p.w.value(delivery.HaveBucket, key)); err == nil {
add(h.FP)
}
}
return out
}
// staleAsks takes away, at every pass, each hold, restore and record ask that staleAsk names (issue 1906, note 28): an ask
// outlived its fingerprint, and the teller takes its router ask back from the operator when the key goes. With the trunk not
// read at all (no module), nothing is taken away.
func (p *passer) staleAsks(ctx context.Context, trunk map[string]delivery.Module) {
if len(trunk) == 0 {
return
}
for key := range p.w.read(delivery.AssignerAsksBucket) {
if p.staleAsk(key, trunk) {
// The condition is cleared first and the ask goes only once that was said: the ask is what brings this key back at the
// next pass, so a clear that failed once is made again, with no state of its own (issue 1906, note 29).
k, kerr := delivery.ParseAnyAskKey(key)
held := kerr == nil && (k.Kind == delivery.AskKindHold || k.Kind == delivery.AskKindRestore)
if held && !p.clearHeld(k.Module, k.FP) {
continue
}
p.logf("the ask %s is about a fingerprint or composition the trunk has left behind and is taken away", key)
p.remove(ctx, delivery.AssignerAsksBucket, key)
// The held record stays: it is the memory that this fingerprint failed (ADR 0408 §4), and a revert returns the trunk to
// exactly it. Only the condition, which would else ask for an answer that can no longer be given, is cleared, and it is
// raised again if the trunk returns to the fingerprint.
if held {
delete(p.said, assign.HeldCondition(k.Module, k.FP, "", "", false, false).Key+".raised")
}
}
}
}
// remove deletes one key of the assigner's own bucket when it holds it; a delete that fails is said and made again at the next
// pass.
func (p *passer) remove(ctx context.Context, bucket, key string) {
if !p.w.has(bucket, key) {
return
}
if err := p.kvs[bucket].Delete(ctx, key); err != nil && !errors.Is(err, context.Canceled) {
p.logf("%s %s could not be taken away: %v", bucket, key, err)
return
}
p.w.put(bucket, key, nil)
}
// heldHold is the hold of fp as written: the node whose health check failed and its words.
func (p *passer) heldHold(module, fp string, fresh *delivery.Held) delivery.Held {
if fresh != nil {
@@ -539,8 +245,6 @@ func (p *passer) heldCondition(ctx context.Context, module, fp string, fresh *de
return
}
p.said[c.Key+".raised"] = true
delete(p.said, c.Key+".lifted") // a hold raised again and lifted again is said lifted again
delete(p.said, c.Key+".cleared") // and released again is cleared again
}
// restoreAsk writes, once, the ask about a module holding data whose held fingerprint failed its health check on node.
@@ -554,36 +258,24 @@ func (p *passer) restoreAsk(ctx context.Context, module, node, fp string) {
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
}
// answeredWith says whether the teller wrote an answer to the ask at key as it stands now and the option chosen is option. The
// answer is the operator's, given in the view or on the phone, and stays under its key (ADR 0539 §14, §16, §19); it counts only
// for the asking it answered (its warrant names that asking's id), so an answer to an ask taken away, or to an earlier hold of
// the same fingerprint, does not carry over to a later one (issue 1906).
// answeredWith says whether the teller wrote an answer to the ask at key and the option chosen is option. The answer is the
// operator's, given in the view or on the phone, and stays under its key (ADR 0539 §14, §16, §19).
func (p *passer) answeredWith(key, option string) bool {
a, ok := p.w.answer(key)
if !ok || a.Option != option {
return false
}
ask, err := delivery.DecodeAssignerAsk(key, p.w.value(delivery.AssignerAsksBucket, key))
if err != nil {
return false
}
id, _, _ := strings.Cut(a.Warrant, "@")
return id == ask.Ask.ID
return ok && a.Option == option
}
// clearHeld clears, once, the condition of a hold the operator released.
// It says whether nothing is left to say: the clear was said now or before, or there is no router to say it to.
func (p *passer) clearHeld(module, fp string) bool {
func (p *passer) clearHeld(module, fp string) {
c := assign.HeldCondition(module, fp, "", "", false, false)
if p.say == nil || p.said[c.Key+".cleared"] {
return true
return
}
if err := p.say(c); err != nil {
p.logf("the condition of the released %s could not be cleared, and is cleared at the next pass: %v", module, err)
return false
return
}
p.said[c.Key+".cleared"] = true
return true
}
// recordAsk writes, once, the ask about a move of a module of policy record that waits for the operator's yes.
@@ -1,715 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provider"
"mesh-delivery-assigner/internal/assign"
)
// memKV is the part of a bucket a pass writes: puts and deletes.
type memKV struct {
jetstream.KeyValue
values map[string][]byte
}
func (m *memKV) Put(_ context.Context, key string, value []byte) (uint64, error) {
m.values[key] = value
return uint64(len(m.values)), nil
}
func (m *memKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error {
delete(m.values, key)
return nil
}
// newPasser is a pass on a fresh world, as after a start, writing to the buckets given, at the clock given.
func newPasser(kvs buckets, at time.Time) *passer {
return &passer{kvs: kvs, w: newWorld(), logf: func(string, ...any) {}, said: map[string]bool{}, answeredAt: map[string]uint64{},
now: func() time.Time { return at }}
}
var fpTest = delivery.SourcePrefix2 + strings.Repeat("b", 64)
// 1906/H2: given a hold lifted by itself, when the assigner is started again and the same fingerprint is held again, then the
// hold ask is written under another asking, with an id of its own, so the teller raises it (the router drops a second ask under
// an id it keeps).
func TestAHoldAskedAgainAfterALiftAndARestartHasANewID(t *testing.T) {
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
kvs := buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}
key, _ := delivery.HoldAskKey("web", fpTest)
held := &delivery.Held{Node: "ace", Words: "health failed"}
first := newPasser(kvs, time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC))
first.holdAsk(context.Background(), "web", fpTest, held)
a1, err := delivery.DecodeAssignerAsk(key, asksKV.values[key])
if err != nil {
t.Fatal(err)
}
first.lift(context.Background(), "web", fpTest)
if _, there := asksKV.values[key]; there {
t.Fatal("the lift left the hold ask")
}
again := newPasser(kvs, time.Date(2026, 10, 25, 2, 0, 0, 0, time.UTC)) // started again: it remembers nothing
again.holdAsk(context.Background(), "web", fpTest, held)
a2, err := delivery.DecodeAssignerAsk(key, asksKV.values[key])
if err != nil {
t.Fatal(err)
}
if a2.Ask.ID == a1.Ask.ID {
t.Errorf("the ask written again after a lift and a restart has the id %s of the first", a1.Ask.ID)
}
}
// 1906/H4: an answer counts only for the asking it answered: an answer to an earlier hold of the same fingerprint, or to an ask
// taken away, is not taken for the hold that stands now.
func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
key, _ := delivery.HoldAskKey("web", fpTest)
p := newPasser(buckets{}, time.Now())
put := func(asking int) string {
a := assign.HoldAsk(key, "web", fpTest, "ace", "health failed", asking, time.Now())
v, err := delivery.EncodeAssignerAsk(key, a)
if err != nil {
t.Fatal(err)
}
p.w.put(delivery.AssignerAsksBucket, key, v)
return a.Ask.ID
}
first := put(100)
v, _ := delivery.EncodeAnswer(delivery.Answer{Option: "release", Warrant: first + "@2026-10-25T01:00:00Z", Words: "the operator"})
p.w.put(delivery.AnswersBucket, key, v)
if !p.answeredWith(key, "release") {
t.Fatal("the answer to the ask that stands is not taken")
}
if p.answeredWith(key, "keep") {
t.Error("an answer was taken for an option not chosen")
}
put(200) // the same fingerprint held again, after a lift
if p.answeredWith(key, "release") {
t.Error("the answer to an earlier hold carried over to a later one")
}
p.w.put(delivery.AssignerAsksBucket, key, nil) // the ask taken away
if p.answeredWith(key, "release") {
t.Error("an answer to an ask taken away was taken")
}
t.Run("a put of the bus acts on the operator's warrant alone", putOfTheBus)
}
// memEntry is a value a memKV holds, as a bucket's Get answers it.
type memEntry struct {
jetstream.KeyValueEntry
value []byte
revision uint64
}
func (e memEntry) Value() []byte { return e.value }
func (e memEntry) Revision() uint64 { return e.revision }
func (m *memKV) Get(_ context.Context, key string) (jetstream.KeyValueEntry, error) {
v, ok := m.values[key]
if !ok {
return nil, jetstream.ErrKeyNotFound
}
return memEntry{value: v, revision: uint64(len(v))}, nil
}
func (m *memKV) Update(_ context.Context, key string, value []byte, revision uint64) (uint64, error) {
if uint64(len(m.values[key])) != revision {
return 0, errors.New("wrong last sequence")
}
m.values[key] = value
return uint64(len(value)), nil
}
// memStore is the runtime's states for the put's tools, over the same memKVs a pass writes.
type memStore map[string]*memKV
func (s memStore) get(state, key string) ([]byte, uint64, bool, error) {
v, ok := s[state].values[key]
return v, uint64(len(v)), ok, nil
}
func (s memStore) put(state, key string, value []byte) error {
s[state].values[key] = value
return nil
}
func (s memStore) keys(state string) ([]string, error) {
var out []string
for k := range s[state].values {
out = append(out, k)
}
return out, nil
}
// fakeBus is the assigner's process account in a test: every call recorded, each answered at once.
type fakeBus struct {
calls chan string
events []string
}
func (b *fakeBus) Call(key string, body any) (json.RawMessage, error) {
b.calls <- fmt.Sprintf("%s %v", key, body)
return json.RawMessage(`{"ok":true,"output":""}`), nil
}
func (b *fakeBus) CallOn(node, key string, body any) (json.RawMessage, error) {
return b.Call(node+"/"+key, body)
}
func (b *fakeBus) Emit(eventType string, body any) error {
b.events = append(b.events, fmt.Sprintf("%s %+v", eventType, body))
return nil
}
// ADR 0546, the first put (2026-10-25, want.novox.nats): the tool writes the put; a pass waits for the composition and grant
// the controller's take-in writes; once they are there it asks the operator with a yes that approves; only on that warrant does it have
// the streams snapshotted, then writes the want held and records the warrant; the have running it healthy holds it; a release
// gives the want back to the rules. Each step is said as the event put.
func putOfTheBus(t *testing.T) {
ctx := context.Background()
now := time.Date(2026, 10, 25, 22, 0, 0, 0, time.UTC)
fp := delivery.SourcePrefix2 + "fd497ab8" + strings.Repeat("0", 56)
digest := "sha256:" + strings.Repeat("d", 64)
mem := func() *memKV { return &memKV{values: map[string][]byte{}} }
puts, asksKV, wantsKV := mem(), mem(), mem()
kvs := buckets{delivery.AssignerPutsBucket: puts, delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}
p := newPasser(kvs, now)
bus := &fakeBus{calls: make(chan string, 8)}
woke := make(chan struct{}, 8) // each act's goroutine wakes a pass once it is done
p.acts = newPutActs(bus, func() { woke <- struct{}{} }, func(string, ...any) {})
p.acts.snapshot = func(context.Context, string, string) (string, error) {
return "novox's restore point of nats (streams) taken 22:01", nil
}
// The world: nats assigned to novox, built at fp, the bus by its manifest, novox running it from before the switch.
world := func(bucket, key string, v any) {
var data []byte
switch x := v.(type) {
case []byte:
data = x
default:
data, _ = json.Marshal(x)
}
p.w.put(bucket, key, data)
}
world(provider.AssignmentsBucket, "novox.nats", map[string]string{"node": "novox", "module": "nats"})
mdigest := strings.Repeat("e", 64)
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: mdigest, Repository: "novox/mesh-catalog", Commit: strings.Repeat("c", 40)})
world(delivery.ModulesBucket, "nats", mod)
p.w.manifests[mdigest] = parsed{module: "nats", manifest: catalogue.Manifest{Module: "nats", Provides: []catalogue.Offer{{Name: "mesh-bus"}}}}
bkey, _ := delivery.BuilderBuildKey("nats", fp)
built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: mdigest})
world(delivery.BuilderBuildsBucket, bkey, built)
hkey, _ := delivery.HaveKey("novox", "nats")
before, _ := delivery.EncodeHave(delivery.Have{FP: delivery.SourcePrefix2 + strings.Repeat("a", 64), Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
world(delivery.HaveBucket, hkey, before)
builds := mem()
builds.values[bkey] = built
modules := mem()
modules.values["nats"] = mod
s := newPutTools(memStore{putsState: puts, delivery.BuilderModule + ".builds": builds, delivery.IndexerModule + ".modules": modules})
s.now = func() time.Time { return now }
was := answerWithin
answerWithin = 0
defer func() { answerWithin = was }()
if _, err := s.doPut(map[string]any{"node": "novox", "module": "nats", "version": "fd497ab8ffff", "why": "x"}, "shanks.node-tools"); err == nil {
t.Error("a prefix of a fingerprint the builder never built was taken")
}
if got, err := s.version("nats", "fd497ab80000"); err != nil || got != fp {
t.Errorf("the prefix of the fingerprint built reads %q, %v", got, err)
}
if got, err := s.version("nats", "newest"); err != nil || got != fp {
t.Errorf("newest reads %q, %v", got, err)
}
ans, err := s.doPut(map[string]any{"node": "novox", "module": "nats", "version": fp, "hold": "true", "why": "the bus's user list"}, "shanks.node-tools")
if err != nil {
t.Fatal(err)
}
if a := ans.(putAnswer); a.Step != delivery.PutAsked || !a.Hold || !strings.Contains(a.Follow, "mesh_wait") {
t.Fatalf("put answered %+v", a)
}
pkey, _ := delivery.PutKey("novox", "nats")
step := func() delivery.Put {
t.Helper()
world(delivery.AssignerPutsBucket, pkey, puts.values[pkey])
p.pass(ctx)
got, err := delivery.DecodePut(puts.values[pkey])
if err != nil {
t.Fatal(err)
}
return got
}
if got := step(); got.Step != delivery.PutCompose || !strings.Contains(got.Words, "no composition") {
t.Fatalf("the first pass left the put at %s: %s", got.Step, got.Words)
}
select {
case call := <-bus.calls:
t.Errorf("a put asked %q: it reads the compositions and asks nobody for one", call)
default:
}
ckey, _ := delivery.ComposedKey("novox", "nats", fp)
cd, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digest, Judgement: delivery.JudgedCannot, Words: "the have names no composition"})
world(delivery.ControllerComposedBucket, ckey, cd)
gkey, _ := delivery.ControllerGrantKey("nats", fp)
g, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
world(delivery.ControllerGrantsBucket, gkey, g)
got := step()
akey, _ := delivery.RecordAskKey("novox", "nats", digest)
if got.Step != delivery.PutPerson || got.Ask != akey {
t.Fatalf("composed, the put stands at %s (%s), ask %q", got.Step, got.Words, got.Ask)
}
a, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
if err != nil || a.Ask.Options[0].Level != asks.Approve {
t.Fatalf("the ask is %+v, %v; want a yes that approves", a, err)
}
if _, there := wantsKV.values["want.novox.nats"]; there {
t.Fatal("the bus's want was written before the operator's warrant")
}
world(delivery.AssignerAsksBucket, akey, asksKV.values[akey])
yes, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a.Ask.ID + "@2026-10-25T22:01:00Z", Words: "the operator, verified"})
world(delivery.AnswersBucket, akey, yes)
if got := step(); got.Step != delivery.PutSnapshot {
t.Fatalf("on the warrant the put stands at %s: %s", got.Step, got.Words)
}
<-woke
got = step()
w, err := delivery.DecodeWant(wantsKV.values["want.novox.nats"])
if got.Step != delivery.PutWanted || err != nil || w.FP != fp || w.Composed != digest || !w.Put || !w.Hold {
t.Fatalf("snapshotted, the put stands at %s (%s); the want %+v, %v", got.Step, got.Words, w, err)
}
if call := <-bus.calls; !strings.Contains(call, "warranted") || !strings.Contains(call, a.Ask.ID) {
t.Errorf("the warrant was recorded as %q", call)
}
world(delivery.AssignerWantsBucket, "want.novox.nats", wantsKV.values["want.novox.nats"])
running, _ := delivery.EncodeHave(delivery.Have{FP: fp, Composed: digest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
world(delivery.HaveBucket, hkey, running)
if got := step(); got.Step != delivery.PutHolding {
t.Fatalf("running healthy, the put stands at %s: %s", got.Step, got.Words)
}
if _, err := s.doRelease(map[string]any{"node": "novox", "module": "nats", "why": "done"}, "shanks.node-tools"); err != nil {
t.Fatal(err)
}
if got := step(); got.Step != delivery.PutReleased {
t.Fatalf("released, the put stands at %s", got.Step)
}
listed, _ := s.doPuts(map[string]any{"module": "nats"})
if l := listed.(map[string]any)["puts"].([]putAnswer); len(l) != 1 || l[0].Step != delivery.PutReleased || len(l[0].Steps) < 6 {
t.Errorf("puts lists %+v", l)
}
if len(bus.events) < 5 || !strings.Contains(strings.Join(bus.events, "\n"), "holding") {
t.Errorf("the steps said as events: %v", bus.events)
}
// The same bus put again at the same composition after the release is its own asking: its ask is written again under an
// id of its own, and the earlier yes, still at the answers' key, does not count for it (ADR 0546 clause 8).
world(delivery.HaveBucket, hkey, before)
s.now = func() time.Time { return now.Add(time.Minute) }
if _, err := s.doPut(map[string]any{"node": "novox", "module": "nats", "version": fp, "hold": "true", "why": "again"}, "shanks.node-tools"); err != nil {
t.Fatal(err)
}
again := step()
a2, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
if again.Step != delivery.PutPerson || err != nil || a2.Ask.ID == a.Ask.ID {
t.Fatalf("put again, it stands at %s (%s); its ask %q, %v; the first ask was %q", again.Step, again.Words, a2.Ask.ID, err, a.Ask.ID)
}
world(delivery.AssignerAsksBucket, akey, asksKV.values[akey])
yes2, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a2.Ask.ID + "@2026-10-25T22:03:00Z", Words: "the operator, verified"})
world(delivery.AnswersBucket, akey, yes2)
if got := step(); got.Step != delivery.PutSnapshot {
t.Fatalf("on the second warrant the put stands at %s: %s", got.Step, got.Words)
}
<-woke
// Released between the pass's read and its write: the pass writes no want for the put that no longer stands.
oldWant, _ := delivery.EncodeWant(delivery.Want{FP: delivery.SourcePrefix2 + strings.Repeat("a", 64)})
wantsKV.values["want.novox.nats"] = oldWant
world(delivery.AssignerWantsBucket, "want.novox.nats", oldWant)
if _, err := s.doRelease(map[string]any{"node": "novox", "module": "nats", "why": "changed my mind"}, "shanks.node-tools"); err != nil {
t.Fatal(err)
}
p.pass(ctx)
if string(wantsKV.values["want.novox.nats"]) != string(oldWant) {
t.Errorf("a want was written for a put released meanwhile: %s", wantsKV.values["want.novox.nats"])
}
}
// 1945/AC21: given a module on neither list on a node, when passes follow, then the assigner raises its condition once, says it
// again after a say that failed, and clears it once when the module has a want again; a module on neither list is also written to
// the journal, so it is loud with no operator channel.
func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
p := newPasser(buckets{}, time.Date(2026, 10, 25, 15, 0, 0, 0, time.UTC))
var said []asks.Condition
failing := true
p.say = func(c asks.Condition) error {
if failing {
failing = false
return errors.New("the channel is away")
}
said = append(said, c)
return nil
}
var logged []string
p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
one := map[string]neitherOne{"g14.screen-lock": {node: "g14", module: "screen-lock",
holding: assign.Holding{Rule: delivery.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}}}
p.sayNeither(one, nil) // the say fails
p.sayNeither(one, nil)
p.sayNeither(one, nil)
if len(said) != 1 || said[0].State != asks.StateRaised || said[0].Subject.Machine != "g14" || said[0].Subject.ID != "screen-lock" {
t.Fatalf("said = %+v, want one raised condition of screen-lock on g14", said)
}
if !strings.Contains(strings.Join(logged, "\n"), "screen-lock") {
t.Errorf("the journal does not name the module on neither list: %q", logged)
}
p.sayNeither(map[string]neitherOne{}, nil)
p.sayNeither(map[string]neitherOne{}, nil)
if len(said) != 2 || said[1].State != asks.StateCleared || said[1].Key != said[0].Key {
t.Errorf("said = %+v, want the condition cleared once under its key", said)
}
}
// 1945/AC21: the pass gathers the modules on neither list of every module's outcome, keyed by node and module.
func TestThePassGathersTheModulesOnNeitherList(t *testing.T) {
got := map[string]neitherOne{}
gatherNeither(got, "screen-lock", assign.Outcome{Neither: map[string]assign.Holding{"g14": {Rule: delivery.RuleGrant, Words: "w"}}})
gatherNeither(got, "rofi", assign.Outcome{})
if len(got) != 1 || got["g14.screen-lock"].node != "g14" || got["g14.screen-lock"].module != "screen-lock" || got["g14.screen-lock"].holding.Rule != delivery.RuleGrant {
t.Errorf("gathered = %+v", got)
}
// 1945 (a first want held back silently): the journal says why a node gets no want when the reason appears or changes, and
// when it has a want again: one line per change, not per pass.
p := newPasser(buckets{}, time.Date(2026, 10, 25, 21, 30, 0, 0, time.UTC))
var logged []string
p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
notDone := assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: delivery.RuleBuildDone, Words: "the build of rofi at 4e7b118e is not done"}}}
wanted := assign.Outcome{Wants: map[string]delivery.Want{"shanks": {FP: fpTest, Composed: "sha256:" + strings.Repeat("c", 64)}}}
for i, c := range []struct {
before map[string]delivery.Want
out assign.Outcome
line string // the line said in this pass, "" for none
}{
{nil, notDone, "rofi on shanks gets no want: rule 1: the build of rofi at 4e7b118e is not done"},
{nil, notDone, ""},
{map[string]delivery.Want{"shanks": {Removed: true}}, notDone, ""},
{nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: delivery.RuleGrant, Words: "no credential"}}}, "rofi on shanks gets no want: rule 6: no credential"},
{nil, wanted, "rofi on shanks has a want"},
{map[string]delivery.Want{"shanks": wanted.Wants["shanks"]}, assign.Outcome{}, ""},
} {
logged = nil
p.sayNoWant("rofi", []string{"shanks"}, c.before, c.out)
if got := strings.Join(logged, "\n"); got != c.line {
t.Errorf("pass %d said %q, want %q", i, got, c.line)
}
}
}
// A neither or held condition a previous process raised is cleared by the first pass after a start: a fresh pass remembers
// nothing it said, so it clears, once, the neither condition of every assigned module not on neither list and the held condition of
// every module that is not held. Clearing a condition that is not open is quiet at the router.
func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) {
wantsKV := &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerWantsBucket: wantsKV, delivery.AssignerAsksBucket: &memKV{values: map[string][]byte{}}}, time.Now())
var said []asks.Condition
p.say = func(c asks.Condition) error { said = append(said, c); return nil }
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
p.w.put(delivery.ModulesBucket, "web", mod)
p.w.put(provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-1"}`))
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
p.w.put(delivery.PresenceBucket, "g14", on)
have, _ := delivery.EncodeHave(delivery.Have{FP: fpTest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
p.w.put(delivery.HaveBucket, "g14.web", have)
key, _ := delivery.WantKey("g14", "web")
want, _ := delivery.EncodeWant(delivery.Want{FP: fpTest, Composed: "sha256:" + strings.Repeat("c", 64)})
p.w.put(delivery.AssignerWantsBucket, key, want)
p.wants(context.Background())
cleared := map[string]bool{}
for _, c := range said {
if c.State != asks.StateCleared {
t.Errorf("the first pass raised %+v", c)
}
cleared[c.Key] = true
}
neither := assign.NeitherCondition("g14", "web", assign.Holding{}, false).Key
held := assign.HeldCondition("web", fpTest, "", "", false, false).Key
if !cleared[neither] || !cleared[held] {
t.Errorf("cleared = %v, want %s and %s", cleared, neither, held)
}
n := len(said)
p.wants(context.Background())
if len(said) != n {
t.Errorf("a second pass said again: %+v", said[n:])
}
}
// 1945/AC24: given a module held at F under rule 2 on a node that runs it with no want, when the assigner passes, then it raises no
// neither condition for it, a neither condition this process raised for it before is cleared, and so is one a previous process
// raised (the first pass after a start), each once.
func TestAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
setup := func() (*passer, *[]asks.Condition) {
p := newPasser(buckets{delivery.AssignerWantsBucket: &memKV{values: map[string][]byte{}}, delivery.AssignerAsksBucket: &memKV{values: map[string][]byte{}}}, time.Now())
said := &[]asks.Condition{}
p.say = func(c asks.Condition) error { *said = append(*said, c); return nil }
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/resolved", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
p.w.put(delivery.ModulesBucket, "systemd-resolved", mod)
p.w.put(provider.AssignmentsBucket, "shanks.systemd-resolved", []byte(`{"node":"shanks","module":"systemd-resolved","build":"b-1"}`))
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
p.w.put(delivery.PresenceBucket, "shanks", on)
have, _ := delivery.EncodeHave(delivery.Have{FP: fpTest, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed})
p.w.put(delivery.HaveBucket, "shanks.systemd-resolved", have)
hkey, _ := delivery.HeldKey("systemd-resolved", fpTest)
held, _ := delivery.EncodeHeld(delivery.Held{Node: "shanks", Words: "health failed"})
p.w.put(delivery.AssignerWantsBucket, hkey, held)
return p, said
}
key := assign.NeitherCondition("shanks", "systemd-resolved", assign.Holding{}, false).Key
neithers := func(said []asks.Condition) (raised, cleared int) {
for _, c := range said {
if c.Key != key {
continue
}
if c.State == asks.StateRaised {
raised++
} else {
cleared++
}
}
return
}
// a standing neither this process raised
p, said := setup()
p.neither = map[string]neitherOne{"shanks.systemd-resolved": {node: "shanks", module: "systemd-resolved", holding: assign.Holding{Rule: delivery.RuleNotHeld}}}
p.wants(context.Background())
p.wants(context.Background())
if r, c := neithers(*said); r != 0 || c != 1 {
t.Errorf("standing in this process: %d raised, %d cleared, want 0 and 1: %+v", r, c, *said)
}
// one a previous process raised: the first pass after a start clears it
p, said = setup()
p.wants(context.Background())
p.wants(context.Background())
if r, c := neithers(*said); r != 0 || c != 1 {
t.Errorf("after a start: %d raised, %d cleared, want 0 and 1: %+v", r, c, *said)
}
}
// 1906 note 28: an ask outlives its fingerprint. Each pass takes away the hold, restore and record asks of a module whose
// fingerprint (or composition) is not the trunk's now, or whose module left the trunk; a trunk not read takes nothing away; a
// backup ask is left; and a lift takes the restore asks of its fingerprint on every node.
func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
fpOld, fpNew := delivery.SourcePrefix2+strings.Repeat("a", 64), delivery.SourcePrefix2+strings.Repeat("c", 64)
digOld, digNew := "sha256:"+strings.Repeat("d", 64), "sha256:"+strings.Repeat("e", 64)
trunkOf := func(fp string) map[string]delivery.Module {
return map[string]delivery.Module{"web": {FP2: fp, Manifest: "sha256:" + strings.Repeat("1", 64), Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath}}
}
type ask struct{ name, key string }
holdOld, _ := delivery.HoldAskKey("web", fpOld)
holdNew, _ := delivery.HoldAskKey("web", fpNew)
holdGone, _ := delivery.HoldAskKey("gone", fpOld)
restOld, _ := delivery.RestoreAskKey("novox", "web", fpOld)
restNew, _ := delivery.RestoreAskKey("novox", "web", fpNew)
recOld, _ := delivery.RecordAskKey("novox", "web", digOld)
recNew, _ := delivery.RecordAskKey("novox", "web", digNew)
backOld, _ := delivery.BackupAskKey("novox", "web", fpOld)
all := []string{holdOld, holdNew, holdGone, restOld, restNew, recOld, recNew, backOld}
for _, c := range []struct {
name string
trunk map[string]delivery.Module
gone []string // the keys taken away
noDigit bool // the controller's composition is not read
heldAt string // the node's want is at this fingerprint, whose composition is digOld
}{
{"the trunk moved on", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false, ""},
{"the trunk is at the old fingerprint", trunkOf(fpOld), []string{holdNew, holdGone, restNew, recOld}, false, ""},
{"the trunk's fingerprint is not read yet", trunkOf(""), []string{holdGone}, false, ""},
{"the composition is not read yet", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, true, ""},
{"a node held back at the old fingerprint is still asked about its composition", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, false, fpOld},
{"the trunk is not read at all", nil, nil, false, ""},
} {
t.Run(c.name, func(t *testing.T) {
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
var said []asks.Condition
p.say = func(c asks.Condition) error { said = append(said, c); return nil }
for _, key := range all {
k, _ := delivery.ParseAnyAskKey(key)
var a delivery.AssignerAsk
switch k.Kind {
case delivery.AskKindHold:
a = assign.HoldAsk(key, k.Module, k.FP, "novox", "w", 1, time.Now())
case delivery.AskKindRestore:
a = assign.RestoreAsk(key, k.Node, k.Module, k.FP, "w", time.Now())
case delivery.AskKindRecord:
a = assign.RecordAsk(key, k.Node, k.Module, fpNew, assign.RecordNeed{Node: k.Node, Digest: k.Composed}, time.Now())
default:
a = assign.BackupAsk(key, k.Node, k.Module, k.FP, digOld, "w", 2, time.Now())
}
v, err := delivery.EncodeAssignerAsk(key, a)
if err != nil {
t.Fatal(err)
}
asksKV.values[key] = v
p.w.put(delivery.AssignerAsksBucket, key, v)
}
if !c.noDigit {
ckey, _ := delivery.ComposedKey("novox", "web", fpNew)
v, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digNew, Judgement: delivery.JudgedUnchanged})
p.w.put(delivery.ControllerComposedBucket, ckey, v)
okey, _ := delivery.ComposedKey("novox", "web", fpOld)
old, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digNew, Judgement: delivery.JudgedUnchanged})
if c.heldAt != "" {
old, _ = delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digOld, Judgement: delivery.JudgedUnchanged})
}
p.w.put(delivery.ControllerComposedBucket, okey, old)
}
if c.heldAt != "" {
wkey, _ := delivery.WantKey("novox", "web")
w, _ := delivery.EncodeWant(delivery.Want{FP: c.heldAt, Composed: digOld})
p.w.put(delivery.AssignerWantsBucket, wkey, w)
}
heldOld, _ := delivery.HeldKey("web", fpOld)
wantsKV.values[heldOld] = []byte("x")
p.w.put(delivery.AssignerWantsBucket, heldOld, []byte("x"))
firstAsk, _ := delivery.DecodeAssignerAsk(holdOld, asksKV.values[holdOld])
p.staleAsks(context.Background(), c.trunk)
// the held record and condition of a fingerprint whose hold ask went are taken away with it
heldOld, _ = delivery.HeldKey("web", fpOld)
_, heldThere := wantsKV.values[heldOld]
cleared := slices.ContainsFunc(said, func(c asks.Condition) bool {
return c.Key == assign.HeldCondition("web", fpOld, "", "", false, false).Key && c.State == asks.StateCleared
})
if gone := slices.Contains(c.gone, holdOld); !heldThere || cleared != gone {
t.Errorf("hold ask of the old fingerprint gone = %v: its held record stays = %v (must), its condition cleared = %v", gone, heldThere, cleared)
}
for _, key := range all {
_, there := asksKV.values[key]
want := !slices.Contains(c.gone, key)
if there != want {
t.Errorf("%s: left in the bucket = %v, want %v", key, there, want)
}
}
if slices.Contains(c.gone, holdOld) {
// the trunk returns to the fingerprint (a revert): the held record still holds it, and it is asked and said again
p.now = func() time.Time { return time.Now().Add(time.Hour) }
p.holdAsk(context.Background(), "web", fpOld, nil)
p.heldCondition(context.Background(), "web", fpOld, nil, false)
again, err := delivery.DecodeAssignerAsk(holdOld, asksKV.values[holdOld])
raised := slices.ContainsFunc(said, func(c asks.Condition) bool { return c.State == asks.StateRaised })
if err != nil || again.Ask.ID == firstAsk.Ask.ID || !raised {
t.Errorf("the trunk returned to the held fingerprint: ask %q (first %q, %v), condition raised again = %v", again.Ask.ID, firstAsk.Ask.ID, err, raised)
}
}
})
}
t.Run("a clear that failed to be said is said at the next pass, and the ask goes after it", func(t *testing.T) {
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
var said []asks.Condition
fail := true
p.say = func(c asks.Condition) error {
if fail {
fail = false
return errors.New("router unreachable")
}
said = append(said, c)
return nil
}
a := assign.HoldAsk(holdOld, "web", fpOld, "novox", "w", 1, time.Now())
v, err := delivery.EncodeAssignerAsk(holdOld, a)
if err != nil {
t.Fatal(err)
}
asksKV.values[holdOld] = v
p.w.put(delivery.AssignerAsksBucket, holdOld, v)
p.staleAsks(context.Background(), trunkOf(fpNew))
if _, there := asksKV.values[holdOld]; !there || len(said) != 0 {
t.Fatalf("after a failed clear: ask left = %v, conditions said = %d; want the ask kept and nothing said", there, len(said))
}
p.staleAsks(context.Background(), trunkOf(fpNew))
if _, there := asksKV.values[holdOld]; there || len(said) != 1 || said[0].State != asks.StateCleared {
t.Errorf("at the next pass: ask left = %v, conditions said = %v; want the ask gone and the clear said", there, said)
}
})
t.Run("a lift takes the restore asks of its fingerprint on every node", func(t *testing.T) {
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
keep, _ := delivery.RestoreAskKey("ace", "web", fpNew)
keepOther, _ := delivery.RestoreAskKey("ace", "other", fpOld)
for _, key := range []string{restOld, mustKey(delivery.RestoreAskKey("ace", "web", fpOld)), keep, keepOther} {
asksKV.values[key] = []byte("x")
p.w.put(delivery.AssignerAsksBucket, key, []byte("x"))
}
p.lift(context.Background(), "web", fpOld)
for key, gone := range map[string]bool{restOld: true, mustKey(delivery.RestoreAskKey("ace", "web", fpOld)): true, keep: false, keepOther: false} {
if _, there := asksKV.values[key]; there == gone {
t.Errorf("%s: left in the bucket = %v after the lift", key, there)
}
}
})
}
func mustKey(key string, err error) string {
if err != nil {
panic(err)
}
return key
}
// 1906 note 28, review: a node held back at an earlier fingerprint is asked about the revised composition of that fingerprint
// (ADR 0545). That record ask is not about the trunk's fingerprint: it survives every pass, so the operator is asked, and on a
// yes the want names the revised digest.
func TestARecordAskOfAHeldBackNodeSurvivesPassesAndItsYesMovesTheWant(t *testing.T) {
fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64)
revised, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64)
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
p.logf = t.Logf
put := func(bucket, key string, v []byte) { p.w.put(bucket, key, v) }
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
put(delivery.ModulesBucket, "web", mod) // the trunk's build is not done: rule 1 holds g14 back at fpOld
put(provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-1"}`))
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
put(delivery.PresenceBucket, "g14", on)
have, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
put(delivery.HaveBucket, "g14.web", have)
wkey, _ := delivery.WantKey("g14", "web")
want, _ := delivery.EncodeWant(delivery.Want{FP: fpOld, Composed: was})
wantsKV.values[wkey] = want
put(delivery.AssignerWantsBucket, wkey, want)
ckey, _ := delivery.ComposedKey("g14", "web", fpOld)
cd, cerr := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: revised, Judgement: delivery.JudgedRecreates, Policy: delivery.PolicyRecord, PolicyFrom: "default", Against: was, Words: "recreates"})
if cerr != nil {
t.Fatal(cerr)
}
put(delivery.ControllerComposedBucket, ckey, cd)
tkey, _ := delivery.ComposedKey("g14", "web", fpTest) // the trunk's own composition, another digest than the revised one
tcd, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("f", 64), Judgement: delivery.JudgedUnchanged})
put(delivery.ControllerComposedBucket, tkey, tcd)
akey, _ := delivery.RecordAskKey("g14", "web", revised)
for i := 0; i < 2; i++ {
p.wants(context.Background())
if _, there := asksKV.values[akey]; !there {
t.Fatalf("pass %d: the record ask of the held-back node is not there", i+1)
}
}
a, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
if err != nil {
t.Fatal(err)
}
yes, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a.Ask.ID + "@2026-10-26T01:00:00Z", Words: "the operator"})
put(delivery.AnswersBucket, akey, yes)
p.wants(context.Background())
got, err := delivery.DecodeWant(wantsKV.values[wkey])
if err != nil || got.FP != fpOld || got.Composed != revised {
t.Errorf("after the yes the want is %+v (%v), want it at %s naming %s", got, err, fpOld, revised)
}
if _, there := asksKV.values[akey]; !there {
t.Error("the answered record ask was taken away")
}
}
@@ -1,449 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-assigner/internal/assign"
)
// The puts (novox/hq ADR 0546): the tool `assigner_put` writes the key `<node>.<module>` of the assigner's puts, and each pass
// takes every put that has not ended one step further (assign.StepPut), doing the acts the step asks for: the put's ask, the
// bus's snapshot, the warranted record. Each act's outcome is heard back as a change of the world (the composition, the grant,
// the answer) or, for the calls, as a wake once they answer; nothing polls but the snapshot's read-back of the backup holder,
// as the controller's bus step reads it (ADR 0235), bounded. Every step is said as the event mesh-delivery-assigner.put, so a
// caller waits on it.
// PutEvent is the event a put's every step is said as: mesh.mod.mesh-delivery-assigner.event.put.
const PutEvent = "put"
// caller is what the puts need of the bus as the assigner's own process account: a call of a seat's verb, on one node or on
// the seat held once, and an event said. process.Conn, or a test's.
type caller interface {
Call(key string, body any) (json.RawMessage, error)
CallOn(node, key string, body any) (json.RawMessage, error)
Emit(eventType string, body any) error
}
// putActs keeps, by put (its key and when it was asked), what the acts of a put came to while this process runs: a restart
// takes the snapshot again, without harm.
type putActs struct {
mu sync.Mutex
bus caller
wake func()
logf func(string, ...any)
snapping map[string]bool
snapped map[string]string
snapNo map[string]string
warranted map[string]bool
warrantNo map[string]string
said map[string]string
// snapshot takes the bus's snapshot on node; a variable so a test takes none.
snapshot func(ctx context.Context, node, module string) (string, error)
}
func newPutActs(bus caller, wake func(), logf func(string, ...any)) *putActs {
a := &putActs{bus: bus, wake: wake, logf: logf, snapping: map[string]bool{}, snapped: map[string]string{}, snapNo: map[string]string{}, warranted: map[string]bool{}, warrantNo: map[string]string{},
said: map[string]string{}}
a.snapshot = a.snapshotTheBus
return a
}
// identity names one put: its key and when it was asked, so a put asked again under the key is another.
func identity(key string, p delivery.Put) string {
return key + "@" + p.At.UTC().Format(time.RFC3339Nano)
}
// putsByModule is every put the assigner holds, by module then node.
func (p *passer) putsByModule() map[string]map[string]delivery.Put {
out := map[string]map[string]delivery.Put{}
for key, value := range p.w.read(delivery.AssignerPutsBucket) {
node, module, err := delivery.ParsePutKey(key)
if err != nil {
continue
}
put, err := delivery.DecodePut(value)
if err != nil {
p.logf("the put %s cannot be read, and is left: %v", key, err)
continue
}
if out[module] == nil {
out[module] = map[string]delivery.Put{}
}
out[module][node] = put
}
return out
}
// puts takes every put that has not ended one step further.
func (p *passer) puts(ctx context.Context) {
if p.acts == nil {
return
}
all := p.putsByModule()
if len(all) == 0 {
return
}
assigned, _ := p.w.assigned()
trunk := p.w.trunk()
get := kvGetter(p.kvs[delivery.ManifestsBucket])
wanted := p.w.wantsOf()
modules := make([]string, 0, len(all))
for m := range all {
modules = append(modules, m)
}
sort.Strings(modules)
for _, module := range modules {
var manifest catalogue.Manifest
if mod, ok := trunk[module]; ok && mod.Manifest != "" {
if got := p.w.manifest(ctx, get, mod.Manifest); got.err == "" {
manifest = got.manifest
}
}
for node, put := range all[module] {
if delivery.PutEnded(put.Step) {
continue
}
key, _ := delivery.PutKey(node, module)
id := identity(key, put)
world := p.putWorld(node, module, put, manifest, assigned[node])
world.Snapshot, world.Snapshotting, world.SnapshotFailed = p.acts.state(id)
next := assign.StepPut(world)
// A held bus whose user list the rules moved on (ADR 0539 §7): the put follows the composition its want names.
if w, ok := wanted[module][node]; ok && w.Put && w.FP == next.Put.FP && w.Composed != "" && w.Composed != next.Put.Composed &&
(next.Put.Step == delivery.PutWanted || next.Put.Step == delivery.PutHolding) {
if d, _ := p.composedFor(node, module, w.FP); d != nil && d.UserListOnly && d.Digest == w.Composed {
next.Put = next.Put.Said(world.Now, next.Put.Step, fmt.Sprintf("the bus's user list changed: the want names composition %s", shortHex(w.Composed)))
next.Put.Composed = w.Composed
}
}
// The bus's want is written only for the act the operator's yes bound (ADR 0546 clause 8).
if next.Put.Step == delivery.PutWanted && put.Step != delivery.PutWanted && world.Bus && next.Put.Ask != "" {
if err := p.performs(next.Put.Ask, assign.PutAct(node, module, next.Put.FP, next.Put.Composed)); err != nil {
next = assign.PutNext{Put: next.Put.Said(world.Now, delivery.PutRefused, "the bus's want is not written: "+err.Error())}
}
}
if failed := p.acts.warrantFailed(id); failed != "" && (next.Put.Step == delivery.PutWanted || next.Put.Step == delivery.PutHolding) &&
!strings.Contains(next.Put.Words, failed) {
next.Put = next.Put.Said(world.Now, next.Put.Step, next.Put.Words+"; "+failed)
}
p.actOn(ctx, node, module, id, world, next)
written := samePut(put, next.Put) || p.writePut(ctx, key, put, next.Put)
if !written {
continue // released or put again meanwhile: nothing is done for the put that no longer stands
}
if next.Put.Step == delivery.PutWanted && put.Step != delivery.PutWanted {
// The want is written in this pass too, before the next one would: Module.Puts writes the same.
if wkey, err := delivery.WantKey(node, module); err == nil {
want := assign.PutWant(next.Put)
if before, ok := wanted[module][node]; !ok || !delivery.SameWant(normalWant(before), normalWant(want)) {
p.put(ctx, delivery.AssignerWantsBucket, wkey, func() ([]byte, error) { return delivery.EncodeWant(want) })
}
}
if next.Warranted {
p.acts.warrant(id, delivery.AskingID(next.Put.Ask, assign.PutAsking(next.Put)))
}
}
}
}
}
func normalWant(w delivery.Want) delivery.Want {
w.V = 0
return w
}
// putWorld is what one step of a put reads from the world.
func (p *passer) putWorld(node, module string, put delivery.Put, manifest catalogue.Manifest, assignedHere []string) assign.PutWorld {
w := assign.PutWorld{Node: node, Module: module, Put: put, Now: p.clock(), Bus: catalogue.ProvidesBus(manifest), Newest: p.w.trunk()[module].FP2}
for _, m := range assignedHere {
if m == module {
w.Assigned = true
}
}
if bkey, err := delivery.BuilderBuildKey(module, put.FP); err == nil {
if raw := p.w.value(delivery.BuilderBuildsBucket, bkey); raw != nil {
if b, err := delivery.DecodeBuilderBuild(raw); err == nil {
w.Build, w.BuildWords = b.State, b.Error
} else {
w.Build = "unreadable: " + err.Error()
}
}
}
w.Composed, w.Granted = p.composedFor(node, module, put.FP)
if d := w.Composed; d != nil {
policy := d.Policy
if policy == "" {
policy, _, _ = catalogue.DerivedUpgrade(manifest)
}
w.Recorded = policy == catalogue.PolicyRecord
if akey, err := delivery.RecordAskKey(node, module, d.Digest); err == nil && put.Ask == akey {
// Only this put's own ask counts: an ask (and its answer) of an earlier put under the same key is another asking.
if a, err := delivery.DecodeAssignerAsk(akey, p.w.value(delivery.AssignerAsksBucket, akey)); err == nil && a.Asking == assign.PutAsking(put) {
switch {
case p.answeredWith(akey, assign.OptionYes):
w.Answer = assign.OptionYes
case p.answeredWith(akey, assign.OptionNo):
w.Answer = assign.OptionNo
default:
w.AskExpired = !a.Ask.Expires.IsZero() && w.Now.After(a.Ask.Expires)
}
}
}
}
if hkey, err := delivery.HaveKey(node, module); err == nil {
if raw := p.w.value(delivery.HaveBucket, hkey); raw != nil {
if h, err := delivery.DecodeHave(raw); err == nil {
w.Have = &h
}
}
}
return w
}
// composedFor is the composition of module for node at fp and whether the grant at fp names the node's accounts: the one
// place a put reads them (ADR 0546 clause 4). Today their writer is the controller (mesh-controller_composed and
// mesh-controller_grants); when composing and grants move out of the controller into delivery's own parts, this reads them
// from their new owner and nothing else changes.
func (p *passer) composedFor(node, module, fp string) (*delivery.ComposedDigest, bool) {
var out *delivery.ComposedDigest
if d, ok := p.w.composed(node, module, fp); ok {
out = &d
}
granted := false
if gkey, err := delivery.ControllerGrantKey(module, fp); err == nil {
if raw := p.w.value(delivery.ControllerGrantsBucket, gkey); raw != nil {
if g, err := delivery.DecodeControllerGrant(raw); err == nil {
granted = g.NamesNode(node, module)
}
}
}
return out, granted
}
// actOn does the acts a step asks for.
func (p *passer) actOn(ctx context.Context, node, module, id string, w assign.PutWorld, next assign.PutNext) {
if next.Put.Step == delivery.PutPerson && next.Put.Ask != "" && w.Composed != nil {
// The put's own ask, written again over one an earlier put left under the key (its asking differs), so the teller
// raises it anew under its own id.
akey := next.Put.Ask
current, err := delivery.DecodeAssignerAsk(akey, p.w.value(delivery.AssignerAsksBucket, akey))
if err != nil || current.Asking != assign.PutAsking(next.Put) {
a, err := assign.PutAsk(akey, node, module, next.Put, *w.Composed, w.Bus, w.Now)
if err != nil {
p.logf("the ask about putting %s on %s is not written: %v", module, node, err)
} else {
p.put(ctx, delivery.AssignerAsksBucket, akey, func() ([]byte, error) { return delivery.EncodeAssignerAsk(akey, a) })
}
}
}
if next.Snapshot {
p.acts.snap(ctx, id, node, module)
}
}
// performs checks that the act about to be performed is the one the yes of the ask at key bound (asks.Option.Performs).
func (p *passer) performs(key string, act asks.Act) error {
a, err := delivery.DecodeAssignerAsk(key, p.w.value(delivery.AssignerAsksBucket, key))
if err != nil {
return fmt.Errorf("the ask %s cannot be read: %w", key, err)
}
for _, o := range a.Ask.Options {
if o.ID == assign.OptionYes {
return o.Performs(act)
}
}
return fmt.Errorf("the ask %s offers no yes", key)
}
// samePut says two puts say the same: their step, words and what they found.
func samePut(a, b delivery.Put) bool {
return a.Step == b.Step && a.Words == b.Words && a.Composed == b.Composed && a.Ask == b.Ask && a.Snapshot == b.Snapshot
}
// writePut writes a put's next value over the one the pass read, by compare-and-set, unless the tool has put again or
// released it since; and says it as the event put. It says whether it wrote.
func (p *passer) writePut(ctx context.Context, key string, read, next delivery.Put) bool {
kv := p.kvs[delivery.AssignerPutsBucket]
if kv == nil {
return false
}
e, err := kv.Get(ctx, key)
if err != nil {
p.logf("the put %s cannot be read again, and is taken further at the next pass: %v", key, err)
return false
}
now, err := delivery.DecodePut(e.Value())
if err != nil || !now.At.Equal(read.At) || now.Step != read.Step {
return false // asked again or released meanwhile: the next pass reads that
}
data, err := delivery.EncodePut(next)
if err != nil {
p.logf("the put %s is not written: %v", key, err)
return false
}
if _, err := kv.Update(ctx, key, data, e.Revision()); err != nil {
if !errors.Is(err, context.Canceled) {
p.logf("the put %s could not be written, and is taken further at the next pass: %v", key, err)
}
return false
}
p.w.put(delivery.AssignerPutsBucket, key, data)
p.logf("put %s: %s: %s", key, next.Step, next.Words)
p.acts.say(key, next)
return true
}
// state is what this process knows of a put's acts.
func (a *putActs) state(id string) (snapshot string, snapping bool, snapNo string) {
a.mu.Lock()
defer a.mu.Unlock()
return a.snapped[id], a.snapping[id], a.snapNo[id]
}
// snap has the bus's snapshot taken once for a put, in the background.
func (a *putActs) snap(ctx context.Context, id, node, module string) {
a.mu.Lock()
if a.snapping[id] || a.snapped[id] != "" || a.snapNo[id] != "" {
a.mu.Unlock()
return
}
a.snapping[id] = true
a.mu.Unlock()
go func() {
defer a.wake()
where, err := a.snapshot(ctx, node, module)
a.mu.Lock()
defer a.mu.Unlock()
a.snapping[id] = false
if err != nil {
a.snapNo[id] = err.Error()
return
}
a.snapped[id] = where
}()
}
// SnapshotWithin is how long the bus node's backup holder is given to take the bus's snapshot (as the controller's bus step).
var SnapshotWithin = 15 * time.Minute
// snapshotLook is how often the backup holder's record is read back while the snapshot is taken.
var snapshotLook = 10 * time.Second
// backedUp is one module as the backup holder's `backed-up` answers it.
type backedUp struct {
Module string `json:"module"`
Data []struct {
Item string `json:"item"`
LastBackup *time.Time `json:"last_backup"`
Error string `json:"error"`
} `json:"data"`
}
// snapshotTheBus asks the bus node's backup holder to back the bus module up now, whose dump is the streams' snapshot
// (ADR 0235), and reads its record back until it names a backup newer than the ask: where the snapshot is.
func (a *putActs) snapshotTheBus(ctx context.Context, node, module string) (string, error) {
asked := time.Now()
if _, err := a.bus.CallOn(node, "seat:"+catalogue.BackupSeat+".now", map[string]any{"module": module}); err != nil {
return "", fmt.Errorf("%s's backup holder was not asked to take the bus's snapshot: %w", node, err)
}
deadline := asked.Add(SnapshotWithin)
for {
if raw, err := a.bus.CallOn(node, "seat:"+catalogue.BackupSeat+".backed-up", map[string]any{}); err == nil {
var modules []backedUp
if json.Unmarshal(raw, &modules) == nil {
for _, m := range modules {
for _, it := range m.Data {
if m.Module == module && it.LastBackup != nil && it.LastBackup.After(asked) && it.Error == "" {
return fmt.Sprintf("%s's restore point of %s (%s) taken %s", node, module, it.Item, it.LastBackup.UTC().Format(time.RFC3339)), nil
}
}
}
}
}
if time.Now().After(deadline) {
return "", fmt.Errorf("%s's backup holder did not say the bus's snapshot was taken within %s", node, SnapshotWithin)
}
select {
case <-ctx.Done():
return "", ctx.Err()
case <-time.After(snapshotLook):
}
}
}
// warrant records once, in the controller's hand-act log, that the bus's want was written on the operator's warrant: the
// router's record of the teller's ask says who chose and how (ADR 0274).
func (a *putActs) warrant(id, ask string) {
a.mu.Lock()
if a.warranted[id] {
a.mu.Unlock()
return
}
a.warranted[id] = true
a.mu.Unlock()
go func() {
if _, err := a.bus.Call("seat:"+catalogue.ControllerSeatName+".warranted", map[string]any{"asker": delivery.TellerModule, "ask": ask}); err != nil {
a.logf("the put's warrant %s is not recorded in the hand-act log: %v", ask, err)
a.mu.Lock()
a.warrantNo[id] = fmt.Sprintf("the warrant %s is not recorded in the controller's hand-act log: %v", ask, err)
a.mu.Unlock()
a.wake()
}
}()
}
// warrantFailed says why the put's warrant was not recorded, "" when it was or is not yet known.
func (a *putActs) warrantFailed(id string) string {
a.mu.Lock()
defer a.mu.Unlock()
return a.warrantNo[id]
}
// putSaid is the body of the event put.
type putSaid struct {
Node string `json:"node"`
Module string `json:"module"`
FP string `json:"fp"`
Hold bool `json:"hold"`
Step string `json:"step"`
Words string `json:"words"`
Composed string `json:"composed,omitempty"`
}
// say says a put's step as the event put, once per step and words.
func (a *putActs) say(key string, p delivery.Put) {
node, module, err := delivery.ParsePutKey(key)
if err != nil || a.bus == nil {
return
}
a.mu.Lock()
if a.said[key] == p.Step+p.Words {
a.mu.Unlock()
return
}
a.said[key] = p.Step + p.Words
a.mu.Unlock()
if err := a.bus.Emit(PutEvent, putSaid{Node: node, Module: module, FP: p.FP, Hold: p.Hold, Step: p.Step, Words: p.Words, Composed: p.Composed}); err != nil {
a.logf("the put %s's step %s is not said as an event: %v", key, p.Step, err)
}
}
// shortHex is a digest's first twelve hex digits.
func shortHex(digest string) string {
h := strings.TrimPrefix(digest, "sha256:")
if len(h) > 12 {
return h[:12]
}
return h
}
@@ -1,364 +0,0 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"os"
"regexp"
"slices"
"sort"
"strings"
"sync"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The assigner's tools (novox/hq ADR 0546), served through the tool runner from its bundle: a process never serves (ADR
// 0436, mesh-sdk process). They write and read the puts; the assigner's process takes each put further (puts.go) and says
// every step as the event put. They are the module's own tools, not a seat's verbs: a new seat needs a change of the
// controller, which takes none; they become the verbs of a seat once composing leaves the controller.
// putsState is the assigner's state of puts, as the runtime names a module's own state.
const putsState = "puts"
// answerWithin is how long assigner_put waits for the assigner's process to take the put up before it answers what stands.
var answerWithin = 20 * time.Second
// store is what the tools need of the states: the runtime's (stdio.State), or a test's.
type store interface {
get(state, key string) ([]byte, uint64, bool, error)
put(state, key string, value []byte) error
keys(state string) ([]string, error)
}
// putTools serves the tools over a store, and keeps the newest value of every put it heard, to wait on.
type putTools struct {
st store
now func() time.Time
mu sync.Mutex
// heard is every put the watch handed, by key; changed is closed and replaced at each.
heard map[string][]byte
changed chan struct{}
}
func newPutTools(st store) *putTools {
return &putTools{st: st, now: time.Now, heard: map[string][]byte{}, changed: make(chan struct{})}
}
// hear takes a put's change in from the watch.
func (s *putTools) hear(key string, value []byte) {
s.mu.Lock()
s.heard[key] = value
close(s.changed)
s.changed = make(chan struct{})
s.mu.Unlock()
}
// fullFP is a fingerprint of the new form, whole; prefix at least its first twelve hex digits.
var (
fullFP = regexp.MustCompile(`^src2:[0-9a-f]{64}$`)
prefix = regexp.MustCompile(`^(src2:)?[0-9a-f]{12,63}$`)
)
// version reads the version a put names: newest (the indexer's fingerprint at the trunk), a whole fingerprint, or the start
// of one the builder built, which must name one build alone.
func (s *putTools) version(module, version string) (string, error) {
v := strings.ToLower(strings.TrimSpace(version))
switch {
case v == "newest":
raw, _, found, err := s.st.get(delivery.IndexerModule+".modules", module)
if err != nil {
return "", fmt.Errorf("the indexer's modules cannot be read: %w", err)
}
if !found {
return "", fmt.Errorf("the indexer holds no module %s at the trunk", module)
}
m, err := delivery.DecodeModule(raw)
if err != nil {
return "", fmt.Errorf("the indexer's %s cannot be read: %w", module, err)
}
if m.FP2 == "" {
return "", fmt.Errorf("the indexer holds no fingerprint of the new form for %s: %s", module, m.None2)
}
return m.FP2, nil
case fullFP.MatchString(v):
return v, nil
case prefix.MatchString(v):
start := strings.TrimPrefix(v, delivery.SourcePrefix2)
keys, err := s.st.keys(delivery.BuilderModule + ".builds")
if err != nil {
return "", fmt.Errorf("the builder's builds cannot be listed: %w", err)
}
var found []string
for _, k := range keys {
m, fp, err := delivery.ParseBuilderBuildKey(k)
if err == nil && m == module && strings.HasPrefix(strings.TrimPrefix(fp, delivery.SourcePrefix2), start) {
found = append(found, fp)
}
}
switch len(found) {
case 0:
return "", fmt.Errorf("the builder built no %s whose fingerprint starts %s: give it whole", module, start)
case 1:
return found[0], nil
}
sort.Strings(found)
return "", fmt.Errorf("%s starts %d fingerprints of %s the builder built (%s): give more of it", start, len(found), module, strings.Join(found, ", "))
}
return "", fmt.Errorf("%q is no version: newest, a fingerprint (src2: and 64 hex digits) or at least its first 12 hex digits", version)
}
// putAnswer is what assigner_put and assigner_puts answer of one put.
type putAnswer struct {
Node string `json:"node"`
Module string `json:"module"`
delivery.Put
Follow string `json:"follow,omitempty"`
}
// doPut is the tool assigner_put.
func (s *putTools) doPut(args map[string]any, by string) (any, error) {
node, module, why := arg(args, "node"), arg(args, "module"), arg(args, "why")
key, err := delivery.PutKey(node, module)
if err != nil {
return nil, err
}
if why == "" {
return nil, errors.New("put says why: it is kept with the put and said in any ask")
}
hold := false
switch arg(args, "hold") {
case "true":
hold = true
case "", "false":
default:
return nil, fmt.Errorf("hold is true or false, not %q", arg(args, "hold"))
}
fp, err := s.version(module, arg(args, "version"))
if err != nil {
return nil, err
}
now := s.now().UTC()
words := fmt.Sprintf("asked by %s: %s", by, why)
if raw, _, found, err := s.st.get(putsState, key); err != nil {
return nil, fmt.Errorf("the put of %s on %s cannot be read: %w", module, node, err)
} else if found {
if before, err := delivery.DecodePut(raw); err == nil && !delivery.PutEnded(before.Step) {
if before.FP == fp && before.Hold == hold {
return putAnswer{Node: node, Module: module, Put: before, Follow: followPut(node, module)}, nil
}
words += fmt.Sprintf("; it replaces the put of %s at %s, which stood at %s", module, shortFP(before.FP), before.Step)
}
}
p := delivery.Put{FP: fp, Hold: hold, Why: why, By: by, At: now}.Said(now, delivery.PutAsked, words)
data, err := delivery.EncodePut(p)
if err != nil {
return nil, err
}
if err := s.st.put(putsState, key, data); err != nil {
return nil, fmt.Errorf("the put of %s on %s is not written: %w", module, node, err)
}
s.hear(key, data)
return putAnswer{Node: node, Module: module, Put: s.await(key, p), Follow: followPut(node, module)}, nil
}
// await is the put at key once the assigner's process took it up past its asking, or as it stands after answerWithin.
func (s *putTools) await(key string, p delivery.Put) delivery.Put {
deadline := time.After(answerWithin)
for {
s.mu.Lock()
raw, changed := s.heard[key], s.changed
s.mu.Unlock()
if now, err := delivery.DecodePut(raw); err == nil && now.At.Equal(p.At) {
p = now
if now.Step != delivery.PutAsked && now.Step != delivery.PutBuild && now.Step != delivery.PutCompose {
return p // it waits on a person or the node, or ended
}
}
select {
case <-changed:
case <-deadline:
return p
}
}
}
// followPut says how a caller follows a put after the answer.
func followPut(node, module string) string {
return fmt.Sprintf("the put goes on: mesh_wait on mesh-delivery-assigner.put with filter {\"node\": %q, \"module\": %q}, then "+
"mesh-delivery-assigner.assigner_puts for where it stands", node, module)
}
// doRelease is the tool assigner_release.
func (s *putTools) doRelease(args map[string]any, by string) (any, error) {
node, module, why := arg(args, "node"), arg(args, "module"), arg(args, "why")
key, err := delivery.PutKey(node, module)
if err != nil {
return nil, err
}
if why == "" {
return nil, errors.New("release says why: it is kept with the put")
}
raw, _, found, err := s.st.get(putsState, key)
if err != nil {
return nil, fmt.Errorf("the put of %s on %s cannot be read: %w", module, node, err)
}
if !found {
return nil, fmt.Errorf("%s holds no put of %s: nothing to release", node, module)
}
p, err := delivery.DecodePut(raw)
if err != nil {
return nil, fmt.Errorf("the put of %s on %s cannot be read: %w", module, node, err)
}
if delivery.PutEnded(p.Step) {
return nil, fmt.Errorf("the put of %s on %s ended already (%s: %s): nothing to release", module, node, p.Step, p.Words)
}
p = p.Said(s.now(), delivery.PutReleased, fmt.Sprintf("released by %s: %s; the assigner's rules write the want again", by, why))
data, err := delivery.EncodePut(p)
if err != nil {
return nil, err
}
if err := s.st.put(putsState, key, data); err != nil {
return nil, fmt.Errorf("the release of %s on %s is not written: %w", module, node, err)
}
s.hear(key, data)
return putAnswer{Node: node, Module: module, Put: p}, nil
}
// doPuts is the tool assigner_puts.
func (s *putTools) doPuts(args map[string]any) (any, error) {
keys, err := s.st.keys(putsState)
if err != nil {
return nil, fmt.Errorf("the puts cannot be listed: %w", err)
}
sort.Strings(keys)
out := []putAnswer{}
for _, key := range keys {
node, module, err := delivery.ParsePutKey(key)
if err != nil || (arg(args, "node") != "" && node != arg(args, "node")) || (arg(args, "module") != "" && module != arg(args, "module")) {
continue
}
raw, _, found, err := s.st.get(putsState, key)
if err != nil || !found {
continue
}
p, err := delivery.DecodePut(raw)
if err != nil {
continue
}
out = append(out, putAnswer{Node: node, Module: module, Put: p})
}
return map[string]any{"puts": out}, nil
}
func arg(args map[string]any, name string) string {
switch v := args[name].(type) {
case string:
return strings.TrimSpace(v)
case bool:
return fmt.Sprint(v)
}
return ""
}
// The tools' names, as the manifest lists them.
const (
toolPut = "assigner_put"
toolRelease = "assigner_release"
toolPuts = "assigner_puts"
)
// schema is a tool's arguments, each a string: required ones first named, and switches true or false.
func schema(properties map[string]string, required []string, switches ...string) map[string]any {
props := map[string]any{}
for name, description := range properties {
p := map[string]any{"type": "string", "description": description}
if slices.Contains(switches, name) {
p["enum"] = []string{"true", "false"}
}
props[name] = p
}
out := map[string]any{"type": "object", "properties": props}
if len(required) > 0 {
out["required"] = required
}
return out
}
// tools are the assigner's tools as the runtime serves them (ADR 0546 clause 1).
func (s *putTools) tools() []stdio.Tool {
return []stdio.Tool{
{Name: toolPut, Description: "Put one version of a module on one machine (novox/hq ADR 0546): the build at that fingerprint, " +
"the composition and grant the controller wrote for that machine, then the machine's want naming it, held there until " +
"released when hold is true. A module whose moves a person takes (the bus, a module of policy record) is asked of the " +
"operator first, and the bus has its streams snapshotted after the yes. Answers what was done or what it waits on; the " +
"put goes on after the answer, each step said as the event mesh-delivery-assigner.put, and assigner_puts says where it stands.",
Input: schema(map[string]string{
"node": "the machine, where the module is assigned",
"module": "the module",
"version": "newest, a fingerprint (src2: and 64 hex digits), or at least its first 12 hex digits",
"hold": "\"true\": the want stays at this version until released; \"false\" (the default): the rules move it again once it runs",
"why": "why, kept with the put and said in any ask",
}, []string{"node", "module", "version", "why"}, "hold"),
Run: func(a map[string]any) (any, error) { return s.doPut(a, stdio.CallerOf(a)) }},
{Name: toolRelease, Description: "End a put on one machine (novox/hq ADR 0546): its hold is taken away and the assigner's " +
"rules write the machine's want again from their next pass. Nothing changes on the machine until they move it.",
Input: schema(map[string]string{"node": "the machine", "module": "the module", "why": "why, kept with the put"},
[]string{"node", "module", "why"}),
Run: func(a map[string]any) (any, error) { return s.doRelease(a, stdio.CallerOf(a)) }},
{Name: toolPuts, Description: "Every put and where it stands (novox/hq ADR 0546): its version, hold, why, each step with " +
"its time and words. Narrowed by machine or module. Read-only.",
Input: schema(map[string]string{"node": "one machine", "module": "one module"}, nil),
Run: s.doPuts},
}
}
// runtimeStore is the runtime's states.
type runtimeStore struct{}
func (runtimeStore) get(state, key string) ([]byte, uint64, bool, error) {
e, err := stdio.State(state).Get(key)
if err != nil || e == nil {
return nil, 0, false, err
}
return e.Value, e.Revision, true, nil
}
func (runtimeStore) put(state, key string, value []byte) error {
_, err := stdio.State(state).Put(key, json.RawMessage(value))
return err
}
func (runtimeStore) keys(state string) ([]string, error) { return stdio.State(state).Keys() }
// serveTools serves the tools to the runtime until it stops, hearing every put to answer assigner_put with its first steps.
func serveTools() error {
s := newPutTools(runtimeStore{})
go func() {
<-stdio.Served()
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
err := stdio.State(putsState).Watch("", func(c stdio.StateChange) error {
s.hear(c.Key, c.Value)
return nil
})
if err == nil {
return
}
fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: the puts are not heard yet (%v); asked again in %s\n", err, wait)
time.Sleep(wait)
}
}()
return stdio.Serve("", s.tools())
}
// shortFP is a fingerprint's first twelve hex digits.
func shortFP(fp string) string {
part, err := delivery.FingerprintKeyPart(fp)
if err != nil || len(part) < 12 {
return fp
}
return part[:12]
}
@@ -326,13 +326,6 @@ func (w *world) nodes(manifests map[string]catalogue.Manifest, assigned map[stri
return n
}
// value is what a bucket holds at a key now, nil when nothing.
func (w *world) value(bucket, key string) []byte {
w.mu.Lock()
defer w.mu.Unlock()
return w.values[bucket][key]
}
// has says whether a bucket holds a key now.
func (w *world) has(bucket, key string) bool {
w.mu.Lock()
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
-8
View File
@@ -1,13 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025195051-20a6dc4920f8 h1:jA7V7hlfJAigVUvJDpKwfgd9ylQ9bqppg3rK9XgLWoA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025195051-20a6dc4920f8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8 h1:vhYk/iv43u2WGSTrFyQGDvGBxLaRITgMAiiI9DjuA10=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 h1:gE80bR0tPME7MTMBLcdP62v+F/9soVXmdcBYNMOye44=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -10,17 +10,14 @@ import (
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (delivery.HoldAskKey): keep it held or
// release it. Releasing writes the held fingerprint's want again by the rules (ADR 0539 §19). node and words are the held
// record's: the machine whose health check failed and the hold's words (issue 1906). asking numbers the writing: 0 the first, a
// larger number an ask written again after a hold lifted by itself, so the router takes it as new.
func HoldAsk(key, module, fp, node, words string, asking int, now time.Time) delivery.AssignerAsk {
return delivery.AssignerAsk{Asking: asking, Ask: asks.Ask{
ID: delivery.AskingID(key, asking),
// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (delivery.HoldAskKey), first asking: keep
// it held or release it. Releasing writes the held fingerprint's want again by the rules (ADR 0539 §19).
func HoldAsk(key, module, fp string, now time.Time) delivery.AssignerAsk {
return delivery.AssignerAsk{Ask: asks.Ask{
ID: delivery.AskingID(key, 0),
Headline: fmt.Sprintf("%s is held at %s", module, short(fp)),
Explanation: fmt.Sprintf("The health check of %s failed on %s at %s (%s), so the assigner holds that version and wants "+
"no node to move to it. It lifts the hold by itself once every node that runs %s runs it healthy; releasing it "+
"lets the wants move to it now.", module, node, short(fp), words, module),
Explanation: fmt.Sprintf("A node's health check of %s failed at %s, so the assigner holds that version and wants "+
"no node to move to it. Releasing it lets the wants move to it again.", module, short(fp)),
Options: []asks.Option{
{ID: "keep", Label: "Keep it held", Does: "the version stays held", Level: asks.Acknowledge},
{ID: "release", Label: "Release it", Does: "the version is released and the wants move to it by the rules again", Level: asks.Acknowledge},
@@ -81,35 +78,6 @@ func HeldCondition(module, fp, node, words string, raised, needsYou bool) asks.C
return c
}
// NeitherCondition is the condition of a module assigned to node that holds no want for it (issue 1945): the node's own composition
// gives up every module with a fingerprint of the new form, so nothing applies it there and its files are gone. It is raised (urgent)
// naming the module, the node and the rule holding its first want back with the rule's words, under the assigner's own name keyed by
// node and module, and cleared (raised false) when the node has a want for it again.
func NeitherCondition(node, module string, h Holding, raised bool) asks.Condition {
key := delivery.AssignerModule + ".neither." + node + "." + module
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
if !raised {
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared,
Resolved: fmt.Sprintf("%s on %s has a want again, so its own composition is applied there.", module, node)}
}
headline := fmt.Sprintf("%s on %s is on no list", module, node)
if n := len([]rune(headline)); n > asks.HeadlineLength {
headline = string([]rune(headline)[:asks.HeadlineLength])
}
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
Summary: headline, Headline: headline,
Explanation: fmt.Sprintf("%s %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
"hold it either, so nothing applies it there. The assigner's rule %d holds its first want back: %s. The assigner writes the want once the "+
"rule holds, and clears this.", asks.NothingToDo, module, node, h.Rule, h.Words)}
}
// LiftedCondition clears the condition of a held fingerprint the assigner lifted by itself, saying why (issue 1906).
func LiftedCondition(module, fp string) asks.Condition {
c := HeldCondition(module, fp, "", "", false, false)
c.Resolved = LiftedWords(module, fp) + "."
return c
}
// The options of the record ask (ADR 0539 §14) and of the backup ask (§16).
const (
OptionYes = "yes"
@@ -17,25 +17,13 @@ func TestTheHoldAskIsValid(t *testing.T) {
t.Fatal(err)
}
now := time.Now()
a := HoldAsk(key, "mail", fNew, "ace", "health failed, phase running: connection refused", 0, now)
a := HoldAsk(key, "mail", fNew, now)
if err := a.Ask.Check(now); err != nil {
t.Errorf("the router refuses the ask: %v", err)
}
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
t.Errorf("the ask cannot be written: %v", err)
}
// 1906/H3: the ask names the machine whose check failed and the hold's words.
if !strings.Contains(a.Ask.Explanation, "on ace") || !strings.Contains(a.Ask.Explanation, "connection refused") {
t.Errorf("the ask does not name the node and the hold's words: %q", a.Ask.Explanation)
}
// 1906/H2: an ask written again after a lift is another asking, under an id of its own.
again := HoldAsk(key, "mail", fNew, "ace", "x", 7, now)
if again.Asking != 7 || again.Ask.ID != delivery.AskingID(key, 7) || again.Ask.ID == a.Ask.ID {
t.Errorf("the asking after a lift = %d %s", again.Asking, again.Ask.ID)
}
if _, err := delivery.EncodeAssignerAsk(key, again); err != nil {
t.Errorf("the ask asked again cannot be written: %v", err)
}
if _, err := delivery.ViewAnswerKey(key); err != nil {
t.Errorf("the view may not answer it: %v", err)
}
@@ -86,17 +74,6 @@ func TestTheHeldConditionNamesModuleNodeAndWords(t *testing.T) {
}
}
// 1906/H2: a hold lifted by itself clears the held condition under its key, saying why.
func TestALiftedHoldClearsItsConditionSayingWhy(t *testing.T) {
c := LiftedCondition("mail", fNew)
if _, err := asks.EncodeCondition(c); err != nil {
t.Fatalf("the condition is refused: %v", err)
}
if c.State != asks.StateCleared || c.Key != HeldCondition("mail", fNew, "ace", "x", true, false).Key || !strings.Contains(c.Resolved, "lifted its hold by itself") {
t.Errorf("condition = %+v", c)
}
}
// 1906/D4: the record ask and the backup ask are valid asks of the router and of mesh-sdk's layout at their keys, answerable in
// the view, and name the move and the node.
func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
@@ -146,58 +123,4 @@ func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
if _, err := asks.EncodeCondition(c); err != nil {
t.Errorf("the backup condition is refused: %v", err)
}
// ADR 0546 clauses 7 and 8: a put's ask is valid at the record ask's key; for the bus its yes approves and binds the act,
// so the view cannot answer it, and it lasts no longer than an approving ask may.
put := delivery.Put{FP: fNew, Why: "the bus's user list", Hold: true}
d := delivery.ComposedDigest{Digest: dgst, Judgement: delivery.JudgedCannot, Words: "the have names no composition"}
for _, bus := range []bool{false, true} {
a, err := PutAsk(rkey, "ace", "mail", put, d, bus, now)
if err != nil {
t.Fatalf("bus %v: %v", bus, err)
}
if err := a.Ask.Check(now); err != nil {
t.Errorf("bus %v: the router refuses the put's ask: %v", bus, err)
}
if _, err := delivery.EncodeAssignerAsk(rkey, a); err != nil {
t.Errorf("bus %v: the put's ask cannot be written: %v", bus, err)
}
viewed := delivery.ViewAnswerFits(delivery.ViewAnswer{Asking: a.Ask.ID, Option: OptionYes}, a)
yes := a.Ask.Options[0]
switch {
case !strings.Contains(a.Ask.Explanation, "the bus's user list"):
t.Errorf("bus %v: the put's ask does not say why: %s", bus, a.Ask.Explanation)
case bus && (yes.Level != asks.Approve || yes.Binds == "" || viewed == nil || a.Ask.Expires.After(now.Add(asks.ApproveLasts))):
t.Errorf("bus: the yes is %+v, the view's answer %v, expiring %s", yes, viewed, a.Ask.Expires)
case !bus && (yes.Level != asks.Acknowledge || viewed != nil):
t.Errorf("a recorded module: the yes is %+v, the view's answer %v", yes, viewed)
}
later := put
later.At = now.Add(time.Minute)
b, _ := PutAsk(rkey, "ace", "mail", later, d, bus, now)
if b.Ask.ID == a.Ask.ID || b.Asking != PutAsking(later) {
t.Errorf("bus %v: a later put's ask is asking %d under id %q, the earlier's %q", bus, b.Asking, b.Ask.ID, a.Ask.ID)
}
if err := yes.Performs(PutAct("ace", "mail", fNew, dgst)); err != nil {
t.Errorf("bus %v: the yes does not perform the act it bound: %v", bus, err)
}
if err := yes.Performs(PutAct("ace", "mail", fNew, dig(fOld))); bus && err == nil {
t.Errorf("bus: the yes performs another composition's act")
}
}
}
// 1945/AC21: the condition of a module on neither list is urgent, names the module, the node and the rule holding its want back,
// under the assigner's own name keyed by node and module, and is cleared under the same key.
func TestTheNeitherConditionNamesModuleNodeAndRule(t *testing.T) {
c := NeitherCondition("g14", "i3status-rust", Holding{Rule: delivery.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}, true)
if _, err := asks.EncodeCondition(c); err != nil {
t.Fatalf("the condition is refused: %v", err)
}
if c.State != asks.StateRaised || c.Severity != asks.SeverityUrgent || c.Subject.ID != "i3status-rust" || c.Subject.Machine != "g14" ||
!strings.Contains(c.Explanation, "x11-display") || !strings.Contains(c.Explanation, "rule 5") || c.Key != "mesh-delivery-assigner.neither.g14.i3status-rust" {
t.Errorf("condition = %+v", c)
}
if got := NeitherCondition("g14", "i3status-rust", Holding{}, false); got.State != asks.StateCleared || got.Key != c.Key {
t.Errorf("cleared = %+v", got)
}
}
@@ -1,252 +0,0 @@
package assign
import (
"fmt"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// A put (novox/hq ADR 0546): one version of a module put on one node by the assigner's tool assigner_put. The tool writes the put
// key; each pass takes it one step further from what the world holds (StepPut), and the rules pass over the node's want for
// the module from its asking until it ends (Module.Puts).
// ComposeWithin is how long a put waits for the composition and grant it needs (a take-in of a build just done writes them)
// before it is refused.
const ComposeWithin = 10 * time.Minute
// PutWorld is what one step of a put reads.
type PutWorld struct {
Node, Module string
Put delivery.Put
// Assigned says the module stands assigned to the node.
Assigned bool
// Bus says the module provides the mesh's bus (ADR 0539 §15); Recorded that it is of policy record (§14).
Bus, Recorded bool
// Build is the state of the builder's build at the put's fingerprint, "" when no key is written; BuildWords its error.
Build, BuildWords string
// Newest is the module's newest fingerprint at the trunk, which the builder builds by itself.
Newest string
// Composed is the composition of the module for the node at the fingerprint, as its writer holds it; nil when there is
// none. Granted says the grant at the fingerprint names the node's accounts. Both are read, never asked for (clause 4).
Composed *delivery.ComposedDigest
Granted bool
// Answer is the option the operator chose on this put's own ask (its asking, PutAsking), "" when none; AskExpired says
// that ask expired unanswered.
Answer string
AskExpired bool
// Snapshot says where the bus's streams were snapshotted for this put, once they were; Snapshotting that it was
// asked; SnapshotFailed why it was not taken.
Snapshot string
Snapshotting bool
SnapshotFailed string
// Have is what the node says it runs of the module; nil when it says nothing.
Have *delivery.Have
Now time.Time
}
// PutNext is what a step decides: the put as it stands after it, and the acts the pass does for it.
type PutNext struct {
Put delivery.Put
// Ask writes the put's ask about the move (PutAsk).
Ask bool
// Snapshot has the bus's node back the bus up now, its dump the streams' snapshot (ADR 0235).
Snapshot bool
// Warranted records, once, in the controller's hand-act log that the bus's want was written on the operator's warrant.
Warranted bool
}
// StepPut takes a put one step further from what the world holds (ADR 0546 clauses 3 to 8).
func StepPut(w PutWorld) PutNext {
p := w.Put
if delivery.PutEnded(p.Step) {
return PutNext{Put: p}
}
say := func(step, format string, args ...any) PutNext {
return PutNext{Put: p.Said(w.Now, step, fmt.Sprintf(format, args...))}
}
fp := short(p.FP)
if !w.Assigned {
return say(delivery.PutRefused, "%s is not assigned to %s: a put moves a module where it stands assigned (mesh-controller.assign first)", w.Module, w.Node)
}
switch w.Build {
case delivery.BuildDone:
case delivery.BuildFailed:
return say(delivery.PutRefused, "the builder's build of %s at %s failed: %s", w.Module, fp, w.BuildWords)
case "":
if p.FP != w.Newest {
// The builder builds the trunk's newest fingerprint and the candidates', and takes no ask for another yet (ADR 0546
// clause 3): an older fingerprint it never built, or whose key it removed, cannot be put.
return say(delivery.PutRefused, "the builder holds no build of %s at %s, and builds by itself only the trunk's newest (%s): put that, or a version it built", w.Module, fp, short(w.Newest))
}
return say(delivery.PutBuild, "waits for the builder's build of %s at %s, the trunk's newest, which it builds by itself", w.Module, fp)
default:
return say(delivery.PutBuild, "waits for the builder's build of %s at %s, which is %s", w.Module, fp, w.Build)
}
if w.Composed == nil || !w.Granted {
missing := fmt.Sprintf("no composition of %s for %s at %s is written", w.Module, w.Node, fp)
if w.Composed != nil {
missing = fmt.Sprintf("the grant of %s at %s does not name %s's accounts", w.Module, fp, w.Node)
}
since := w.Now
if p.Step == delivery.PutCompose && len(p.Steps) > 0 {
since = p.Steps[len(p.Steps)-1].At
}
if w.Now.Sub(since) > ComposeWithin {
return say(delivery.PutRefused, "%s after %s: only a composition written by the take-in of its build (or one of the five kept) is put; "+
"for another the hand path is mesh-hand-compose (ADR 0539 §10)", missing, ComposeWithin)
}
return say(delivery.PutCompose, "waits up to %s: %s yet", ComposeWithin, missing)
}
d := *w.Composed
if p.Step == delivery.PutWanted || p.Step == delivery.PutHolding {
return PutNext{Put: applied(w, p)}
}
p.Composed = d.Digest
if needsPerson(w, d) {
switch w.Answer {
case OptionNo:
return say(delivery.PutRefused, "the operator answered no to putting %s at %s on %s", w.Module, fp, w.Node)
case OptionYes:
default:
if w.AskExpired {
return say(delivery.PutRefused, "the operator's ask about putting %s at %s on %s expired unanswered: the node keeps what it runs", w.Module, fp, w.Node)
}
next := say(delivery.PutPerson, "waits for the operator's yes to putting %s at %s on %s (composition %s, judged %s)", w.Module, fp, w.Node, short(d.Digest), d.Judgement)
if p.Ask == "" {
next.Ask = true
next.Put.Ask, _ = delivery.RecordAskKey(w.Node, w.Module, d.Digest)
}
return next
}
if w.Bus {
switch {
case w.Snapshot != "":
p.Snapshot = w.Snapshot
case w.SnapshotFailed != "":
return say(delivery.PutRefused, "the bus's streams were not snapshotted, so the bus is not put: %s", w.SnapshotFailed)
case w.Snapshotting:
return say(delivery.PutSnapshot, "the operator said yes; waits for %s's backup holder to snapshot the bus's streams", w.Node)
default:
next := say(delivery.PutSnapshot, "the operator said yes; asked %s's backup holder to snapshot the bus's streams", w.Node)
next.Snapshot = true
return next
}
}
}
words := fmt.Sprintf("the want of %s on %s names %s (composition %s)", w.Module, w.Node, fp, short(d.Digest))
if p.Hold {
words += ", held until released"
}
if p.Snapshot != "" {
words += "; the streams were snapshotted: " + p.Snapshot
}
next := PutNext{Put: applied(w, p.Said(w.Now, delivery.PutWanted, words))}
next.Warranted = w.Bus && p.Ask != ""
return next
}
// applied is a wanted put once the node's have says how it went: holding or done when it runs the composition healthy.
func applied(w PutWorld, p delivery.Put) delivery.Put {
h := w.Have
if h == nil || h.FP != p.FP || h.Composed != p.Composed {
return p
}
switch {
case h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK && p.Hold:
return p.Said(w.Now, delivery.PutHolding, fmt.Sprintf("%s runs %s at %s healthy; held there until released", w.Node, w.Module, short(p.FP)))
case h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK:
return p.Said(w.Now, delivery.PutDone, fmt.Sprintf("%s runs %s at %s healthy; the rules write its want again", w.Node, w.Module, short(p.FP)))
case failed(*h):
return p.Said(w.Now, p.Step, fmt.Sprintf("%s runs %s at %s and its health check failed: %s", w.Node, w.Module, short(p.FP), h.Detail))
}
return p
}
// needsPerson says whether the put's move waits for the operator: any move of the bus but a revision at the fingerprint the
// node runs that changes only its user list (ADR 0539 §7, §15), and a move of a module of policy record judged to recreate a
// container, move data, or that cannot be judged (§14).
func needsPerson(w PutWorld, d delivery.ComposedDigest) bool {
runs := w.Have != nil && w.Have.FP == w.Put.FP
if runs && w.Have.Composed == d.Digest {
return false // the node runs this composition already: nothing moves
}
if w.Bus {
return !(runs && d.UserListOnly)
}
if !w.Recorded {
return false
}
return d.Judgement == delivery.JudgedRecreates || d.Judgement == delivery.JudgedMovesData || d.Judgement == delivery.JudgedCannot
}
// PutWant is the want a put writes once it is wanted: the fingerprint and composition, marked put, and held when it holds.
func PutWant(p delivery.Put) delivery.Want {
return delivery.Want{FP: p.FP, Composed: p.Composed, Put: true, Hold: p.Hold, Words: "put: " + p.Why}
}
// PutOwns says what a put does to the node's want: owns says the rules pass over it, want the want it writes (nil: what
// stands stays). A put owns the want from its asking until it ends; a held put ends only when released.
func PutOwns(p delivery.Put) (want *delivery.Want, owns bool) {
switch p.Step {
case delivery.PutWanted, delivery.PutHolding:
w := PutWant(p)
return &w, true
case delivery.PutDone, delivery.PutReleased, delivery.PutRefused:
return nil, false
}
return nil, true
}
// PutAsking is a put's own asking number: when it was asked, in milliseconds. Each put asked under the same key is another
// asking, so the ask is written again under an id of its own and an answer to an earlier put, whose warrant names that put's
// asking, never counts for a later one (ADR 0546 clause 8: one approval binds one act).
func PutAsking(p delivery.Put) int {
if n := p.At.UnixMilli(); n > 0 {
return int(n)
}
return 0
}
// PutAct is the act a bus put's yes binds and that is performed when its want is written.
func PutAct(node, module, fp, composed string) asks.Act {
return asks.Act{"put": "want", "node": node, "module": module, "fp": fp, "composed": composed}
}
// PutAsk is the ask about a put's move (ADR 0546 clauses 7 and 8), under the record ask's key of the composition so the
// teller relays it and its answer as any record ask's. For the bus its yes is of level approve and binds the act, so only a
// channel that proves who answered takes it (ADR 0259), never the view.
func PutAsk(key, node, module string, p delivery.Put, d delivery.ComposedDigest, bus bool, now time.Time) (delivery.AssignerAsk, error) {
yes := asks.Option{ID: OptionYes, Label: "Yes, put it", Does: "the node is told to apply " + module + " at " + short(p.FP), Level: asks.Acknowledge}
what := map[delivery.Judgement]string{
delivery.JudgedRecreates: "recreates a container",
delivery.JudgedMovesData: "moves the module's data",
delivery.JudgedCannot: "cannot be judged: " + d.Words,
delivery.JudgedNeither: "recreates no container and moves no data",
delivery.JudgedUnchanged: "changes nothing the node runs",
}[d.Judgement]
explanation := fmt.Sprintf("Asked to put %s at version %s on %s: %s. The move %s.", module, short(p.FP), node, p.Why, what)
expires := now.Add(7 * 24 * time.Hour)
if bus {
binds, err := asks.ActDigest(PutAct(node, module, p.FP, d.Digest))
if err != nil {
return delivery.AssignerAsk{}, err
}
yes.Level, yes.Binds = asks.Approve, binds
yes.Does = "the bus's streams are snapshotted, then " + node + " is told to apply the bus at " + short(p.FP)
explanation += " It is the mesh's bus: every module reconnects while it restarts, and its streams are snapshotted first."
expires = now.Add(asks.ApproveLasts)
}
return delivery.AssignerAsk{Ask: asks.Ask{
ID: delivery.AskingID(key, PutAsking(p)),
Headline: fmt.Sprintf("Put %s on %s?", module, node),
Explanation: explanation,
Options: []asks.Option{yes,
{ID: OptionNo, Label: "No", Does: "the node keeps what it runs, and the put is refused", Level: asks.Acknowledge}},
Who: asks.Operator,
Expires: expires,
OnExpiry: "the node keeps what it runs",
About: key,
}, Asking: PutAsking(p)}, nil
}
@@ -4,8 +4,6 @@ import (
"fmt"
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
@@ -15,20 +13,13 @@ import (
// trunk (the indexer's modules), whether the builder's build of F is done, whether F is held, whether the controller
// issued its credential for F, its manifest at F, and the nodes it is assigned to.
type Module struct {
Name string
FP string
Built bool
Held bool
Granted bool
// GrantedAt says whether the controller issued the module's credential for another fingerprint than FP: the first want of a node
// that runs an older one names it (issue 1945). Nil: none.
GrantedAt func(fp string) bool
// GrantedOn says whether the controller's grant of FP names the module's accounts on a node, when the grant names any:
// a grant written for the module's other nodes says nothing of a node whose account is not issued and sealed (issue
// 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus). Nil: the grant alone.
GrantedOn func(node string) bool
Manifest catalogue.Manifest
Nodes []string
Name string
FP string
Built bool
Held bool
Granted bool
Manifest catalogue.Manifest
Nodes []string
// Composed answers the controller's composition of the module for a node at a fingerprint (mesh-controller_composed): its
// digest and the judgement of the move to it, and whether it exists. Nil: none exists (ADR 0539 §1, §6).
Composed func(node, fp string) (delivery.ComposedDigest, bool)
@@ -40,12 +31,6 @@ type Module struct {
// Restored says whether the operator answered "restore" to the restore ask of this data module on node: it is then set back
// as any module is, losing the writes made since (ADR 0539 §19). Nil: none did.
Restored func(node string) bool
// Kept says the operator answered "keep" to the hold ask of FP: a hold kept so is not lifted by itself.
Kept bool
// Now is when this pass reads the haves: the held record names it as when the health check was seen to fail. Zero: not said.
Now time.Time
// Puts are the puts of the module, by node (ADR 0546): while one owns a node's want (PutOwns), the rules pass over it.
Puts map[string]delivery.Put
}
func (m Module) composition(node, fp string) (delivery.ComposedDigest, bool) {
@@ -84,40 +69,27 @@ func Canary(nodes []string, online map[string]bool) string {
// with why (ADR 0429 §6). Rule 4, at most `step` others ahead, always holds: the mesh keeps no step, and step 0 is no
// limit (ADR 0528 clause 3).
func Holds(m Module, node string, n Nodes) (bool, int, string) {
return holds(m, node, n, true)
}
// holds is Holds; with ordering false it leaves out the two rules that order a move among nodes and modules, the canary (3)
// and the providers (5), and keeps those without which there is nothing to want: the build done, not held, the grant and the
// composition.
func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
switch {
case !m.Built:
return false, delivery.RuleBuildDone, fmt.Sprintf("the build of %s at %s is not done", m.Name, short(m.FP))
case m.Held:
return false, delivery.RuleNotHeld, fmt.Sprintf("%s is held at %s", m.Name, short(m.FP))
}
if ordering {
canary := Canary(m.Nodes, n.Online)
switch {
case canary == "":
return false, delivery.RuleCanary, fmt.Sprintf("none of the nodes of %s is online, so it has no canary", m.Name)
case node != canary && !healthy(n.Haves[canary][m.Name], m.FP):
return false, delivery.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP))
}
for _, provision := range m.Manifest.Requires {
if ok, why := providedHealthy(provision, node, n); !ok {
return false, delivery.RuleProviders, why
}
canary := Canary(m.Nodes, n.Online)
switch {
case canary == "":
return false, delivery.RuleCanary, fmt.Sprintf("none of the nodes of %s is online, so it has no canary", m.Name)
case node != canary && !healthy(n.Haves[canary][m.Name], m.FP):
return false, delivery.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP))
}
for _, provision := range m.Manifest.Requires {
if ok, why := providedHealthy(provision, node, n); !ok {
return false, delivery.RuleProviders, why
}
}
if !m.Granted {
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s for %s", m.Name, short(m.FP))
}
if m.GrantedOn != nil && !m.GrantedOn(node) {
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s on %s for %s", m.Name, node,
short(m.FP))
}
if _, ok := m.composition(node, m.FP); !ok {
return false, delivery.RuleComposed, fmt.Sprintf("the controller has not composed %s for %s at %s", m.Name, node, short(m.FP))
}
@@ -166,9 +138,6 @@ type Outcome struct {
Wants map[string]delivery.Want
// Held is the hold of m.FP to write, when a node's health check failed on it and it is not held yet.
Held *delivery.Held
// Lifted is the hold of m.FP lifted by itself: every node that runs the module runs m.FP healthy (issue 1906). The held
// record and its ask are taken away, and the wants move by the rules again in this pass.
Lifted bool
// Record is the moves a person must allow before the want names them: each is asked once (ADR 0539 §14).
Record []RecordNeed
// Restore is the nodes of a module holding data whose held fingerprint failed its health check there and that are not set
@@ -176,11 +145,6 @@ type Outcome struct {
Restore []string
// Rules is, by node, the rule holding the node back from m.FP and why.
Rules map[string]Holding
// Neither is, by node, the nodes the module is assigned to that run it (or whose have names it), hold no want for it after this
// pass and are not held (a held module is on the held list), with the rule holding the first want back: the node's own composition does not hold a module with a fingerprint of the new form, so such a module is on
// neither list and nothing applies it on that node (issue 1945). A module with no fingerprint of the new form goes with the node's
// own composition and is never here.
Neither map[string]Holding
}
// RecordNeed is a move of a module of policy record on a node that waits for the operator's yes: the composition's digest, the
@@ -204,15 +168,12 @@ type Holding struct {
// module holding data is not set back but asked about (Restore). A
// node that wants m but is no longer among its nodes gets its want written removed, never deleted (ADR 0408 §4).
func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[string]string) Outcome {
out := Outcome{Wants: map[string]delivery.Want{}, Rules: map[string]Holding{}, Neither: map[string]Holding{}}
out := Outcome{Wants: map[string]delivery.Want{}, Rules: map[string]Holding{}}
held := m.Held
if held && !m.Kept && m.FP != "" && allRunHealthy(m, n) {
out.Lifted, held = true, false
}
if !held && !out.Lifted && !m.Released && m.FP != "" {
if !held && !m.Released && m.FP != "" {
for _, node := range slices.Sorted(slices.Values(m.Nodes)) {
if h, ok := n.Haves[node][m.Name]; ok && h.FP == m.FP && failed(h) {
out.Held = &delivery.Held{Node: node, Words: HeldWords(h, m.Now)}
if h, ok := n.Haves[node][m.Name]; ok && h.FP == m.FP && h.Health == delivery.HealthFailed {
out.Held = &delivery.Held{Node: node, Words: h.Detail}
held = true
break
}
@@ -226,29 +187,6 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
}
before, had := wants[node]
var next delivery.Want
p, put := m.Puts[node]
if !put && had && before.Put && before.Hold {
// The want is the truth (ADR 0546 clause 5): a want marked put and held stands while its put is not heard (the
// puts not open, or not yet handed after a start); only a put heard released gives it back to the rules. A put
// without hold is not kept this way: its put takes it further once heard.
p, put = delivery.Put{FP: before.FP, Composed: before.Composed, Hold: true, Why: strings.TrimPrefix(before.Words, "put: "),
Step: delivery.PutHolding}, true
}
if put {
if want, owns := PutOwns(p); owns {
if want != nil {
w := *want
// A held bus still takes a revision of its composition that changes only its user list (ADR 0539 §7).
if ul, ok := m.userListRevision(node, before, had, n); ok && ul.FP == w.FP {
w.Composed = ul.Composed
}
if !had || !delivery.SameWant(normal(before), normal(w)) {
out.Wants[node] = w
}
}
continue
}
}
// A revision of the bus module's composition that differs only in its user list is written at once, at the fingerprint the
// node runs, outside the rules, the record hold and the bar on a new fingerprint: a new module's credential must reach the
// bus before the module can connect (ADR 0539 §7). The controller marks it; whatever it judges of the file's mount stands.
@@ -258,49 +196,15 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
}
continue
}
ok, rule, words := Holds(mm, node, n)
target := m.FP
if have, ran := n.Haves[node][m.Name]; !ok && (!had || before.Removed) && (rule == delivery.RuleCanary || rule == delivery.RuleProviders) && runs(have) {
// A node that runs the module and has no want for it has no want the ordering rules could hold it back to: holding its first
// want back leaves the module on neither list, since the node's own composition gives up every module with a fingerprint of
// the new form (issue 1945: on g14 the bundles of clipmenu, screen-lock and every module requiring x11-display were removed
// and never made again, because xorg's health is unknown and screen-lock's canary never ran it healthy). Its first want names
// what it runs, never a newer fingerprint, so the canary is never passed over: F when it runs F (by the rules without which
// there is nothing to want), or an older fingerprint of the new form whose composition and grant exist. Rules 3 and 5 then
// order the move from it as from any want.
switch _, composed := m.composition(node, have.FP); {
case !ran:
case have.FP == m.FP:
ok, rule, words = holds(mm, node, n, false)
case !strings.HasPrefix(have.FP, delivery.SourcePrefix2):
// An old-form build (not src2:) has no composition, so no want can ever name it: its first want names F, and only by every
// rule as usual, the canary (3) and the providers (5) included, which here hold it back. Nothing is written and the module
// stays on neither list until the canary runs F healthy; then Holds passes and the want names F (issue 1945, a design
// limit set by the lead: the canary stays in force and old-form modules leave neither list by themselves).
case composed && m.GrantedAt != nil && m.GrantedAt(have.FP):
ok, target = true, have.FP
}
}
switch {
switch ok, rule, words := Holds(mm, node, n); {
case ok:
d, _ := m.composition(node, target)
next = delivery.Want{FP: target, Composed: d.Digest}
at := m
if target != m.FP {
// the first want at the older fingerprint the node runs: held back from F by the rule, and judged as a want at it
next.Newest, next.Rule, next.Words = m.FP, rule, words
at.FP = target
}
if had && !before.Removed && !before.Put && before.FP == target && before.Composed == d.Digest {
d, _ := m.composition(node, m.FP)
next = delivery.Want{FP: m.FP, Composed: d.Digest}
if had && !before.Removed && before.FP == m.FP && before.Composed == d.Digest {
next.Backup, next.Words = before.Backup, before.Words // the answer to a failed backup stays while its target stays (ADR 0539 §16)
}
running, ran := n.Haves[node][m.Name]
// A removed want is no want: a first want after it goes through the record ask as a first want does.
asked, hadLive := before, had
if before.Removed {
asked, hadLive = delivery.Want{}, false
}
if need, wait := at.mustAsk(node, asked, hadLive, ran, running.FP, running.Composed == "", d); wait {
if need, wait := m.mustAsk(node, before, had, ran, running.FP, running.Composed == "", d); wait {
if need != nil {
out.Record = append(out.Record, *need)
}
@@ -329,43 +233,11 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
default:
out.Rules[node] = Holding{Rule: rule, Words: words}
if !had || before.Removed {
// The node runs the module, or its have names it, and gets no want for it: the module is on neither list (issue 1945), and
// the pass says so. A new assignment, with no have, waits for the rules and is on no list yet by design. A held module (rule 2,
// held before or by this pass) is on the held list instead, whose hold is said and answered on its own: it is never on neither
// list (issue 1945: systemd-resolved, held on shanks by the operator, was said as urgent every hour).
h, there := n.Haves[node][m.Name]
switch {
case there && h.Phase != delivery.PhaseRemoved && !held:
out.Neither[node] = Holding{Rule: rule, Words: words}
case had && (!there || h.Phase == delivery.PhaseRemoved):
// A removed want of a node that does not run the module says why no want follows it (issue 1945), so the view's
// assigner_want answers it; it stays removed. A node with no key gets none: a removed want written where none was
// would have its node agent remove the module and stop holding its composition's resources back.
next = delivery.Want{Removed: true, Words: NoWantWords(rule, words)}
if !delivery.SameWant(normal(before), normal(next)) {
out.Wants[node] = next
}
}
continue
}
next = delivery.Want{FP: before.FP, Composed: before.Composed, Newest: m.FP, Rule: rule, Words: words}
if before.FP == m.FP {
// Held at F by the rules: a new revision of F's composition waits for them, as a new fingerprint does (ADR 0539 §5).
next = delivery.Want{FP: m.FP, Composed: before.Composed}
} else if d, there := m.composition(node, before.FP); there && d.Digest != "" && d.Digest != before.Composed {
// Held back at an earlier fingerprint whose kept composition the controller composed again with today's config
// (novox/hq ADR 0545): the want follows its new revision, so the node does not run that build with the config of
// the moment it was made (on g14 a capacity of 0 after a rollback). Asked as any move of a record module is.
at := m
at.FP = before.FP
running, ran := n.Haves[node][m.Name]
need, wait := at.mustAsk(node, before, true, ran, running.FP, running.Composed == "", d)
if need != nil {
out.Record = append(out.Record, *need)
}
if !wait {
next.Composed = d.Digest
}
}
}
if !had || !delivery.SameWant(normal(before), normal(next)) {
@@ -380,63 +252,6 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
return out
}
// NoWantWords say why a node gets no want for a module: the rule that holds its first want back and why (issue 1945).
func NoWantWords(rule int, words string) string {
return fmt.Sprintf("no want: rule %d: %s", rule, words)
}
// failed says whether a have's health check failed once the node is done moving: phase running or failed (never removed, held, or
// moving). A failed health while it
// backs up, applies or checks is a step of the move, not its outcome (on novox at 15:27:36 on 2026-10-25, "health failed, phase
// applying" held the controller's canary two seconds before the new build was healthy). Every healthy read (healthy, providedHealthy,
// the world's lastHealthy) already asks for phase running.
func failed(h delivery.Have) bool {
return h.Health == delivery.HealthFailed && (h.Phase == delivery.PhaseRunning || h.Phase == delivery.PhaseFailed)
}
// runs says whether a have is of a module the node runs: running, whatever its health.
func runs(h delivery.Have) bool {
return h.Phase == delivery.PhaseRunning
}
// allRunHealthy says whether every node of m that runs the module (its have's phase is running, or failed) runs m.FP healthy, and
// at least one does: a hold whose fingerprint runs healthy wherever the module runs holds nothing back any more (issue 1906: seven
// modules stayed held on every machine after one transient failed check). A node set back to another fingerprint does not run
// m.FP, and a failed one does not run it healthy, so the hold stays; a have that is removed, held for its backup or still moving
// (backing up, applying, checking) is not counted, or it would keep the hold up for ever.
func allRunHealthy(m Module, n Nodes) bool {
ran := 0
for _, node := range m.Nodes {
h, ok := n.Haves[node][m.Name]
if !ok || (h.Phase != delivery.PhaseRunning && h.Phase != delivery.PhaseFailed) {
continue
}
if !healthy(h, m.FP) {
return false
}
ran++
}
return ran > 0
}
// HeldWords are the words a hold keeps of its cause (issue 1906): the health and phase the node reported, when the assigner saw
// it, and the node's own words. The node is the held record's own field.
func HeldWords(h delivery.Have, at time.Time) string {
words := fmt.Sprintf("health %s, phase %s", h.Health, h.Phase)
if !at.IsZero() {
words += ", seen at " + at.UTC().Format(time.RFC3339)
}
if d := strings.TrimSpace(h.Detail); d != "" {
return words + ": " + d
}
return words + ": the node gave no words"
}
// LiftedWords say why a hold was lifted by itself.
func LiftedWords(module, fp string) string {
return fmt.Sprintf("every node that runs %s runs %s healthy, so the assigner lifted its hold by itself", module, short(fp))
}
// normal is a want without its version, as a decoded and a composed one are compared.
func normal(w delivery.Want) delivery.Want {
w.V = 0
@@ -514,9 +329,3 @@ func (m Module) userListRevision(node string, before delivery.Want, had bool, n
}
return w, true
}
// GrantNamesNode says whether a grant of module names its accounts on node (mesh-sdk's ControllerGrant.NamesNode, which a
// put reads by too).
func GrantNamesNode(g delivery.ControllerGrant, node, module string) bool {
return g.NamesNode(node, module)
}
@@ -4,7 +4,6 @@ import (
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
@@ -86,33 +85,6 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
t.Errorf("rule %d broken: g14's want is %+v", rule, w)
}
}
// Rule 6 per node: a grant written for the module's other nodes does not grant a node whose account it does not name
// (issue 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus).
m, n := ready()
grant := delivery.ControllerGrant{Accounts: []string{"ace.mail", "ace.mail.send"}}
m.GrantedOn = func(node string) bool { return GrantNamesNode(grant, node, "mail") }
out := Wants(m, n, wantedOld(), nil)
if w := out.Wants["ace"]; w.FP != fNew {
t.Errorf("ace, whose account the grant names, wants %+v", w)
}
if got := out.Rules["g14"]; got.Rule != delivery.RuleGrant || !strings.Contains(got.Words, "on g14") {
t.Errorf("g14, whose account the grant does not name, is held by %+v", got)
}
for _, c := range []struct {
accounts []string
node, module string
named bool
}{
{nil, "g14", "mail", true},
{[]string{"g14.mail.send"}, "g14", "mail", true},
{[]string{"g14.mailer"}, "g14", "mail", false},
{[]string{"ace.mail"}, "g14", "mail", false},
} {
if got := GrantNamesNode(delivery.ControllerGrant{Accounts: c.accounts}, c.node, c.module); got != c.named {
t.Errorf("%v names %s of %s: %v, want %v", c.accounts, c.module, c.node, got, c.named)
}
}
}
// 1297/B4, ADR 0435 §3: a canary that is offline is passed over: the next node by name is the canary, and moves first.
@@ -138,10 +110,8 @@ func TestAnOfflineCanaryIsPassedOver(t *testing.T) {
func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
m, n := ready()
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"}
m.Now = time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC)
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fOld, Composed: dig(fOld)}}, map[string]string{"ace": fOld})
// 1906/H3: the held record keeps its cause in words: which machine (its node), which health, when, and the node's words.
if out.Held == nil || out.Held.Node != "ace" || out.Held.Words != "health failed, phase running, seen at 2026-10-25T01:00:00Z: connection refused" {
if out.Held == nil || out.Held.Node != "ace" || out.Held.Words != "connection refused" {
t.Fatalf("held = %+v", out.Held)
}
if w := out.Wants["ace"]; w.FP != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule != delivery.RuleNotHeld {
@@ -248,25 +218,6 @@ func TestARevisionAtTheSameFingerprintIsAWantByTheSameRules(t *testing.T) {
if w, ok := out.Wants["ace"]; ok && w.Composed == revised {
t.Errorf("a revision was written while a rule holds: %+v", w)
}
// A want held back at an earlier fingerprint follows the new revision of that fingerprint's kept composition, which the
// controller composed again with today's config (novox/hq ADR 0545: on g14 a capacity of 0 after a rollback), while
// the rules still hold F; for a record module whose revision recreates a container, only after the ask.
out = Wants(m, n, wantedOld(), nil)
if w := out.Wants["g14"]; w.FP != fOld || w.Composed != revised || w.Newest != fNew {
t.Errorf("g14, held back at its earlier fingerprint, wants %+v; want it at that fingerprint naming the revised digest", w)
}
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
return delivery.ComposedDigest{Digest: revised, Judgement: delivery.JudgedRecreates, Policy: delivery.PolicyRecord,
Against: dig(fOld), Words: "recreates"}, true
}
out = Wants(m, n, wantedOld(), nil)
if w := out.Wants["g14"]; w.Composed == revised {
t.Errorf("a recorded module's revision that recreates a container was written with no ask: %+v", w)
}
if !slices.ContainsFunc(out.Record, func(r RecordNeed) bool { return r.Node == "g14" && r.Digest == revised }) {
t.Errorf("the revision was not asked: %+v", out.Record)
}
}
// 1906/D2: a set-back names the composition of the fingerprint it sets back to, and when that is gone the want stays and says so.
@@ -350,7 +301,6 @@ func TestAMoveJudgedNeitherOrNeverRunOrNotRecordedIsWrittenAtOnce(t *testing.T)
}
m, n, _ = movingTo(delivery.JudgedRecreates)
recording(&m)
m.Nodes = []string{"ace"} // g14 runs it with no want, so its first want is asked (1945/AC19): this case is ace's alone
delete(n.Haves["ace"], "mail")
if out := Wants(m, n, nil, nil); out.Wants["ace"].FP != fNew || len(out.Record) != 0 {
t.Errorf("a node that never ran it: %+v", out)
@@ -384,142 +334,6 @@ func TestTheBusIsNotMovedButItsUserListRevisionIsWrittenAtOnce(t *testing.T) {
if len(out.Record) != 0 {
t.Errorf("a user-list revision was asked: %+v", out.Record)
}
// ADR 0546: a put owns the node's want from its asking until it ends. While it waits the rules write nothing there; once
// wanted its want stands, marked put and held, and a held bus still takes its user-list revision (ADR 0539 §7).
put := delivery.Put{FP: fNew, Hold: true, Why: "the bus's user list", Composed: dig(fNew)}
for _, c := range []struct {
name string
step string
wanted map[string]delivery.Want
want *delivery.Want // nil: nothing written on ace
}{
{"waiting for the operator", delivery.PutPerson, wantedOld(), nil},
{"wanted", delivery.PutWanted, wantedOld(), &delivery.Want{FP: fNew, Composed: dig(fNew), Put: true, Hold: true, Words: "put: the bus's user list"}},
{"holding, written already", delivery.PutHolding, map[string]delivery.Want{"ace": PutWant(put)}, nil},
{"released: the rules again", delivery.PutReleased, map[string]delivery.Want{"ace": PutWant(put)}, &delivery.Want{FP: fNew, Composed: dig(fNew)}},
{"not heard: a held want at an older fingerprint stands", "", map[string]delivery.Want{"ace": {FP: fOld, Composed: dig(fOld), Put: true, Hold: true, Words: "put: back"}}, nil},
{"not heard: a held want the rules would rewrite stands", "", map[string]delivery.Want{"ace": PutWant(put)}, nil},
} {
p := put
p.Step = c.step
m.Puts = map[string]delivery.Put{"ace": p}
if c.step == "" {
m.Puts = nil
}
out := Wants(m, n, c.wanted, nil)
got, wrote := out.Wants["ace"]
switch {
case c.want == nil && wrote:
t.Errorf("put %s: the rules wrote ace's want %+v", c.name, got)
case c.want != nil && (!wrote || !delivery.SameWant(normal(got), *c.want)):
t.Errorf("put %s: ace's want = %+v (written %v), want %+v", c.name, got, wrote, *c.want)
}
}
userList := "sha256:" + strings.Repeat("7", 64)
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
return delivery.ComposedDigest{Digest: userList, Judgement: delivery.JudgedNeither, UserListOnly: true}, true
}
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Composed: dig(fNew), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
held := put
held.Step = delivery.PutHolding
m.Puts = map[string]delivery.Put{"ace": held}
if w := Wants(m, n, map[string]delivery.Want{"ace": PutWant(held)}, nil).Wants["ace"]; w.Composed != userList || !w.Put || !w.Hold {
t.Errorf("a held bus did not take its user-list revision as a put: %+v", w)
}
m.Puts = nil
stepsOfAPut(t)
}
// stepsOfAPut takes a put through its steps (ADR 0546 clauses 3 to 8): the build, the controller's composition, the
// operator's yes, the bus's snapshot, the want, and the have.
func stepsOfAPut(t *testing.T) {
t.Helper()
now := time.Date(2026, 10, 25, 22, 0, 0, 0, time.UTC)
base := delivery.Put{FP: fNew, Hold: true, Why: "the bus's user list", At: now}.Said(now, delivery.PutAsked, "asked")
d := delivery.ComposedDigest{Digest: dig(fNew), Judgement: delivery.JudgedCannot, Words: "the have names no composition"}
running := delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
bus := func() PutWorld {
return PutWorld{Node: "novox", Module: "nats", Put: base, Assigned: true, Bus: true, Build: delivery.BuildDone, Newest: fNew,
Composed: &d, Granted: true, Have: &running, Now: now}
}
cases := []struct {
name string
world func() PutWorld
step string
says string
ask, snap bool
warranted bool
}{
{"not assigned", func() PutWorld { w := bus(); w.Assigned = false; return w }, delivery.PutRefused, "not assigned", false, false, false},
{"no build yet of the newest", func() PutWorld { w := bus(); w.Build = ""; return w }, delivery.PutBuild, "builds by itself", false, false, false},
{"no build of an older one", func() PutWorld { w := bus(); w.Build, w.Newest = "", fpOf('c'); return w }, delivery.PutRefused, "holds no build", false, false, false},
{"a failed build", func() PutWorld { w := bus(); w.Build, w.BuildWords = delivery.BuildFailed, "go vet"; return w }, delivery.PutRefused, "go vet", false, false, false},
{"no composition yet: waits", func() PutWorld { w := bus(); w.Composed = nil; return w }, delivery.PutCompose, "no composition of nats for novox", false, false, false},
{"composed, grant not naming the node", func() PutWorld { w := bus(); w.Granted = false; return w }, delivery.PutCompose, "does not name novox", false, false, false},
{"none within the bound: refused", func() PutWorld {
w := bus()
w.Composed = nil
w.Put = w.Put.Said(now.Add(-ComposeWithin-time.Second), delivery.PutCompose, "waits")
return w
}, delivery.PutRefused, "mesh-hand-compose", false, false, false},
{"the bus asks the operator", bus, delivery.PutPerson, "waits for the operator's yes", true, false, false},
{"the operator said no", func() PutWorld { w := bus(); w.Answer = OptionNo; return w }, delivery.PutRefused, "answered no", false, false, false},
{"the ask expired unanswered", func() PutWorld { w := bus(); w.AskExpired = true; return w }, delivery.PutRefused, "expired unanswered", false, false, false},
{"yes: the snapshot is asked", func() PutWorld { w := bus(); w.Answer = OptionYes; return w }, delivery.PutSnapshot, "asked novox's backup holder", false, true, false},
{"the snapshot is taken", func() PutWorld { w := bus(); w.Answer, w.Snapshotting = OptionYes, true; return w }, delivery.PutSnapshot, "waits for novox's backup holder", false, false, false},
{"the snapshot failed", func() PutWorld { w := bus(); w.Answer, w.SnapshotFailed = OptionYes, "no restic"; return w }, delivery.PutRefused, "no restic", false, false, false},
{"snapshotted: the want, warranted", func() PutWorld {
w := bus()
w.Answer, w.Snapshot = OptionYes, "novox's restore point"
w.Put.Ask, _ = delivery.RecordAskKey("novox", "nats", d.Digest)
return w
}, delivery.PutWanted, "held until released; the streams were snapshotted: novox's restore point", false, false, true},
{"a user-list revision of the bus needs nobody", func() PutWorld {
w := bus()
ul := d
ul.UserListOnly = true
w.Composed, w.Have = &ul, &delivery.Have{FP: fNew, Composed: dig(fOld), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
return w
}, delivery.PutWanted, "held until released", false, false, false},
{"a module not recorded needs nobody", func() PutWorld { w := bus(); w.Bus = false; return w }, delivery.PutWanted, "names", false, false, false},
{"a recorded module judged to recreate asks", func() PutWorld {
w := bus()
w.Bus, w.Recorded = false, true
r := d
r.Judgement = delivery.JudgedRecreates
w.Composed = &r
return w
}, delivery.PutPerson, "waits for the operator's yes", true, false, false},
{"wanted, the node runs it: holding", func() PutWorld {
w := bus()
w.Put = w.Put.Said(now, delivery.PutWanted, "wanted")
w.Put.Composed = d.Digest
w.Have = &delivery.Have{FP: fNew, Composed: d.Digest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
return w
}, delivery.PutHolding, "held there until released", false, false, false},
{"wanted without hold: done", func() PutWorld {
w := bus()
w.Put.Hold = false
w.Put = w.Put.Said(now, delivery.PutWanted, "wanted")
w.Put.Composed = d.Digest
w.Have = &delivery.Have{FP: fNew, Composed: d.Digest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
return w
}, delivery.PutDone, "the rules write its want again", false, false, false},
{"released: nothing more", func() PutWorld { w := bus(); w.Put = w.Put.Said(now, delivery.PutReleased, "released"); return w }, delivery.PutReleased, "released", false, false, false},
}
for _, c := range cases {
next := StepPut(c.world())
if next.Put.Step != c.step || !strings.Contains(next.Put.Words, c.says) {
t.Errorf("put, %s: at %s, %q; want %s saying %q", c.name, next.Put.Step, next.Put.Words, c.step, c.says)
}
if next.Ask != c.ask || next.Snapshot != c.snap || next.Warranted != c.warranted {
t.Errorf("put, %s: acts ask %v snapshot %v warranted %v; want %v %v %v", c.name, next.Ask, next.Snapshot, next.Warranted,
c.ask, c.snap, c.warranted)
}
if next.Ask && next.Put.Ask == "" {
t.Errorf("put, %s: an ask without its key", c.name)
}
}
}
// 1906/D4 (ADR 0539 §16): the answer to a failed backup stays in the want while its target stays.
@@ -706,370 +520,3 @@ func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) {
}
}
}
// 1906/H3: a failed check whose node gave no words is held saying so, never with empty words.
func TestAHoldOfAFailureWithoutWordsSaysSo(t *testing.T) {
m, n := ready()
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}}, map[string]string{"ace": fOld})
if out.Held == nil || out.Held.Words != "health failed, phase running: the node gave no words" {
t.Fatalf("held = %+v", out.Held)
}
}
// 1906/H2: given a held fingerprint that every node running the module now runs healthy, when the assigner passes, then the
// hold is lifted by itself and the wants move to it by the rules again. A node set back to another fingerprint, a node whose
// check fails, or a hold the operator answered "keep" leaves it held; a node that does not run the module at all does not count.
func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
everywhere := func() (Module, Nodes) {
m, n := ready()
m.Held = true
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
return m, n
}
m, n := everywhere()
out := Wants(m, n, wantedOld(), map[string]string{"ace": fOld, "g14": fOld})
if !out.Lifted || out.Held != nil {
t.Fatalf("lifted = %v, held = %+v: want the hold lifted", out.Lifted, out.Held)
}
for _, node := range []string{"ace", "g14"} {
if w := out.Wants[node]; w.FP != fNew || w.Rule != 0 {
t.Errorf("%s's want = %+v, want it moved to the lifted fingerprint", node, w)
}
}
m, n = everywhere()
delete(n.Haves["g14"], "mail")
if out := Wants(m, n, wantedOld(), nil); !out.Lifted {
t.Error("a node that does not run the module kept the hold")
}
for name, change := range map[string]func(*Module, *Nodes){
"set back": func(_ *Module, n *Nodes) {
n.Haves["g14"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
},
"failing": func(_ *Module, n *Nodes) {
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
},
"kept": func(m *Module, _ *Nodes) { m.Kept = true },
"no haves": func(_ *Module, n *Nodes) { delete(n.Haves["g14"], "mail"); delete(n.Haves["ace"], "mail") },
"not held": func(m *Module, _ *Nodes) { m.Held = false },
} {
m, n := everywhere()
change(&m, &n)
if out := Wants(m, n, wantedOld(), map[string]string{"ace": fOld, "g14": fOld}); out.Lifted {
t.Errorf("%s: the hold was lifted", name)
}
}
}
// x11 is screen-lock as on g14 and shanks on 2026-10-25 (issue 1945): built, granted and composed at F and at the older fOld
// on both, requiring x11-display, whose provider xorg runs on each node with its health unknown; each node runs screen-lock at fOld
// (health unknown) and holds no want for it. The canary g14 does not run F.
func x11() (Module, Nodes) {
m := Module{Name: "screen-lock", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"g14", "shanks"},
GrantedAt: func(fp string) bool { return fp == fNew || fp == fOld },
Manifest: catalogue.Manifest{Module: "screen-lock", Requires: []string{"x11-display"}}}
n := Nodes{
Online: map[string]bool{"g14": true, "shanks": true},
Haves: map[string]map[string]delivery.Have{
"g14": {"screen-lock": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown},
"xorg": {FP: fpOf('x'), Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}},
"shanks": {"screen-lock": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown},
"xorg": {FP: fpOf('x'), Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}},
},
Providers: map[string]map[string][]string{"x11-display": {"xorg": {"g14", "shanks"}}},
}
return m, n
}
// 1945/AC19: given a node that runs an older fingerprint of the new form, composed and granted, with no want, held back from F by
// the providers or the canary rule, when the assigner passes, then its first want names the fingerprint it runs with that
// composition, never the newest, and says the newest it is held back from; the canary is never passed over.
func TestANodeRunningAModuleWithNoWantGetsItsFirstWantAtWhatItRuns(t *testing.T) {
m, n := x11()
out := Wants(m, n, nil, nil)
for _, node := range []string{"g14", "shanks"} {
if w := out.Wants[node]; w.FP != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule == 0 {
t.Errorf("%s's want = %+v, want its first want at the fingerprint it runs, held back from F", node, w)
}
}
if len(out.Neither) != 0 {
t.Errorf("left on neither list: %+v", out.Neither)
}
// with the provider healthy, only the canary holds shanks back: still at what it runs
m, n = x11()
n.Haves["g14"]["xorg"] = delivery.Have{FP: fpOf('x'), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
n.Haves["shanks"]["xorg"] = n.Haves["g14"]["xorg"]
if w := Wants(m, n, nil, nil).Wants["shanks"]; w.FP != fOld || w.Rule != delivery.RuleCanary {
t.Errorf("shanks held back by the canary with no want: %+v, want at fOld by rule 3", w)
}
// a node already running F gets its want at F; a removed want is no want either
m, n = x11()
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
out = Wants(m, n, map[string]delivery.Want{"g14": {Removed: true}}, nil)
if w := out.Wants["g14"]; w.FP != fNew || w.Composed != dig(fNew) || w.Removed {
t.Errorf("g14 running F with a removed want = %+v, want F", w)
}
}
// 1945/AC19: a node running a fingerprint that is not of the new form, or one whose composition or grant does not exist, gets no
// want past the rules, and is named on neither list; the ordering rules still order every move from a want.
func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) {
for what, change := range map[string]func(*Module, *Nodes){
"an old-form build": func(_ *Module, n *Nodes) {
n.Haves["shanks"]["screen-lock"] = delivery.Have{FP: "build-2", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
},
"no grant": func(m *Module, _ *Nodes) { m.GrantedAt = func(fp string) bool { return fp == fNew } },
"no composition": func(m *Module, _ *Nodes) {
m.Composed = func(_, fp string) (delivery.ComposedDigest, bool) {
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedNeither}, fp == fNew
}
},
} {
m, n := x11()
change(&m, &n)
out := Wants(m, n, nil, nil)
if w, ok := out.Wants["shanks"]; ok {
t.Errorf("%s: shanks got a want: %+v", what, w)
}
if h, ok := out.Neither["shanks"]; !ok || h.Rule == 0 {
t.Errorf("%s: shanks on neither list = %+v (named %v)", what, h, ok)
}
}
m, n := x11()
before := map[string]delivery.Want{"g14": {FP: fOld, Composed: dig(fOld)}, "shanks": {FP: fOld, Composed: dig(fOld)}}
out := Wants(m, n, before, nil)
if len(out.Neither) != 0 {
t.Errorf("nodes holding a want named on neither list: %+v", out.Neither)
}
for node, w := range out.Wants {
if w.FP != fOld {
t.Errorf("%s's want moved from its want past the ordering rules: %+v", node, w)
}
}
}
// 1945/AC20: a new assignment (a node with no have of the module) waiting for the canary, the build or the grant gets no want and is
// never named on neither list: only a node that runs the module, or whose have names it, is; a module with no fingerprint of the new
// form goes with the node's own composition and is never named.
func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) {
m, n := x11()
delete(n.Haves["shanks"], "screen-lock")
out := Wants(m, n, nil, nil)
if w, ok := out.Wants["shanks"]; ok {
t.Errorf("a new assignment waiting for the canary got a want: %+v", w)
}
if _, ok := out.Neither["shanks"]; ok {
t.Errorf("a new assignment waiting for the canary is named on neither list: %+v", out.Neither)
}
m.Built = false
if out := Wants(m, n, nil, nil); len(out.Neither) != 1 || out.Neither["g14"].Rule != delivery.RuleBuildDone {
t.Errorf("with no build: neither = %+v, want g14 alone (it runs the module), by rule 1", out.Neither)
}
// 1945 (the assigner holds a first want back silently): with no build, a node whose want was removed and that does not run
// the module has the reason written into its removed want, once; a node with no key gets no want written, and a node that
// runs the module is on neither list, its removed want left as it is.
gone := delivery.Want{Removed: true, Words: "screen-lock is no longer assigned to shanks"}
for _, c := range []struct {
name string
before map[string]delivery.Want
want map[string]delivery.Want
}{
{"no key: nothing written", nil, map[string]delivery.Want{}},
{"a removed want: it says why", map[string]delivery.Want{"shanks": gone},
map[string]delivery.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is not done"}}},
{"a removed want that says why already: nothing written", map[string]delivery.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is not done"}},
map[string]delivery.Want{}},
{"a node that runs it: on neither list, its removed want left", map[string]delivery.Want{"g14": gone}, map[string]delivery.Want{}},
} {
out := Wants(m, n, c.before, nil)
if len(out.Wants) != len(c.want) {
t.Errorf("%s: wants = %+v, want %+v", c.name, out.Wants, c.want)
continue
}
for node, w := range c.want {
if !delivery.SameWant(normal(out.Wants[node]), w) {
t.Errorf("%s: want of %s = %+v, want %+v", c.name, node, out.Wants[node], w)
}
}
}
m.FP = ""
if out := Wants(m, n, nil, nil); len(out.Neither) != 0 {
t.Errorf("a module of the node's own composition named on neither list: %+v", out.Neither)
}
}
// 1945/AC21: a node whose have names an old-form build (no fingerprint of the new form, so no composition of it) can never be
// named in a want at what it runs; its first want names F, under the canary rule as usual and with everything F needs and the
// providers rule. Given the canary runs F healthy, shanks's first want names F; given it does not yet, shanks gets no want and
// stays on neither list. The old-form node that is itself the canary gets F once its providers run healthy.
func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) {
oldForm := func(n Nodes, node string) {
n.Haves[node]["screen-lock"] = delivery.Have{FP: "build-1792260876956849831", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
}
providersHealthy := func(n Nodes) {
for _, node := range []string{"g14", "shanks"} {
n.Haves[node]["xorg"] = delivery.Have{FP: fpOf('x'), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
}
}
// the canary g14 runs F healthy: shanks's first want names F
m, n := x11()
providersHealthy(n)
oldForm(n, "shanks")
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
out := Wants(m, n, nil, nil)
if w := out.Wants["shanks"]; w.FP != fNew || w.Composed != dig(fNew) || w.Newest != "" || w.Rule != 0 {
t.Errorf("canary healthy on F: shanks's want = %+v, want F", w)
}
if _, ok := out.Neither["shanks"]; ok {
t.Errorf("canary healthy on F: shanks still on neither list: %+v", out.Neither)
}
// the canary does not run F healthy yet: no want, and shanks stays on neither list by rule 3
m, n = x11()
providersHealthy(n)
oldForm(n, "shanks")
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
out = Wants(m, n, nil, nil)
if w, ok := out.Wants["shanks"]; ok {
t.Errorf("canary not healthy on F: shanks got a want: %+v", w)
}
if h, ok := out.Neither["shanks"]; !ok || h.Rule != delivery.RuleCanary {
t.Errorf("canary not healthy on F: shanks on neither list = %+v (named %v), want rule 3", h, ok)
}
// the canary healthy on F but the providers not healthy on shanks: rule 5 still holds it back
m, n = x11()
oldForm(n, "shanks")
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
out = Wants(m, n, nil, nil)
if w, ok := out.Wants["shanks"]; ok {
t.Errorf("providers not healthy: shanks got a want: %+v", w)
}
if h := out.Neither["shanks"]; h.Rule != delivery.RuleProviders {
t.Errorf("providers not healthy: shanks on neither list = %+v, want rule 5", h)
}
// F not granted: nothing, even with the canary healthy
m, n = x11()
providersHealthy(n)
oldForm(n, "shanks")
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
m.Granted = false
if w, ok := Wants(m, n, nil, nil).Wants["shanks"]; ok {
t.Errorf("F not granted: shanks got a want: %+v", w)
}
// the old-form node is the canary itself: F once its providers run healthy
m, n = x11()
providersHealthy(n)
oldForm(n, "g14")
if w := Wants(m, n, nil, nil).Wants["g14"]; w.FP != fNew || w.Composed != dig(fNew) {
t.Errorf("old-form canary: g14's want = %+v, want F", w)
}
}
// A failed health counts only once the node is done moving: phase running or failed, never backing-up, applying or checking.
// The live row on novox at 15:27:36 on 2026-10-25 ("health failed, phase applying" at the controller's new fingerprint) held the
// canary two seconds before the new build was healthy.
func TestAFailedHealthWhileTheNodeMovesHoldsNothing(t *testing.T) {
for _, phase := range []delivery.Phase{delivery.PhaseBackingUp, delivery.PhaseApplying, delivery.PhaseChecking} {
m := Module{Name: "mesh-controller", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"novox"},
Manifest: catalogue.Manifest{Module: "mesh-controller"}}
n := Nodes{Online: map[string]bool{"novox": true}, Haves: map[string]map[string]delivery.Have{
"novox": {"mesh-controller": {FP: fNew, Phase: phase, Health: delivery.HealthFailed}}}}
before := map[string]delivery.Want{"novox": {FP: fNew, Composed: dig(fNew)}}
if out := Wants(m, n, before, map[string]string{"novox": fOld}); out.Held != nil {
t.Errorf("phase %s, health failed: held %+v, want no hold while the node moves", phase, out.Held)
}
}
for _, phase := range []delivery.Phase{delivery.PhaseRunning, delivery.PhaseFailed} {
m := Module{Name: "mesh-controller", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"novox"},
Manifest: catalogue.Manifest{Module: "mesh-controller"}}
n := Nodes{Online: map[string]bool{"novox": true}, Haves: map[string]map[string]delivery.Have{
"novox": {"mesh-controller": {FP: fNew, Phase: phase, Health: delivery.HealthFailed}}}}
before := map[string]delivery.Want{"novox": {FP: fNew, Composed: dig(fNew)}}
if out := Wants(m, n, before, map[string]string{"novox": fOld}); out.Held == nil || out.Held.Node != "novox" {
t.Errorf("phase %s, health failed: held %+v, want a hold on novox", phase, out.Held)
}
}
}
// A hold is lifted by itself once every node that runs the module runs F healthy: a have of a node that is not running the module
// (removed, held for its backup, or still moving) is not counted, so it does not keep the hold up for ever; a node running another
// fingerprint, or failed, still does.
func TestAHoldIsLiftedOverHavesThatAreNotRunning(t *testing.T) {
setup := func(phase delivery.Phase, fp string, health delivery.Health) (Module, Nodes) {
m := Module{Name: "web", FP: fNew, Built: true, Granted: true, Held: true, Composed: composedEverywhere, Nodes: []string{"ace", "g14"}}
n := Nodes{Online: map[string]bool{"ace": true, "g14": true}, Haves: map[string]map[string]delivery.Have{
"ace": {"web": {FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}},
"g14": {"web": {FP: fp, Phase: phase, Health: health}}}}
return m, n
}
for _, phase := range []delivery.Phase{delivery.PhaseRemoved, delivery.PhaseHeld, delivery.PhaseApplying, delivery.PhaseBackingUp, delivery.PhaseChecking} {
m, n := setup(phase, fOld, delivery.HealthUnknown)
if out := Wants(m, n, nil, nil); !out.Lifted {
t.Errorf("g14's have in phase %s kept the hold up", phase)
}
}
for what, h := range map[string]delivery.Have{
"running another fingerprint": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK},
"failed": {FP: fNew, Phase: delivery.PhaseFailed, Health: delivery.HealthFailed},
} {
m, n := setup(h.Phase, h.FP, h.Health)
if out := Wants(m, n, nil, nil); out.Lifted {
t.Errorf("g14 %s: the hold was lifted", what)
}
}
// a failed health on a removed have holds nothing
m, n := setup(delivery.PhaseRemoved, fNew, delivery.HealthFailed)
m.Held = false
if out := Wants(m, n, nil, nil); out.Held != nil {
t.Errorf("a removed have with a failed health held F: %+v", out.Held)
}
}
// A module of policy record whose want was removed but which the node still runs gets its first want through the record ask: a
// removed want is no want, so a move the controller judges to recreate or move data waits for the operator's yes.
func TestAFirstWantAfterARemovedWantGoesThroughTheRecordAsk(t *testing.T) {
m, n := x11()
recording(&m)
m.Composed = func(_, fp string) (delivery.ComposedDigest, bool) {
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedRecreates, Policy: delivery.PolicyRecord}, true
}
out := Wants(m, n, map[string]delivery.Want{"shanks": {Removed: true}}, nil)
if w, ok := out.Wants["shanks"]; ok {
t.Errorf("shanks's first want after a removed want was written with no record ask: %+v", w)
}
asked := false
for _, r := range out.Record {
asked = asked || (r.Node == "shanks" && r.Digest == dig(fOld) && r.Judgement == delivery.JudgedRecreates)
}
if !asked {
t.Errorf("the record asks = %+v, want one for shanks at the composition it runs", out.Record)
}
}
// 1945/AC23: given a module held at F under rule 2 (the operator holds systemd-resolved on shanks, 2026-10-25), or held by the
// assigner in this pass after a failed health check of F, when a node that runs it holds no want for it, then the node is not on
// neither list: a held module is on the held list, whose hold is said and answered on its own.
func TestAHeldModuleWithNoWantIsNotOnNeitherList(t *testing.T) {
for what, change := range map[string]func(*Module, *Nodes){
"held at F": func(m *Module, _ *Nodes) { m.Held, m.Kept = true, true },
"held at F, running an old-form build": func(m *Module, n *Nodes) {
m.Held, m.Kept = true, true
n.Haves["shanks"]["screen-lock"] = delivery.Have{FP: "build-2", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
},
"held by this pass": func(_ *Module, n *Nodes) {
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
n.Haves["shanks"]["screen-lock"] = delivery.Have{FP: "build-2", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
},
} {
m, n := x11()
change(&m, &n)
out := Wants(m, n, nil, nil)
if len(out.Neither) != 0 {
t.Errorf("%s: a held module is on neither list: %+v", what, out.Neither)
}
if out.Rules["shanks"].Rule != delivery.RuleNotHeld {
t.Errorf("%s: shanks is held back by %+v, want rule 2", what, out.Rules["shanks"])
}
}
}
@@ -24,7 +24,7 @@ func load(t *testing.T) catalogue.Manifest {
// 1297/B2a: the controller's reader (mesh-sdk catalogue, which the controller reads manifests with) takes the
// manifest; it reads exactly the buckets the assigner opens (the program's list, held equal in its own test), keeps
// the states it declares (the differences kept read-only, 1906/D5), and holds a credential of its own.
// its four states (the differences kept read-only, 1906/D5), and holds a credential of its own.
func TestManifest(t *testing.T) {
m := load(t)
if m.Module != "mesh-delivery-assigner" {
@@ -42,7 +42,7 @@ func TestManifest(t *testing.T) {
for _, s := range m.State {
states[s.Name] = s.History
}
if want := map[string]int{"asks": 5, "compositions": 5, "differences": 64, "puts": 20, "wants": 20}; !reflect.DeepEqual(states, want) {
if want := map[string]int{"asks": 5, "compositions": 5, "differences": 64, "wants": 20}; !reflect.DeepEqual(states, want) {
t.Errorf("state = %v, want %v", states, want)
}
if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 {
@@ -56,16 +56,7 @@ func TestManifest(t *testing.T) {
if !slices.Equal(m.Uses, []string{"node-build-go", "operator-channel"}) {
t.Errorf("uses = %v, want exactly node-build-go and operator-channel", m.Uses)
}
// ADR 0546: it serves its own tools (put, release, puts) from its bundle through the tool runner (the artifact loads the
// module), claims no seat (a new seat needs a change of the controller), says each put's step as the event put, and
// calls the controller's warranted and the bus node's backup holder.
if len(m.Claims) != 0 || !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts"}) {
t.Errorf("the assigner claims %v and serves tools %v, want no claim and its three put tools", m.Claims, m.Tools)
}
if want := []string{"seat:mesh-controller.warranted", "seat:node-backup.backed-up", "seat:node-backup.now"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) {
t.Errorf("invokes = %v, want %v", m.Invokes, want)
}
if !slices.Equal(m.Emits, []string{"put"}) {
t.Errorf("emits = %v, want put", m.Emits)
if len(m.Claims) != 0 || len(m.Tools) != 0 || len(m.Invokes) != 0 {
t.Errorf("the assigner claims, serves or calls something: claims %v, tools %v, invokes %v", m.Claims, m.Tools, m.Invokes)
}
}
+1 -18
View File
@@ -20,24 +20,10 @@
"node-build-go",
"operator-channel"
],
"invokes": [
"seat:mesh-controller.warranted",
"seat:node-backup.now",
"seat:node-backup.backed-up"
],
"emits": [
"put"
],
"tools": [
"assigner_put",
"assigner_release",
"assigner_puts"
],
"state": [
{ "name": "asks", "history": 5 },
{ "name": "compositions", "history": 5 },
{ "name": "differences", "history": 64 },
{ "name": "puts", "history": 20 },
{ "name": "wants", "history": 20 }
],
"own-secrets": {
@@ -85,10 +71,7 @@
"language": "go",
"system": "arch",
"from": "cmd/mesh-delivery-assigner",
"binary": "mesh-delivery-assigner",
"loads": [
"mesh-delivery-assigner"
]
"binary": "mesh-delivery-assigner"
}
]
}
@@ -1,178 +0,0 @@
// Command mesh-delivery-builder-tools serves the builder's tools (novox/hq issue 1945): the runtime launches it for a call.
// builder_stuck reads the builder's builds and reasks states and the build modules' steps through the builder's state and
// reads; builder_ask_again writes one value of the reasks state, which the builder watches and answers. The builder is the
// other program: it serves no tool (ADR 0437 §4).
package main
import (
"context"
"encoding/json"
"fmt"
"os"
"strings"
"sync"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-builder/internal/build"
)
// queueAsker writes a queues value and waits for the builder's answer to it.
type queueAsker func(key string, value []byte) (json.RawMessage, error)
// Tools are the builder's tools: stuck reads, askAgain writes the reasks value, queue writes a queues value and waits for
// the builder's read.
func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue queueAsker, now func() time.Time) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
return []stdio.Tool{
{Name: build.StuckTool,
Description: "Every build key that waits (building or queued), oldest first: its module, fingerprint, step words and since when, and each ask it " +
"waits on by language with its id, when it was put, its asking (1 for the first; the builder asks an ask again when it can no longer be " +
"answered, at most 3 times) and what its build module's steps hold of it (none, or its state on its node since its start). The " +
"seat workers' delivery counts are not readable by the builder. With module, that module's keys only. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "only this module's keys"}}},
Run: func(args map[string]any) (any, error) { return r.Stuck(str(args, "module")) }},
{Name: build.AskAgainTool,
Description: "Ask a waiting build key's asks again now, under new ids the key then names, as the builder does by itself for an ask its build " +
"seat's stream gave up on. The builder answers in the reasks state, which " + build.StuckTool + " shows beside the key; a key no ask of " +
"which waits is answered so and nothing is asked. An ask whose step already runs on a node is asked again too: it is then " +
"built twice, the first run's work wasted and its end read by nobody, never a wrong key; builder_stuck shows whether a step runs.",
Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "the module's name"},
"fp": map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>, as builder_stuck shows it"},
"why": map[string]any{"type": "string", "description": "why it is asked again, in plain words"}}},
Run: func(args map[string]any) (any, error) {
key, value, err := build.NewReAsk(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
if err != nil {
return nil, err
}
if err := askAgain(key, value); err != nil {
return nil, fmt.Errorf("the ask could not be written to %s: %w", build.ReAsksBucket, err)
}
return map[string]any{"key": key, "asked": true,
"next": "the builder answers in " + build.ReAsksState + "." + key + "; " + build.StuckTool + " shows the answer and the new ask"}, nil
}},
{Name: build.SeatQueueTool,
Description: "What waits on a build seat's stream (SEAT_NODE_BUILD_<LANGUAGE>), as the builder reads it: the stream's message count and " +
"sequences, and per kind (delivery, check, recheck) its messages oldest first, at most limit each, with sequence, ask id, when asked " +
"and put, age, the state of the step a build node wrote for it, and whether the builder counts it as still waiting or as given up " +
"on (its worker delivered it its most times) or taken; and the ask asked last that a node took. The workers' delivery counts are " +
"not readable by the builder (nats.nats_streams shows them). Read by the builder, which alone may read the stream; read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"language": map[string]any{"type": "string", "description": "go, node or image (default go)"},
"seat": map[string]any{"type": "string", "description": "or the seat: node-build-go, node-build-node, node-build-image"},
"limit": map[string]any{"type": "integer", "description": fmt.Sprintf("messages read of each kind (default %d, at most %d)", build.DefaultQueueLimit, build.MaxQueueLimit)}}},
Run: func(args map[string]any) (any, error) {
language := build.LanguageOf(str(args, "language"))
if language == "" {
language = build.LanguageOf(str(args, "seat"))
}
if language == "" {
language = delivery.AskGo
}
limit, _ := args["limit"].(float64)
key, value, err := build.NewQueueAsk(language, int(limit), stdio.CallerOf(args), now())
if err != nil {
return nil, err
}
return queue(key, value)
}},
}
}
// askQueue writes the queues value at key and waits, on a watch of the key, at most bound for the builder's answer.
func askQueue(s stdio.KeptState, bound time.Duration) queueAsker {
return func(key string, value []byte) (json.RawMessage, error) {
answered := make(chan json.RawMessage, 1)
var rev uint64
var mu sync.Mutex
if err := s.Watch(key, func(c stdio.StateChange) error {
mu.Lock()
mine := rev
mu.Unlock()
if c.Op != "put" || mine == 0 || c.Revision <= mine {
return nil
}
if a, err := build.DecodeQueueAsk(c.Value); err == nil && !a.Answered.IsZero() {
select {
case answered <- c.Value:
default:
}
}
return nil
}); err != nil {
return nil, fmt.Errorf("watching %s.%s: %w", build.QueuesState, key, err)
}
r, err := s.Put(key, json.RawMessage(value))
if err != nil {
return nil, fmt.Errorf("the read could not be written to %s: %w", build.QueuesBucket, err)
}
mu.Lock()
rev = r
mu.Unlock()
// The watch may have heard the answer before the revision was known: read it once too.
if e, err := s.Get(key); err == nil && e != nil && e.Revision > r {
if a, err := build.DecodeQueueAsk(e.Value); err == nil && !a.Answered.IsZero() {
return e.Value, nil
}
}
wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8)
defer cancel()
select {
case v := <-answered:
return v, nil
case <-wait.Done():
return nil, fmt.Errorf("the builder did not answer %s.%s within %s: it is not running, or cannot read %s", build.QueuesState, key, bound, build.QueuesBucket)
}
}
}
func read(s stdio.KeptState) build.StateRead {
return build.StateRead{Keys: s.Keys, Get: func(key string) (json.RawMessage, time.Time, error) {
e, err := s.Get(key)
if err != nil || e == nil {
return nil, time.Time{}, err
}
return e.Value, time.Time{}, nil
}, Written: func(key string) time.Time {
e, err := s.Get(key)
if err != nil || e == nil {
return time.Time{}
}
return written(s, key, e.Revision)
}}
}
// written is when the key's revision was written, from its history, or zero.
func written(s stdio.KeptState, key string, revision uint64) time.Time {
h, err := s.History(key)
if err != nil {
return time.Time{}
}
for _, r := range h {
if r.Revision == revision {
return r.At
}
}
return time.Time{}
}
func main() {
steps := map[string]build.StateRead{}
for _, bm := range build.StuckBuildModules {
steps[bm] = read(stdio.State(bm + ".steps"))
}
reasks := stdio.State(build.ReAsksState)
r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps}
put := func(key string, value []byte) error {
_, err := reasks.Put(key, json.RawMessage(value))
return err
}
if err := stdio.Serve("", Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), time.Now)); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err)
os.Exit(1)
}
}
@@ -66,11 +66,6 @@ func settle(ctx context.Context, kv jetstream.KeyValue, each func(jetstream.KeyV
if err != nil {
return nil, err
}
return settled(ctx, w, each)
}
// settled hands each entry the watch delivers before its end-of-initial marker to each, and answers the watch.
func settled(ctx context.Context, w jetstream.KeyWatcher, each func(jetstream.KeyValueEntry)) (jetstream.KeyWatcher, error) {
for {
select {
case <-ctx.Done():
@@ -4,303 +4,73 @@ import (
"context"
"errors"
"fmt"
"sort"
"strings"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-builder/internal/build"
)
// heldOnTheBus says whether an ask a build key names is on its build seat's stream, or a build node wrote a step for it
// (issue 1910), read once at the take-over of a key at start; and, at the builder's check bound, whether an ask can still
// be answered (issue 1945). Never polled.
// (issue 1910): read once, at the take-over of a key at start, never polled.
type heldOnTheBus struct{ js jetstream.JetStream }
// Held is build.Builder's Held: a step key `<node>.<ask id>` in the build module's steps, or a message of the stream under
// the ask's subject whose Nats-Msg-Id is the ask's id.
func (h heldOnTheBus) Held(ctx context.Context, a delivery.StepAsk) (bool, error) {
seen, err := h.seen(ctx, a, false)
module, err := delivery.BuildModuleOf(a.Language)
if err != nil {
return false, err
}
return seen.Step || seen.OnStream, nil
}
// Gone is build.Builder's Gone: why the ask can no longer be answered (build.AskSeen.Why), or "".
func (h heldOnTheBus) Gone(ctx context.Context, a delivery.StepAsk) (string, error) {
seen, err := h.seen(ctx, a, true)
pattern, err := delivery.AskStepWatch(a.ID)
if err != nil {
return "", err
}
return seen.Why(), nil
}
// maxOnSubject bounds the messages of one subject a read walks (each a read by sequence, next by subject), and maxSteps
// the steps it reads whole to find the last ask taken: past them the read says it cannot tell, and the ask is waited for.
const (
maxOnSubject = 5000
maxSteps = 200
)
// seen reads what the bus holds of the ask; with later, the ask asked last that a build node took too (AskSeen.Last), when
// the ask is on its stream.
func (h heldOnTheBus) seen(ctx context.Context, a delivery.StepAsk, later bool) (build.AskSeen, error) {
seen := build.AskSeen{Kind: a.Kind, At: build.AskedAt(a.ID)}
module, err := delivery.BuildModuleOf(a.Language)
if err != nil {
return seen, err
return false, err
}
steps, err := h.js.KeyValue(ctx, delivery.StepsBucket(module))
if err != nil {
return seen, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
}
pattern, err := delivery.AskStepWatch(a.ID)
if err != nil {
return seen, err
return false, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
}
keys, err := steps.ListKeysFiltered(ctx, pattern)
if err != nil {
return seen, err
return false, err
}
for range keys.Keys() {
keys.Stop()
seen.Step = true // a node took it
return seen, nil
}
stream, name, subject, err := h.stream(ctx, a.Language, a.Kind)
seen.Stream = name
if errors.Is(err, jetstream.ErrStreamNotFound) {
return seen, nil
return true, nil // a node took it
}
subject, err := delivery.AskSubject(a.Language, a.Kind)
if err != nil {
return seen, err
}
var ahead []build.Queued
err = onSubject(ctx, stream, subject, func(q build.Queued) bool {
if q.Ask == a.ID {
seen.OnStream, seen.PutAt = true, q.PutAt
return false
}
ahead = append(ahead, q)
return true
})
if err != nil || !seen.OnStream || !later {
return seen, err
}
taken, err := stepKeys(ctx, steps)
if err != nil {
return seen, err
}
for i := range ahead {
ahead[i].Step = stepState(ctx, steps, taken[ahead[i].Ask])
}
seen.Ahead = ahead
since := seen.At
if since.IsZero() {
since = seen.PutAt
}
for _, q := range ahead {
if q.At.Before(since) {
since = q.At
}
}
seen.Last = lastTaken(ctx, steps, taken, a.Kind, since)
return seen, nil
}
// stream is the build seat's stream of language, its name and the subject of kind on it.
func (h heldOnTheBus) stream(ctx context.Context, language, kind string) (jetstream.Stream, string, string, error) {
subject, err := delivery.AskSubject(language, kind)
if err != nil {
return nil, "", "", err
return false, err
}
// The seat's stream by its name (delivery.AskStream), as the broker grants the builder its information and its
// messages by sequence and nothing wider (issue 1910).
name, err := delivery.AskStream(language)
name, err := delivery.AskStream(a.Language)
if err != nil {
return nil, "", "", err
return false, err
}
stream, err := h.js.Stream(ctx, name)
return stream, name, subject, err
}
// onSubject hands each message of subject on stream to each, in order, until each answers false or the subject's
// messages end. A seat's stream is a work queue, which takes no second consumer over its subjects: its messages are read
// by sequence instead, each the next of the subject (one read per message, none for the sequences taken between), and
// none consumed.
func onSubject(ctx context.Context, stream jetstream.Stream, subject string, each func(build.Queued) bool) error {
for seq, n := uint64(1), 0; ; n++ {
if n >= maxOnSubject {
return fmt.Errorf("%s holds more than %d messages before the one asked after: not read further", subject, maxOnSubject)
}
m, err := stream.GetMsg(ctx, seq, jetstream.WithGetMsgSubject(subject))
if errors.Is(err, jetstream.ErrMsgNotFound) {
return nil
}
if err != nil {
return err
}
q := build.Queued{Seq: m.Sequence, Ask: m.Header.Get(jetstream.MsgIDHeader), PutAt: m.Time}
if _, kind, err := delivery.ParseAskSubject(m.Subject); err == nil {
q.Kind = kind
}
if q.At = build.AskedAt(q.Ask); q.At.IsZero() {
q.At = q.PutAt
}
if !each(q) {
return nil
}
seq = m.Sequence + 1
}
}
// stepKeys are the build module's step keys by ask id: `<node>.<ask id>` (a node's own key, such as its pause, has no
// ask id and is left out).
func stepKeys(ctx context.Context, steps jetstream.KeyValue) (map[string][]string, error) {
all, err := steps.ListKeys(ctx)
if err != nil {
if errors.Is(err, jetstream.ErrNoKeysFound) {
return map[string][]string{}, nil
}
return nil, err
}
defer all.Stop()
out := map[string][]string{}
for k := range all.Keys() {
if _, id, ok := strings.Cut(k, "."); ok && !build.AskedAt(id).IsZero() {
out[id] = append(out[id], k)
}
}
return out, nil
}
// stepState is the state of the first of keys a node wrote, "" for none.
func stepState(ctx context.Context, steps jetstream.KeyValue, keys []string) string {
for _, k := range keys {
e, err := steps.Get(ctx, k)
if err != nil {
continue
}
if s, err := delivery.DecodeStep(e.Value()); err == nil {
return s.State
}
}
return ""
}
// lastTaken is the ask asked last after since that a build node took, of a kind whose taking proves kind passed
// (build.Proves), read newest first and at most maxSteps of them; nil for none. A step's tool is "<kind> ask"
// (mesh-sdk buildstep).
func lastTaken(ctx context.Context, steps jetstream.KeyValue, taken map[string][]string, kind string, since time.Time) *build.Taken {
var ids []string
for id := range taken {
if build.AskedAt(id).After(since) {
ids = append(ids, id)
}
}
sort.Slice(ids, func(i, j int) bool { return build.AskedAt(ids[i]).After(build.AskedAt(ids[j])) })
read := 0
for _, id := range ids {
for _, k := range taken[id] {
if read >= maxSteps {
return nil
}
read++
e, err := steps.Get(ctx, k)
if err != nil {
continue
}
s, err := delivery.DecodeStep(e.Value())
if err != nil {
continue
}
stepKind, ok := strings.CutSuffix(s.Tool, " ask")
if !ok || !build.Proves(stepKind, kind) {
continue
}
return &build.Taken{Ask: id, Kind: stepKind, At: build.AskedAt(id), Node: s.Node, Started: s.Started}
}
}
return nil
}
// Drop is build.Dropper's Drop: it removes the ask's message from its build seat's stream, found by its Nats-Msg-Id on its
// subject, when the stream still holds it. The builder's account must be granted $JS.API.STREAM.MSG.DELETE.<stream>.
func (h heldOnTheBus) Drop(ctx context.Context, a delivery.StepAsk) error {
stream, name, subject, err := h.stream(ctx, a.Language, a.Kind)
if errors.Is(err, jetstream.ErrStreamNotFound) {
return nil
return false, nil
}
if err != nil {
return err
}
var seq uint64
if err := onSubject(ctx, stream, subject, func(q build.Queued) bool {
if q.Ask == a.ID {
seq = q.Seq
return false
}
return true
}); err != nil || seq == 0 {
return err
}
if err := stream.DeleteMsg(ctx, seq); err != nil && !errors.Is(err, jetstream.ErrMsgNotFound) {
return fmt.Errorf("deleting message %d of %s (is $JS.API.STREAM.MSG.DELETE.%s granted to the builder?): %w", seq, name, name, err)
}
return nil
}
// Queue is what the tool builder_seat_queue answers (build.SeatQueue): the messages of language's build seat's stream, of
// every kind, oldest first and at most limit of each kind, each with its step's state and whether it still waits.
func (h heldOnTheBus) Queue(ctx context.Context, language string, limit int) (build.SeatQueue, error) {
q := build.SeatQueue{Language: language, Note: build.QueueNote}
module, err := delivery.BuildModuleOf(language)
if err != nil {
return q, err
}
stream, name, _, err := h.stream(ctx, language, delivery.AskDelivery)
q.Stream = name
if errors.Is(err, jetstream.ErrStreamNotFound) {
q.Note = name + " does not exist"
return q, nil
}
if err != nil {
return q, err
return false, err
}
// A seat's stream is a work queue, which takes no second consumer over its subjects: its messages are read by sequence
// instead, each a direct read that consumes nothing.
info, err := stream.Info(ctx)
if err != nil {
return q, err
return false, err
}
q.Messages, q.FirstSeq, q.LastSeq = info.State.Msgs, info.State.FirstSeq, info.State.LastSeq
steps, err := h.js.KeyValue(ctx, delivery.StepsBucket(module))
if err != nil {
return q, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
}
taken, err := stepKeys(ctx, steps)
if err != nil {
return q, err
}
for _, kind := range delivery.AskKinds {
subject, _ := delivery.AskSubject(language, kind)
var ahead []build.Queued
if err := onSubject(ctx, stream, subject, func(m build.Queued) bool {
ahead = append(ahead, m)
return len(ahead) < limit
}); err != nil {
return q, err
for seq := info.State.FirstSeq; seq != 0 && seq <= info.State.LastSeq; seq++ {
m, err := stream.GetMsg(ctx, seq)
if errors.Is(err, jetstream.ErrMsgNotFound) {
continue // taken and acknowledged, or removed
}
if len(ahead) == 0 {
continue
if err != nil {
return false, err
}
for i := range ahead {
ahead[i].Step = stepState(ctx, steps, taken[ahead[i].Ask])
if m.Subject == subject && m.Header.Get(jetstream.MsgIDHeader) == a.ID {
return true, nil
}
seen := build.AskSeen{Kind: kind, Ahead: ahead, Last: lastTaken(ctx, steps, taken, kind, ahead[0].At)}
q.Kinds = append(q.Kinds, build.NewKindQueue(kind, seen, time.Now()))
}
return q, nil
return false, nil
}
@@ -24,8 +24,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-builder/internal/build"
)
// busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the
@@ -164,29 +162,11 @@ func (f *fakeBuildModules) serve(t *testing.T, module string, nodes ...string) {
}
}
// trunkHead is the trunk head of novox/mesh-catalog the bed's source revision names: newer than the module values' commit.
const trunkHead = "3333333333333333333333333333333333333333"
// trunk writes the gate's source.main naming head as novox/mesh-catalog's trunk head, and answers its revision.
func (b *bed) trunk(t *testing.T, head string) uint64 {
t.Helper()
data, err := delivery.EncodeSource(delivery.Source{V: delivery.Version, Group: "g", Heads: map[string]string{"novox/mesh-catalog": head}})
if err != nil {
t.Fatal(err)
}
rev, err := b.source.Put(b.ctx, delivery.SourceMainKey, data)
if err != nil {
t.Fatal(err)
}
return rev
}
type bed struct {
ctx context.Context
js jetstream.JetStream
modules jetstream.KeyValue
candidates jetstream.KeyValue
source jetstream.KeyValue
builds jetstream.KeyValue
tools map[string]jetstream.KeyValue
instances jetstream.KeyValue
@@ -263,7 +243,6 @@ func startBedWith(t *testing.T, closed bool) *bed {
}
b.modules = mk(delivery.ModulesBucket, delivery.ModulesHistory)
b.candidates = mk(delivery.CandidatesBucket, delivery.CandidatesHistory)
b.source = mk(delivery.SourceBucket, delivery.SourceHistory)
mk(delivery.ManifestsBucket, delivery.ManifestsHistory)
b.builds = mk(delivery.BuilderBuildsBucket, delivery.BuilderBuildsHistory)
b.instances = mk(instancesBucket, 1)
@@ -534,9 +513,7 @@ func TestAKeyLeftBuildingForAModuleThatIsGoneIsClosedAtStart(t *testing.T) {
// TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk: the trunk's record of netcheck carries a fingerprint of its
// own; a candidate the indexer wrote gives netcheck the fingerprint fp at a pull request's head; the program writes
// builds.netcheck.<fp> done, its fetch given the head, the ref that carries it and the trunk head of the candidate's source
// revision, newer than the module value's commit (issue 1684; the operator 2026-10-25: a value names the commit its
// fingerprint first appeared at).
// builds.netcheck.<fp> done, its fetch given the head, the ref that carries it and the trunk commit (issue 1684).
func TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk(t *testing.T) { // N7/AC1
b := startBed(t)
other := "src2:" + strings.Repeat("e", 64)
@@ -553,7 +530,7 @@ func TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk(t *tes
pulls := []delivery.CandidatePull{{Head: delivery.Head{Repository: "novox/mesh-catalog", Number: 7, Head: head}, Modules: []string{"netcheck"},
Fps: map[string]delivery.CandidateModule{"netcheck": {FP2: fp, Manifest: digest}}}}
hash, _ := delivery.HeadsHash([]delivery.Head{pulls[0].Head})
cand, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", Base: b.trunk(t, trunkHead), HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
cand, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
if err != nil {
t.Fatal(err)
}
@@ -573,12 +550,12 @@ func TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk(t *tes
defer b.fake.mu.Unlock()
found := false
for _, f := range b.fake.fetches {
if f["commit"] == head && f["ref"] == "refs/pull/7/head" && f["base"] == trunkHead {
if f["commit"] == head && f["ref"] == "refs/pull/7/head" && f["base"] == commit {
found = true
}
}
if !found {
t.Errorf("fetches = %v, want one of the candidate's head with its ref and the trunk head it was previewed onto, not the module value's commit", b.fake.fetches)
t.Errorf("fetches = %v, want one of the candidate's head with its ref and the trunk commit", b.fake.fetches)
}
}
@@ -626,7 +603,7 @@ func TestAnImageIsBuiltOnOneNodeAndOnlyATrunkKeyPublishes(t *testing.T) { // 170
pulls := []delivery.CandidatePull{{Head: delivery.Head{Repository: "novox/mesh-catalog", Number: 7, Head: head}, Modules: []string{"imaged"},
Fps: map[string]delivery.CandidateModule{"imaged": {FP2: fp, Manifest: digest}}}}
hash, _ := delivery.HeadsHash([]delivery.Head{pulls[0].Head})
cand, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", Base: b.trunk(t, trunkHead), HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
cand, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
if err != nil {
t.Fatal(err)
}
@@ -824,34 +801,6 @@ func TestARestartedBuilderPutsNoSecondAsk(t *testing.T) {
t.Fatalf("asks put = %+v within 20 s, want the two the key names", asked())
}
}
// 1945 note 62, on the real bus: the stream read as builder_seat_queue reads it shows the Go ask waiting; the ask can
// still be answered (nothing was taken since); and a message dropped as an ask asked again is gone from the stream.
held := heldOnTheBus{b.js}
goAsk := delivery.StepAsk{ID: v.Asks[delivery.AskGo], Language: delivery.AskGo, Kind: delivery.AskDelivery}
stale := delivery.StepAsk{ID: "mvo53jcm70b5f7", Language: delivery.AskGo, Kind: delivery.AskDelivery}
subject, _ := delivery.AskSubject(delivery.AskGo, delivery.AskDelivery)
if _, err := b.js.Publish(b.ctx, subject, []byte("{}"), jetstream.WithMsgID(stale.ID)); err != nil {
t.Fatal(err)
}
queued := func() []build.QueuedState {
q, err := held.Queue(b.ctx, delivery.AskGo, 10)
if err != nil || len(q.Kinds) != 1 || q.Kinds[0].Kind != delivery.AskDelivery {
t.Fatalf("the Go seat's queue = %+v, %v; want its delivery asks", q, err)
}
return q.Kinds[0].Messages
}
if m := queued(); len(m) != 2 || m[0].Ask != goAsk.ID || !m[0].Waits || m[1].Ask != stale.ID {
t.Fatalf("the Go delivery queue = %+v, want the key's ask waiting, then the stale one", m)
}
if why, err := held.Gone(b.ctx, goAsk); why != "" || err != nil {
t.Errorf("Gone(the waiting ask) = %q, %v; want it answerable", why, err)
}
if err := held.Drop(b.ctx, stale); err != nil {
t.Fatal(err)
}
if m := queued(); len(m) != 1 || m[0].Ask != goAsk.ID {
t.Fatalf("after the drop the Go delivery queue = %+v, want the key's ask alone", m)
}
b.stop()
b.start(t, "n1", nil) // the same builder, started again, while both asks wait
b.fake.open()
@@ -13,19 +13,15 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"io"
"log"
"maps"
"os"
"os/signal"
"slices"
"strings"
"sync"
"syscall"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
@@ -125,20 +121,29 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil {
return err
}
builds, err := process.EssentialBucket(ctx, js, "builder", delivery.BuilderBuildsBucket, "claim or write a build key")
bucket := func(name string) (jetstream.KeyValue, error) {
kv, err := js.KeyValue(ctx, name)
if err != nil {
return nil, fmt.Errorf("the bucket %s: %w", name, err)
}
return kv, nil
}
builds, err := bucket(delivery.BuilderBuildsBucket)
if err != nil {
return err
}
modules, err := process.EssentialBucket(ctx, js, "builder", delivery.ModulesBucket, "know any fingerprint to build")
modules, err := bucket(delivery.ModulesBucket)
if err != nil {
return err
}
manifests, err := process.EssentialBucket(ctx, js, "builder", delivery.ManifestsBucket, "read what a module builds")
manifests, err := bucket(delivery.ManifestsBucket)
if err != nil {
return err
}
candidates, err := bucket(delivery.CandidatesBucket)
if err != nil {
return err
}
// The candidates' fingerprints are built too (issue 1684): a builder that cannot read them builds the trunk's.
candidates := process.OptionalBucket(ctx, js, delivery.CandidatesBucket, "no candidate's fingerprint is built until the builder starts again; the trunk's are", logger.Printf)
bound := defaultStepBound
if v := strings.TrimSpace(getenv("MESH_BUILDER_STEP_BOUND")); v != "" {
if bound, err = time.ParseDuration(v); err != nil {
@@ -161,36 +166,23 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
defer calls.Close()
b := &build.Builder{
Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
Asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: bound},
Held: heldOnTheBus{js},
// An ask that can no longer be answered is asked again (issue 1945), read at the step bound: a context's bound.
Gone: heldOnTheBus{js},
CheckAfter: bound,
Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
Asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: bound},
Held: heldOnTheBus{js},
Toolchains: tools,
Manifests: manifestReader{manifests},
Logf: logger.Printf,
Stored: storedVia(calls.Call),
}
// A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base
// revision (the gate's source state). The source's revisions are watched with their history (a direct get of one revision
// is not what a read grants): a revision heard after the candidate that names it builds that candidate then. Where it
// cannot be read the trunk's keys are built and the candidates wait (issue 1945).
errs := make(chan error, 6)
trunks := &trunkSource{ctx: ctx, js: js, heads: &sourceHeads{}, moved: b.TrunkMoved, errs: errs, logf: logger.Printf}
trunks.start()
b.Trunk = trunks.head
b.Said = func(e build.AskedAgain) {
if err := emitAskedAgain(context.WithoutCancel(ctx), js, cred, e); err != nil {
logger.Printf("%v", err)
}
}
// **Said once every build module answered a reference, taken away when stopped** (issue 1886, ADR 0522): the
// controller delivers from this builder's keys only while an instance says it writes references. A bucket not there
// yet (the manifest's state not made) is said, and the controller's plan waits for the keys, named (no build-agent
// is asked since R4g, ADR 0524 §7).
switch kv := process.OptionalBucket(ctx, js, instancesBucket, "the controller is not told that this builder writes references until it starts again", logger.Printf); {
case kv == nil:
switch kv, err := js.KeyValue(ctx, instancesBucket); {
case errors.Is(err, jetstream.ErrBucketNotFound):
logger.Printf("the bucket %s does not exist yet: the controller is not told that this builder writes references until it starts again", instancesBucket)
case err != nil:
return fmt.Errorf("the bucket %s: %w", instancesBucket, err)
default:
says := newSayer(ctx, kv, cred.Node, []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, build.ImageBuildModule}, logger.Printf)
b.Answered = says.Answered
@@ -270,36 +262,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
b.ForgetCandidate(e.Key())
}
}
if candidates != nil {
cw, err := settle(ctx, candidates, eachCandidate)
if err != nil {
process.NotWatched(logger.Printf, delivery.CandidatesBucket, err, "no candidate's fingerprint is built until the builder starts again")
} else {
go func() { errs <- follow(ctx, cw, eachCandidate) }()
}
}
// A person asks a key's build again with builder_ask_again, which writes the reasks state (issue 1945): watched, as
// every other input. The builder asks again by itself without it.
if kv := process.OptionalBucket(ctx, js, build.ReAsksBucket, "builder_ask_again is not heard until the builder starts again; an ask that can no longer be answered is still asked again by itself", logger.Printf); kv != nil {
eachReAsk := func(e jetstream.KeyValueEntry) { reAsked(ctx, b, kv, e, logger.Printf) }
if rw, err := settle(ctx, kv, eachReAsk); err != nil {
process.NotWatched(logger.Printf, build.ReAsksBucket, err, "builder_ask_again is not heard until the builder starts again")
} else {
go func() { errs <- follow(ctx, rw, eachReAsk) }()
}
}
// A person reads a build seat's stream with builder_seat_queue, which writes the queues state (issue 1945 note 62): the
// builder alone may read the stream, so it answers there.
if kv := process.OptionalBucket(ctx, js, build.QueuesBucket, "builder_seat_queue is not answered until the builder starts again", logger.Printf); kv != nil {
eachRead := func(e jetstream.KeyValueEntry) { queueRead(ctx, heldOnTheBus{js}, kv, e, logger.Printf) }
if qw, err := settle(ctx, kv, eachRead); err != nil {
process.NotWatched(logger.Printf, build.QueuesBucket, err, "builder_seat_queue is not answered until the builder starts again")
} else {
go func() { errs <- follow(ctx, qw, eachRead) }()
}
cw, err := settle(ctx, candidates, eachCandidate)
if err != nil {
return err
}
logger.Printf("started on %s", cred.Node)
errs := make(chan error, 2)
go func() { errs <- follow(ctx, w, each) }()
go func() { errs <- follow(ctx, cw, eachCandidate) }()
select {
case <-ctx.Done():
b.Wait()
@@ -312,50 +282,3 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return err
}
}
// sourceHeads are the trunk heads of each revision of the gate's source.main the watch delivered, the newest
// delivery.SourceHistory of them: what a candidate's base revision names.
type sourceHeads struct {
mu sync.Mutex
byRev map[uint64]map[string]string
}
// put takes one revision in and answers whether it is new.
func (s *sourceHeads) put(e jetstream.KeyValueEntry) bool {
if !opOf(e) {
return false
}
src, err := delivery.DecodeSource(e.Value())
if err != nil {
return false
}
s.mu.Lock()
defer s.mu.Unlock()
if s.byRev == nil {
s.byRev = map[uint64]map[string]string{}
}
if _, had := s.byRev[e.Revision()]; had {
return false
}
s.byRev[e.Revision()] = src.Heads
for len(s.byRev) > delivery.SourceHistory {
delete(s.byRev, slices.Min(slices.Collect(maps.Keys(s.byRev))))
}
return true
}
// head is the head of repository in revision rev; an error when the revision is not heard (yet, or any more).
func (s *sourceHeads) head(_ context.Context, rev uint64, repository string) (string, error) {
s.mu.Lock()
defer s.mu.Unlock()
heads, ok := s.byRev[rev]
if !ok {
return "", fmt.Errorf("source revision %d is not heard (it is built once it is)", rev)
}
for r, head := range heads {
if delivery.SameRepository(r, repository) {
return head, nil
}
}
return "", nil
}
@@ -1,78 +0,0 @@
package main
import (
"context"
"fmt"
"sync"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// **A part does not stop for a bucket it can work without** (mesh-sdk process.EssentialBucket, OptionalBucket) (novox/hq issue 1945, met twice on 2026-10-25: the builder of
// mesh-delivery #48 exited on a refused read of the gate's source, and its reasks bucket would have done the same). A grant
// reaches the bus with a delivery, so a bucket the bus refuses, or times out on, at start is often only not granted yet:
// exiting for it stops every build until a person's hand. A bucket the builder cannot build at all without (its builds,
// the indexer's modules and manifests) still stops the start, naming the grant to look at; every other is said and left,
// and opened again at the next event that needs it where there is one.
// trunkSource is the gate's source history the candidates' previews are merged onto (sourceHeads), opened at start and,
// while it cannot be, again at the next candidate that asks a trunk head: a builder without it builds the trunk's keys,
// and a candidate's preview waits, saying why.
type trunkSource struct {
ctx context.Context
js bucketOpener
heads *sourceHeads
moved func(context.Context)
errs chan<- error
logf func(string, ...any)
mu sync.Mutex
opened bool
}
// ensure opens and settles the source's watch once, and follows it; an error leaves it closed for the next ask.
func (t *trunkSource) ensure() error {
t.mu.Lock()
defer t.mu.Unlock()
if t.opened {
return nil
}
kv, err := t.js.KeyValue(t.ctx, delivery.SourceBucket)
if err != nil {
return fmt.Errorf("the bucket %s cannot be read (%v)", delivery.SourceBucket, err)
}
w, err := kv.Watch(t.ctx, delivery.SourceMainKey, jetstream.IncludeHistory())
if err != nil {
return fmt.Errorf("the bucket %s cannot be watched (%v)", delivery.SourceBucket, err)
}
sw, err := settled(t.ctx, w, func(e jetstream.KeyValueEntry) { t.heads.put(e) })
if err != nil {
return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err)
}
t.opened = true
go func() {
t.errs <- follow(t.ctx, sw, func(e jetstream.KeyValueEntry) {
if t.heads.put(e) && t.moved != nil {
t.moved(t.ctx)
}
})
}()
return nil
}
// start opens the source at start; one that cannot be is said, and the builder goes on.
func (t *trunkSource) start() {
if err := t.ensure(); err != nil {
t.logf("%v: the trunk's keys are built, and a candidate's preview waits until the source can be read, tried again at its next candidate", err)
}
}
// head is Builder.Trunk: the head of repository at source revision rev, opening the source first if it is not yet.
func (t *trunkSource) head(ctx context.Context, rev uint64, repository string) (string, error) {
if err := t.ensure(); err != nil {
return "", err
}
return t.heads.head(ctx, rev, repository)
}
@@ -1,49 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"time"
"github.com/nats-io/nats.go/jetstream"
"mesh-delivery-builder/internal/build"
)
// queueReader is the part of the builder a queues value reads through: heldOnTheBus.
type queueReader interface {
Queue(ctx context.Context, language string, limit int) (build.SeatQueue, error)
}
// queueRead answers one value of the queues state (the tool builder_seat_queue, issue 1945 note 62): a value not yet
// answered is read now, from the stream the builder's account may read, and written back by compare-and-set at its
// revision with what was read or why it could not be. A value answered already (this builder's own write, or another
// instance's) or deleted is left.
func queueRead(ctx context.Context, r queueReader, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) {
if e.Operation() != jetstream.KeyValuePut {
return
}
a, err := build.DecodeQueueAsk(e.Value())
if err != nil {
logf("%s.%s holds what is not a queue read, left: %v", build.QueuesState, e.Key(), err)
return
}
if !a.Answered.IsZero() {
return
}
read, cancel := context.WithTimeout(ctx, 20*time.Second)
q, err := r.Queue(read, e.Key(), max(1, min(a.Limit, build.MaxQueueLimit)))
cancel()
if err != nil {
a.Error = err.Error()
}
a.Queue, a.Answered = &q, time.Now().UTC()
data, err := json.Marshal(a)
if err != nil {
return
}
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
logf("%s.%s: the read could not be written: %v", build.QueuesState, e.Key(), err)
}
}
@@ -1,75 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"mesh-delivery-builder/internal/build"
)
// askAgainer is the part of the builder a reasks value asks through.
type askAgainer interface {
AskAgain(key, why string) []string
}
// reAsked answers one value of the reasks state (the tool builder_ask_again, issue 1945): a value not yet answered asks the
// key's waiting lanes again and is written back, by compare-and-set at its revision, with what the builder did. A value
// answered already (this builder's own write) or deleted is left.
func reAsked(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) {
if e.Operation() != jetstream.KeyValuePut {
return
}
r, err := build.DecodeReAsk(e.Value())
if err != nil {
logf("%s.%s holds what is not an ask to ask again, left: %v", build.ReAsksState, e.Key(), err)
return
}
if r.Answer != "" {
return
}
asked := b.AskAgain(e.Key(), r.Why)
if len(asked) == 0 {
r.Answer = "no ask of this key waits in this builder now (it is not building, or its build has not reached its ask yet): nothing was asked; builder_stuck shows where it stands"
} else {
r.Answer = "asked again: the " + strings.Join(asked, ", ") + " ask, under a new id the key names"
}
r.Answered = time.Now().UTC()
logf("%s: asked again by %s (%s): %s", e.Key(), r.By, r.Why, r.Answer)
data, err := json.Marshal(r)
if err != nil {
return
}
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err)
}
}
// emitAskedAgain says the builder's event build.asked-again (issue 1945): mesh.mod.<module>.event.build.asked-again,
// with the SDK's envelope headers and the new ask's id as the bus's dedup id.
func emitAskedAgain(ctx context.Context, js jetstream.JetStream, cred credential, e build.AskedAgain) error {
data, err := json.Marshal(e)
if err != nil {
return err
}
msg := nats.NewMsg("mesh.mod." + cred.Module + ".event." + build.AskedAgainEvent)
id := "asked-again-" + e.New
msg.Header.Set("x-event-id", id)
msg.Header.Set("x-source", cred.Module)
msg.Header.Set("x-node", cred.Node)
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
msg.Header.Set("content-type", "application/json")
msg.Data = data
pub, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
if _, err := js.PublishMsg(pub, msg, jetstream.WithMsgID(id)); err != nil {
return fmt.Errorf("saying %s for %s: %w", build.AskedAgainEvent, e.Key, err)
}
return nil
}
@@ -28,11 +28,8 @@ import (
//
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
// key <node>.<module>) hold an open assignment of its build module on its node, and only once it was written since that
// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report
// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node
// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran
// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot
// key <node>.<module>) hold an open assignment of its build module on its node; it is passed on as deleted otherwise,
// and passed on again when the module is assigned there again. Until the assignments are settled (or where they cannot
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
// refusal is the guard of novox/hq ADR 0465, and it stays.
type toolchainGate struct {
@@ -41,14 +38,14 @@ type toolchainGate struct {
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
logf func(format string, a ...any)
known bool // the assignments are settled
standing map[string]time.Time // <node>.<module> with an open assignment -> since when it stands
standing map[string]bool // <node>.<module> with an open assignment
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
left map[string]string // <module>/<key> -> why it was said left out, so each reason is said once
left map[string]bool // <module>/<key> said as left out, so it is said once
}
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{},
raw: map[string]map[string][]byte{}, left: map[string]string{}}
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]bool{},
raw: map[string]map[string][]byte{}, left: map[string]bool{}}
for _, m := range modules {
g.modules[m] = true
}
@@ -70,31 +67,20 @@ func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool
return g.pass(module, key)
}
// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers
// whether what it passed on changed.
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool {
// Assignment takes one change of the controller's assignments in and answers whether what it passed on changed.
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool) bool {
node, module, ok := strings.Cut(key, ".")
if !ok || !g.modules[module] {
return false
}
assigned, open := openAssignment(value)
open = open && !deleted
open := !deleted && openAssignment(value)
g.mu.Lock()
defer g.mu.Unlock()
was, stood := g.standing[key]
since := assigned
if since.IsZero() {
// A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it.
since = at
if stood {
since = was
}
}
if open == stood && (!open || since.Equal(was)) {
if g.standing[key] == open {
return false
}
if open {
g.standing[key] = since
g.standing[key] = true
} else {
delete(g.standing, key)
}
@@ -125,58 +111,37 @@ func (g *toolchainGate) Settled() bool {
return changed
}
// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not
// known), deleted otherwise. The caller holds mu.
// pass hands one key on as it stands: its value when its node is assigned (or the assignments are not known), deleted
// otherwise. The caller holds mu.
func (g *toolchainGate) pass(module, key string) bool {
value, held := g.raw[module][key]
if !held {
return g.forward(module, key, nil, true)
}
node, _, err := delivery.ParseToolchainKey(key)
why := ""
if err == nil && g.known {
why = g.leftOut(module, node, value)
}
if why == "" {
if err != nil || !g.known || g.standing[node+"."+module] {
delete(g.left, module+"/"+key)
return g.forward(module, key, value, false)
}
if g.left[module+"/"+key] != why {
g.left[module+"/"+key] = why
g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why)
if !g.left[module+"/"+key] {
g.left[module+"/"+key] = true
g.logf("%s.%s is left out of the agreed toolchain: %s has no assignment on %s, so it is no builder (issue 1940)",
delivery.ToolchainsBucket(module), key, module, node)
}
return g.forward(module, key, nil, true)
}
// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module,
// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse.
// The caller holds mu.
func (g *toolchainGate) leftOut(module, node string, value []byte) string {
since, open := g.standing[node+"."+module]
if !open {
return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node)
}
tc, err := delivery.DecodeToolchain(value)
if err == nil && tc.At.Before(since) {
return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again",
tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node)
}
return ""
}
// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not
// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when
// it was assigned, zero where the value does not say.
func openAssignment(value []byte) (time.Time, bool) {
// openAssignment is whether a value of the controller's assignments holds an assignment that stands: one that has not
// ended and has no state (a pending one is not on the machine yet; mesh-controller internal/chain Assignment).
func openAssignment(value []byte) bool {
var a struct {
Assigned time.Time `json:"assigned"`
Ended time.Time `json:"ended"`
State string `json:"state"`
Ended time.Time `json:"ended"`
State string `json:"state"`
}
if json.Unmarshal(value, &a) != nil {
return time.Time{}, false
return false
}
return a.Assigned, a.Ended.IsZero() && a.State == ""
return a.Ended.IsZero() && a.State == ""
}
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
@@ -190,19 +155,17 @@ func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate,
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
return errs, nil
}
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e)) }
w, err := settle(ctx, kv, each)
if err != nil {
// Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read.
logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
return errs, nil
return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err)
}
if gate.Settled() {
changed(ctx)
}
go func() {
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) {
if gate.Assignment(e.Key(), e.Value(), !opOf(e)) {
changed(ctx)
}
})
@@ -3,19 +3,10 @@ package main
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
import (
"bytes"
"context"
"errors"
"fmt"
"log"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
@@ -53,10 +44,10 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
}
for _, node := range []string{"g14", "novox", "shanks"} {
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{})
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false)
}
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{})
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false)
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false) // another module's: no matter
if !gate.Settled() {
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
}
@@ -72,20 +63,20 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
t.Fatal("an unassigned node's report changed what was passed on")
}
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) {
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false) {
t.Fatal("assigning ace again passed nothing on")
}
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
t.Fatalf("builders that run different versions still disagree; got %q", none)
}
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) {
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false) {
t.Fatal("a pending assignment is no assignment on the machine")
}
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
t.Fatalf("got %q, %q", v, none)
}
gate.Assignment("ace.mesh-build-node", nil, true, time.Time{})
gate.Assignment("ace.mesh-build-node", nil, true)
if v, _ := agreedOf(set); v != "v26.12.1" {
t.Fatalf("got %q", v)
}
@@ -97,132 +88,25 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
if n := len(set.Entries("typescript")); n != 2 {
t.Fatalf("novox and shanks are left; got %d entries", n)
}
// 1945 (a part exits at start for a bucket it works without): a bus that refuses the assignments' watch, or every
// toolchains bucket, leaves the watches unstarted and the part running, said; a refused bucket it cannot work without
// is an error naming the bucket and its grant.
var logs bytes.Buffer
logger := log.New(&logs, "", 0)
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments")}
if _, err := watchAssignments(context.Background(), refused, newToolchainGate(nil, set.Put, t.Logf), func(context.Context) {}, logger); err != nil {
t.Errorf("a refused watch of the assignments stopped the start: %v", err)
}
if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil {
t.Errorf("a refused toolchains bucket stopped the start: %v", err)
}
if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") {
t.Errorf("the refusals were not said: %q", logs.String())
}
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil ||
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
}
// The gate's source (the previews' trunk heads): refused at start, the builder goes on, and a candidate asking a trunk
// head is told why and opens it again then.
src := &trunkSource{ctx: context.Background(), js: refusingBus{refuseOpen: true, err: refused.err}, heads: &sourceHeads{}, errs: make(chan error, 1), logf: logger.Printf}
src.start()
if _, err := src.head(context.Background(), 7, "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), delivery.SourceBucket) {
t.Errorf("a trunk head with the source refused: %v, want why", err)
}
if src.opened {
t.Error("a refused source counts as opened, so it is never opened again")
}
}
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
type refusingBus struct {
refuseOpen bool
err error
}
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
if r.refuseOpen {
return nil, r.err
}
return refusedWatch{err: r.err}, nil
}
type refusedWatch struct {
jetstream.KeyValue
err error
}
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
return nil, r.err
}
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
return nil, r.err
}
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript
// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node
// writes it again. An assignment whose value names no time stands since the revision that opened it.
// A newly assigned build node that has not reported yet changes nothing; its report counts as it comes.
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again
report := func(t *testing.T, node, version string, at time.Time) (string, []byte) {
key, _ := gateReport(t, node, version)
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
if err != nil {
t.Fatal(err)
}
return key, value
set := &buildrun.ToolchainSet{}
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(string, ...any) {})
gate.Settled()
if gate.Assignment("g14.mesh-build-node", []byte(`{}`), false) {
t.Fatal("an assignment with no report passed something on")
}
for _, c := range []struct {
name string
assignment string // ace's assignment value
revision time.Time // when the bus wrote it
at time.Time // when ace wrote its report
counts bool
}{
{"no report yet, then one", `{}`, time.Time{}, time.Now(), true},
{"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false},
{"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false},
{"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true},
} {
t.Run(c.name, func(t *testing.T) {
set := &buildrun.ToolchainSet{}
var said []string
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
for _, node := range []string{"g14", "novox", "shanks"} {
gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{})
key, value := gateReport(t, node, "v26.12.1")
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
}
gate.Settled()
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) {
t.Fatal("an assignment with no report passed something on")
}
key, value := report(t, "ace", "v26.12.0", c.at)
if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts {
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
}
// A rewrite of the assignment that does not move it changes nothing, and nothing is said again.
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) {
t.Fatal("the same assignment written again changed what was passed on")
}
v, none := agreedOf(set)
if c.counts {
if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none)
}
return
}
if v != "v26.12.1" || none != "" {
t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none)
}
if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") {
t.Fatalf("said %q, want once why ace's report is left out", said)
}
// ace writes its report again since it was assigned: it counts.
key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute))
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
t.Fatal("ace's new report was not passed on")
}
if n := len(set.Entries("typescript")); n != 4 {
t.Fatalf("four builders report; got %d", n)
}
})
key, value := gateReport(t, "g14", "v26.12.1")
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
t.Fatal("an assigned node's report was not passed on")
}
key, value = gateReport(t, "ace", "v26.12.0")
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
t.Fatal("a report of a node with no assignment key was passed on after the assignments settled")
}
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
t.Fatalf("got %q, %q", v, none)
}
}
@@ -4,13 +4,11 @@ import (
"context"
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"log"
"strings"
"sync"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
@@ -64,14 +62,11 @@ func (w *toolchainWatch) ensure(module string) error {
w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module)
return nil
case err != nil:
// Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the module's next report.
w.logger.Printf("the bucket %s cannot be read (%v): %s's toolchains are not heard until its next report", name, err, module)
return nil
return fmt.Errorf("the bucket %s: %w", name, err)
}
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.put(module, e.Key(), e.Value(), !opOf(e)) })
if err != nil {
process.NotWatched(w.logger.Printf, name, err, fmt.Sprintf("%s's toolchains are not heard until its next report", module))
return nil
return err
}
w.watched[module] = true
go func() {
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-builder
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
-2
View File
@@ -1,7 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1 h1:fzOkrz9CSp3mEjeV7NfZx8yoivI1sD1AOtIPtyyF8RI=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -1,337 +0,0 @@
package build
import (
"context"
"errors"
"fmt"
"regexp"
"slices"
"strconv"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// **An ask that can no longer be answered is asked again** (novox/hq issue 1945). A build seat's stream gives an ask up
// once it was delivered as often as its worker may (delivery.AskMaxDeliver), and a message given up on, dropped or
// terminated with no step written is answered by no build node: the lane waiting for its step would wait for ever, and on
// restart take over the same id and wait again. The builder notices it at the check bound (CheckAfter, the step bound),
// a context's bound and no timer of its own: it asks Gone, which reads the bus, and asks such an ask again under a new
// id, written into the key, said in its log and as the event build.asked-again, at most Askings times; then the key is
// failed with why. A person asks the same by hand with the tool builder_ask_again (AskAgain).
// DefaultAskings is how many times a build module's part of a build is asked, the first included, when Builder.Askings is 0.
const DefaultAskings = 3
// AskedAgainEvent is the event the builder says when it asked an ask again: mesh.mod.mesh-delivery-builder.event.<this>.
const AskedAgainEvent = "build.asked-again"
// AskedAgain is the body of AskedAgainEvent.
type AskedAgain struct {
Key string `json:"key"`
Module string `json:"module"`
FP string `json:"fp"`
BuildModule string `json:"build_module"`
Language string `json:"language"`
Old string `json:"old"`
New string `json:"new"`
Asking int `json:"asking"`
Askings int `json:"askings"`
Why string `json:"why"`
At time.Time `json:"at"`
}
// AskSeen is what the bus says of an ask whose step has not ended, as the builder may read it: its build seat's stream
// (information and messages by sequence) and its build module's steps. The builder may not read the seat workers'
// delivery counts or ack floors (granted to the delivery view alone: $JS.API.CONSUMER.INFO.<stream>.<worker>), so a
// message given up on is told by what the workers did after it: a build node took an ask asked later, of the same kind or
// of one pulled after it (issue 1945 note 62: seven messages SEAT_NODE_BUILD_GO had given up on sat ahead of a Go ask for
// five hours, each counted as waiting, so the ask was never asked again).
type AskSeen struct {
Stream string
// Kind is the ask's kind, At when it was asked (its id's time).
Kind string
At time.Time
// Step is whether a build node wrote a step for the ask (running: the step bound of buildrun.Asker judges it).
Step bool
// OnStream is whether the stream still holds the ask's message; PutAt is when it was put.
OnStream bool
PutAt time.Time
// Ahead are the messages of its subject before it on the stream.
Ahead []Queued
// Last is the ask asked last that a build node took (wrote a step for) of the ask's kind or of a kind pulled after
// it, when one was taken: every message of the kind asked before it was then delivered already, or given up on.
Last *Taken
}
// Queued is one message of a build seat's stream: its sequence, its ask's id and kind, when the ask was asked (its id's
// time, the put time when the id says none) and put, and the state of the step a build node wrote for it ("" for none).
type Queued struct {
Seq uint64 `json:"seq"`
Ask string `json:"ask"`
Kind string `json:"kind"`
At time.Time `json:"asked"`
PutAt time.Time `json:"put"`
Step string `json:"step,omitempty"`
}
// Taken is an ask a build node took: its id, kind and when it was asked, and the step's node and start.
type Taken struct {
Ask string `json:"ask"`
Kind string `json:"kind"`
At time.Time `json:"asked"`
Node string `json:"node"`
Started time.Time `json:"started"`
}
// Proves is whether a taken ask of kind taken shows that a message of kind waiting asked before it is no longer
// delivered: a holder pulls the kinds in delivery.AskKinds' order, so taking one of the same kind or of a later one means
// its worker had none of waiting's kind left to give.
func Proves(taken, waiting string) bool {
t, w := slices.Index(delivery.AskKinds, taken), slices.Index(delivery.AskKinds, waiting)
return t >= 0 && w >= 0 && t >= w
}
// Waits is whether a message ahead still waits to be delivered: no build node wrote a step for it, and no ask of its
// kind asked after it was taken. A message its worker gave up on (delivered its most times) stays on a work queue until
// its age, and is not counted.
func (s AskSeen) Waits(q Queued) bool {
return q.Step == "" && !s.passed(q.At)
}
func (s AskSeen) passed(at time.Time) bool {
return s.Last != nil && Proves(s.Last.Kind, s.Kind) && s.Last.At.After(at)
}
// Waiting is how many messages ahead still wait to be delivered (Waits).
func (s AskSeen) Waiting() int {
n := 0
for _, q := range s.Ahead {
if s.Waits(q) {
n++
}
}
return n
}
// Why is why the ask can no longer be answered, or "" when it can: it has a step, waits behind others of its kind still
// deliverable, or no build node took a later ask of its kind (the nodes are busy or paused: a backlog, never a fault,
// ADR 0525 §9).
func (s AskSeen) Why() string {
switch {
case s.Step:
return ""
case !s.OnStream:
return fmt.Sprintf("no build node wrote a step for it and %s no longer holds it: it was given up on or dropped", s.Stream)
case s.Waiting() > 0 || !s.passed(s.at()):
return ""
default:
behind := ""
if n := len(s.Ahead); n > 0 {
behind = fmt.Sprintf(" behind %d given up on or taken", n)
}
return fmt.Sprintf("it waits on %s since %s%s while %s took the %s ask %s asked later at %s: its stream no longer delivers it (given up after its most deliveries)",
s.Stream, s.PutAt.UTC().Format(time.RFC3339), behind, s.Last.Node, s.Last.Kind, s.Last.Ask, s.Last.At.UTC().Format(time.RFC3339))
}
}
func (s AskSeen) at() time.Time {
if s.At.IsZero() {
return s.PutAt
}
return s.At
}
// AskedAt is when an ask id was made (buildstep.NewID: its milliseconds in base 36 before six hex digits), or zero.
func AskedAt(id string) time.Time {
if len(id) <= 6 {
return time.Time{}
}
ms, err := strconv.ParseInt(id[:len(id)-6], 36, 64)
if err != nil || ms <= 0 {
return time.Time{}
}
return time.UnixMilli(ms)
}
// Dropper removes an ask's message from its build seat's stream (Builder.Gone may be one): the old message of an ask asked
// again, so a message given up on does not sit ahead of every later ask of its kind until its age (issue 1945 note 62).
type Dropper interface {
Drop(ctx context.Context, a delivery.StepAsk) error
}
// drop removes the old ask's message when Gone can; a refusal is said and changes nothing else: the new ask is put.
func (b *Builder) drop(ctx context.Context, ask delivery.StepAsk, old string) {
d, ok := b.Gone.(Dropper)
if !ok {
return
}
prior := ask
prior.ID = old
if err := d.Drop(ctx, prior); err != nil {
b.logf("the message of the ask %s asked again as %s stays on its stream: %v", old, ask.ID, err)
}
}
func (b *Builder) askings() int {
if b.Askings > 0 {
return b.Askings
}
return DefaultAskings
}
var askingWords = regexp.MustCompile(`asked (\S+) again \(asking (\d+)\)`)
// askingOf is the asking a key's step words name for a build module's ask, 1 when none.
func askingOf(step, buildModule string) int {
for _, m := range askingWords.FindAllStringSubmatch(step, -1) {
if m[1] == buildModule {
if n, err := strconv.Atoi(m[2]); err == nil && n > 0 {
return n
}
}
}
return 1
}
// AskAgain asks the waiting asks of a build key again now, under new ids (the tool builder_ask_again): it answers the
// languages whose lane waits, none when no lane of this builder waits on the key.
func (b *Builder) AskAgain(key, why string) []string {
b.init()
b.mu.Lock()
defer b.mu.Unlock()
var asked []string
for language, c := range b.waiting[key] {
select {
case c <- why:
default: // one is already asked
}
asked = append(asked, language)
}
return asked
}
// Waiting are the build keys whose lanes wait for a step now, each with its languages.
func (b *Builder) Waiting() map[string][]string {
b.init()
b.mu.Lock()
defer b.mu.Unlock()
out := map[string][]string{}
for key, ls := range b.waiting {
for l := range ls {
out[key] = append(out[key], l)
}
}
return out
}
func (b *Builder) listen(key, language string) chan string {
b.init()
c := make(chan string, 1)
b.mu.Lock()
if b.waiting[key] == nil {
b.waiting[key] = map[string]chan string{}
}
b.waiting[key][language] = c
b.mu.Unlock()
return c
}
func (b *Builder) unlisten(key, language string, c chan string) {
b.mu.Lock()
if b.waiting[key][language] == c {
delete(b.waiting[key], language)
if len(b.waiting[key]) == 0 {
delete(b.waiting, key)
}
}
b.mu.Unlock()
}
var (
errCheck = errors.New("the check bound passed")
errAgain = errors.New("asked again")
)
type byHand struct{ why string }
func (h byHand) Error() string { return "asked again by hand: " + h.why }
// answer waits for the step of the lane's ask, and asks it again while it can no longer be answered (Gone) or a person
// asked it again (AskAgain); it answers the step and the ask that has it.
func (b *Builder) answer(ctx context.Context, l *lane, ask delivery.StepAsk, asked *askNotes, module, fp string) (delivery.Step, delivery.StepAsk, error) {
key, _ := delivery.BuilderBuildKey(module, fp)
again := b.listen(key, ask.Language)
defer b.unlisten(key, ask.Language, again)
asking := askingOf(asked.step, l.module)
for {
step, why, err := b.waitOnce(ctx, ask, again)
switch {
case errors.Is(err, errCheck):
continue
case errors.Is(err, errAgain):
if asking >= b.askings() {
return step, ask, fmt.Errorf("the ask %s could not be answered and was asked %d times, the most it is: %s", ask.ID, asking, why)
}
asking++
old := ask.ID
ask.ID, ask.At = buildstep.NewID(), time.Now()
if err := asked.note(ctx, ask.Language, ask.ID, fmt.Sprintf("asked %s again (asking %d): %s", l.module, asking, why)); err != nil {
return step, ask, fmt.Errorf("the ask of %s asked again could not be written to the key: %w", l.module, err)
}
if err := b.Asks.Put(ctx, ask); err != nil {
return step, ask, err
}
b.drop(ctx, ask, old)
b.logf("%s: the ask %s of %s is asked again as %s (asking %d of %d): %s", key, old, l.module, ask.ID, asking, b.askings(), why)
if b.Said != nil {
b.Said(AskedAgain{Key: key, Module: module, FP: fp, BuildModule: l.module, Language: ask.Language, Old: old, New: ask.ID,
Asking: asking, Askings: b.askings(), Why: why, At: ask.At})
}
default:
return step, ask, err
}
}
}
// waitOnce waits for the ask's step until it ends, a person asks it again, or the check bound passes: then it asks Gone
// and answers errAgain with why when the ask can no longer be answered, errCheck when it waits on.
func (b *Builder) waitOnce(ctx context.Context, ask delivery.StepAsk, again <-chan string) (delivery.Step, string, error) {
hand, cancel := context.WithCancelCause(ctx)
defer cancel(nil)
go func() {
select {
case why := <-again:
cancel(byHand{why})
case <-hand.Done():
}
}()
wait := hand
if b.Gone != nil && b.CheckAfter > 0 {
var stop context.CancelFunc
wait, stop = context.WithTimeoutCause(hand, b.CheckAfter, errCheck)
defer stop()
}
step, err := b.Asks.Wait(wait, ask)
if err == nil || ctx.Err() != nil {
return step, "", err
}
var h byHand
if errors.As(context.Cause(hand), &h) {
return step, strings.TrimSpace("asked again by hand: " + h.why), errAgain
}
if errors.Is(context.Cause(wait), errCheck) {
why, gerr := b.Gone.Gone(ctx, ask)
if gerr != nil {
b.logf("whether the ask %s can still be answered cannot be read, so it is waited for: %v", ask.ID, gerr)
return step, "", errCheck
}
if why == "" {
return step, "", errCheck
}
return step, why, errAgain
}
return step, "", err
}
@@ -73,17 +73,9 @@ func (b *Builder) considerCandidate(ctx context.Context, c delivery.Candidate) {
// The named paths' objects the preview gave it: the fetch names the paths (runLane, ADR 0499 clause 4).
m.Inputs2 = &delivery.KeptInputs2{Paths: cm.Paths}
}
// The preview is merged onto the trunk head the candidate was previewed onto: the head of the pull request's
// repository in the gate's source revision c.Base (ADR 0484 §3). Never the module's value's commit: that is the
// commit its fingerprint first appeared at, and may be many merges behind the trunk.
base, err := b.trunkHead(ctx, c.Base, p.Repository)
if err != nil {
b.logf("candidates.%s: the module %s's fingerprint %s is not built: %v", c.Group, name, cm.FP2, err)
continue
}
b.mu.Lock()
if _, onTrunk := b.trunkFP[key]; !onTrunk {
b.via[key] = via{Ref: fmt.Sprintf("refs/pull/%d/head", p.Number), Base: base}
b.via[key] = via{Ref: fmt.Sprintf("refs/pull/%d/head", p.Number), Base: trunk.Commit}
}
b.mu.Unlock()
b.consider(ctx, name, m)
@@ -114,18 +106,3 @@ func (b *Builder) agreedFor(ctx context.Context, manifest string) map[string]del
}
return out
}
// trunkHead is the head of repository in the gate's source revision rev (Builder.Trunk), or why it cannot be said.
func (b *Builder) trunkHead(ctx context.Context, rev uint64, repository string) (string, error) {
if b.Trunk == nil {
return "", fmt.Errorf("the gate's source cannot be read here, so the trunk head %s was previewed onto is not known", repository)
}
head, err := b.Trunk(ctx, rev, repository)
if err != nil {
return "", fmt.Errorf("the trunk head of %s at source revision %d cannot be read: %w", repository, rev, err)
}
if head == "" {
return "", fmt.Errorf("source revision %d names no trunk head of %s", rev, repository)
}
return head, nil
}
@@ -2,13 +2,9 @@ package build
import (
"context"
"encoding/json"
"fmt"
"os"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -44,74 +40,11 @@ func candidateValue(t *testing.T, group string, number int, head string) []byte
}
const (
// trunkHead is the trunk head the rig's source revision names: newer than the module value's commit, which stays at the
// commit its fingerprint first appeared at.
trunkHead = "3333333333333333333333333333333333333333"
headX = "1111111111111111111111111111111111111111"
headY = "2222222222222222222222222222222222222222"
headX = "1111111111111111111111111111111111111111"
headY = "2222222222222222222222222222222222222222"
)
// indexerCandidate is a candidate as the indexer writes it, with the gate's source revision it names (mesh-delivery-indexer
// internal/index testdata/candidate-written.json, kept equal to what the indexer writes by its own test).
func indexerCandidate(t *testing.T) (rev uint64, heads map[string]string, c delivery.Candidate, raw []byte) {
t.Helper()
data, err := os.ReadFile("../../../mesh-delivery-indexer/internal/index/testdata/candidate-written.json")
if err != nil {
t.Fatal(err)
}
var w struct {
Source struct {
Rev uint64 `json:"rev"`
Heads map[string]string `json:"heads"`
} `json:"source"`
Candidate json.RawMessage `json:"candidate"`
}
if err := json.Unmarshal(data, &w); err != nil {
t.Fatal(err)
}
if c, err = delivery.DecodeCandidate(w.Candidate); err != nil {
t.Fatal(err)
}
return w.Source.Rev, w.Source.Heads, c, w.Candidate
}
func TestACandidatesFingerprintIsBuiltOnceAsAKeyOfItsOwnFromTheHeadMergedOntoTheTrunk(t *testing.T) { // N7/AC1, N7/AC2 (issue 1684)
// The candidate the indexer writes is merged onto the trunk head of the source revision it names, never onto the module
// value's commit, which stays at the commit its fingerprint first appeared at (review of the indexer writing on change).
{
rev, heads, c, raw := indexerCandidate(t)
p := c.Pulls[0]
cm := p.Fps["avahi"]
ir := newRig(t, goBundle)
ir.mf.byDigest[cm.Manifest] = catalogue.Manifest{Module: "avahi", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}}
ir.b.Trunk = func(_ context.Context, at uint64, repository string) (string, error) {
if at != rev {
return "", fmt.Errorf("source revision %d is not heard", at)
}
return heads[repository], nil
}
const valueCommit = "7a74cee83f88e6585a5511c3cc53bdf0f5303d20"
value, err := delivery.EncodeModule(delivery.Module{Repository: p.Repository, Commit: valueCommit, FP2: "src2:" + hex64('e'),
Manifest: cm.Manifest, Path: "modules/avahi", Toolchains: agreedToolchains})
if err != nil {
t.Fatal(err)
}
ir.b.Module(context.Background(), "avahi", value)
ir.b.Wait()
ir.bm.mu.Lock()
ir.bm.calls = nil
ir.bm.mu.Unlock()
ir.b.Candidate(context.Background(), "feat%2Fx", raw)
ir.b.Wait()
fetches := ir.bm.callsOf("build_go_fetch")
if len(fetches) != 1 {
t.Fatalf("%d fetches of the indexer's candidate, want one", len(fetches))
}
f := fetches[0].body.(map[string]any)
if want := heads[p.Repository]; want == "" || f["base"] != want || f["base"] == valueCommit || f["commit"] != p.Head.Head {
t.Errorf("fetch args = %+v, want the head merged onto the trunk head %.8s of source revision %d", f, want, rev)
}
}
r := newRig(t, goBundle)
trunkModule(t, r)
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
@@ -125,8 +58,8 @@ func TestACandidatesFingerprintIsBuiltOnceAsAKeyOfItsOwnFromTheHeadMergedOntoThe
t.Fatalf("%d fetches, want one", len(fetches))
}
f := fetches[0].body.(map[string]any)
if f["commit"] != headX || f["ref"] != "refs/pull/7/head" || f["base"] != trunkHead || f["repository"] != "novox/mesh-catalog" {
t.Errorf("fetch args = %+v, want the head, its ref and the trunk head the candidate was previewed onto", f)
if f["commit"] != headX || f["ref"] != "refs/pull/7/head" || f["base"] != "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" || f["repository"] != "novox/mesh-catalog" {
t.Errorf("fetch args = %+v, want the head, its ref and the trunk commit to merge onto", f)
}
if b := r.bm.callsOf("build_go_build")[0].body.(map[string]any); b["module"] != "modules/m" {
t.Errorf("build args = %+v", b)
@@ -1,115 +0,0 @@
package build
import (
"encoding/json"
"fmt"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// **The tool builder_seat_queue reads a build seat's stream as the builder reads it** (novox/hq issue 1945 note 62; the
// instruction "tools are how a module is spoken to"): the messages waiting on SEAT_NODE_BUILD_<LANGUAGE>, bounded, each
// with its ask, its age, the state of the step a build node wrote for it, and whether the builder counts it as still
// waiting (AskSeen.Waits) or as given up on. Only the builder's own account may read the stream (issue 1910): the tools
// bundle writes the read into the builder's state queues, keyed by the language, which the builder watches and answers
// there, as builder_ask_again does through reasks.
const (
SeatQueueTool = "builder_seat_queue"
// QueuesState is the builder's state of queue reads: the bucket mesh-delivery-builder_queues, keyed by the language.
QueuesState = "queues"
QueuesBucket = delivery.BuilderModule + "_" + QueuesState
// DefaultQueueLimit and MaxQueueLimit bound the messages read of each kind.
DefaultQueueLimit = 50
MaxQueueLimit = 500
)
// QueueNote says what the read cannot show.
const QueueNote = "the builder may not read the seat workers' delivery counts or ack floors ($JS.API.CONSUMER.INFO.<stream>.<worker> is " +
"granted to the delivery view alone): a message counts as given up on when no node wrote its step and a node took an ask of its kind " +
"asked after it; nats.nats_streams shows the workers' pending and waiting pulls"
// QueueAsk is a value of the queues state: a read asked (limit, who, when) and, once the builder read the stream, its answer.
type QueueAsk struct {
V int `json:"v"`
Limit int `json:"limit"`
By string `json:"by,omitempty"`
At time.Time `json:"at"`
Answered time.Time `json:"answered,omitzero"`
Error string `json:"error,omitempty"`
Queue *SeatQueue `json:"queue,omitempty"`
}
// SeatQueue is what the builder read of one build seat's stream.
type SeatQueue struct {
Language string `json:"language"`
Stream string `json:"stream"`
Messages uint64 `json:"messages"`
FirstSeq uint64 `json:"first_seq"`
LastSeq uint64 `json:"last_seq"`
Kinds []KindQueue `json:"kinds"`
Note string `json:"note"`
}
// KindQueue is one kind's messages on the stream, oldest first: how many the builder counts as waiting, the ask asked
// last a build node took of the kind (or of one pulled after it), and each message.
type KindQueue struct {
Kind string `json:"kind"`
Read int `json:"read"`
Waiting int `json:"waiting"`
GivenUp int `json:"given_up_or_taken"`
Last *Taken `json:"last_taken,omitempty"`
Messages []QueuedState `json:"messages"`
}
// QueuedState is one message as the tool shows it.
type QueuedState struct {
Queued
AgeSeconds int64 `json:"age_s"`
Waits bool `json:"waits"`
}
// NewKindQueue is seen's messages ahead as one kind's queue at now.
func NewKindQueue(kind string, seen AskSeen, now time.Time) KindQueue {
k := KindQueue{Kind: kind, Read: len(seen.Ahead), Last: seen.Last}
for _, q := range seen.Ahead {
w := seen.Waits(q)
if w {
k.Waiting++
} else {
k.GivenUp++
}
k.Messages = append(k.Messages, QueuedState{Queued: q, AgeSeconds: int64(now.Sub(q.PutAt) / time.Second), Waits: w})
}
return k
}
// NewQueueAsk is the value builder_seat_queue writes for a language: its key and the value.
func NewQueueAsk(language string, limit int, by string, at time.Time) (string, []byte, error) {
if _, err := delivery.AskStream(language); err != nil {
return "", nil, fmt.Errorf("language: %w", err)
}
if limit <= 0 {
limit = DefaultQueueLimit
}
data, err := json.Marshal(QueueAsk{V: 1, Limit: min(limit, MaxQueueLimit), By: by, At: at.UTC()})
return language, data, err
}
// DecodeQueueAsk reads a queues value.
func DecodeQueueAsk(data []byte) (QueueAsk, error) {
var a QueueAsk
if err := json.Unmarshal(data, &a); err != nil {
return a, err
}
if a.V != 1 {
return a, fmt.Errorf("a queues value of version %d, not 1", a.V)
}
return a, nil
}
// LanguageOf is the language a tool's seat or language argument names: go, node or image, or node-build-<language>.
func LanguageOf(arg string) string {
return strings.TrimPrefix(strings.TrimSpace(arg), "node-build-")
}
@@ -379,9 +379,8 @@ func TestACandidatesFetchNamesTheNamedPathsTheIndexerGaveIt(t *testing.T) { // A
t.Fatal("no fetch")
}
req, err := buildstep.ParseFetchRequest(f[0].body.(map[string]any))
// The base is the trunk head the candidate was previewed onto, never the module value's commit (an older one).
if err != nil || req.Dir != "modules/m" || strings.Join(req.Paths, ",") != "package.json,src" || req.Base != trunkHead {
t.Errorf("fetch request = %+v, %v; want the trunk head %.8s as the base, the module's directory and its named paths", req, err, trunkHead)
if err != nil || req.Dir != "modules/m" || strings.Join(req.Paths, ",") != "package.json,src" || req.Base == "" {
t.Errorf("fetch request = %+v, %v; want the candidate's base, the module's directory and its named paths", req, err)
}
}
@@ -53,21 +53,6 @@ type Builder struct {
// Stored answers which references the artifact store no longer holds (the controller's verb held; issue 1937): a done
// key of the trunk's fingerprint is asked before it is left done (ClaimOptions.Stored). Nil asks nothing.
Stored func(ctx context.Context, references []string) (gone []string, err error)
// Trunk answers the head of a repository in the gate's source revision rev (source.main at that revision): the trunk a
// candidate's preview is merged onto (Candidate.Base). Nil builds no candidate's fingerprint, and says why.
Trunk func(ctx context.Context, rev uint64, repository string) (string, error)
// Gone says why an ask with no ended step can no longer be answered, or "" when it can (issue 1945): asked once the
// ask has waited CheckAfter, and again each CheckAfter it goes on waiting. An ask that cannot be answered is asked
// again under a new id, at most Askings times. Nil, or CheckAfter 0, never asks it.
Gone interface {
Gone(ctx context.Context, a delivery.StepAsk) (string, error)
}
CheckAfter time.Duration
// Askings is how many times one build module's part of a build is asked, the first included, before its key is failed
// with why; DefaultAskings when 0.
Askings int
// Said is told each ask asked again (the builder's event build.asked-again). Nil tells nobody.
Said func(AskedAgain)
once sync.Once
sem chan struct{}
@@ -87,8 +72,6 @@ type Builder struct {
// held are the done keys' references the store answered held, by `<key> <references>`, asked once per process: a
// blob collected later is found by the take-in, whose artifacts-gone forgets the key here (issue 1937).
held map[string]bool
// waiting are the lanes that wait for an ask's step, by build key and language: AskAgain asks through them.
waiting map[string]map[string]chan string
}
type planned struct {
@@ -105,7 +88,7 @@ func (b *Builder) init() {
b.sem = make(chan struct{}, limit)
b.cands, b.via, b.trunkFP = map[string]delivery.Candidate{}, map[string]via{}, map[string]bool{}
b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{}
b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{}
b.published, b.held = map[string]bool{}, map[string]bool{}
})
}
@@ -286,13 +269,6 @@ func (b *Builder) Reconsider(ctx context.Context) {
for _, k := range sortedKeys(known) {
b.consider(ctx, k, known[k])
}
b.TrunkMoved(ctx)
}
// TrunkMoved considers every candidate again once a revision of the gate's source is heard: a candidate heard before the
// revision it was previewed onto had no trunk head to be merged onto.
func (b *Builder) TrunkMoved(ctx context.Context) {
b.init()
b.mu.Lock()
cands := make([]delivery.Candidate, 0, len(b.cands))
for _, c := range b.cands {
@@ -687,14 +663,12 @@ type askNotes struct {
b *Builder
cl *Claim
resumed map[string]string
// step is the step words of a key taken over at start: an ask asked again says its asking there (askingOf).
step string
}
func (b *Builder) notes(cl *Claim) *askNotes {
n := &askNotes{b: b, cl: cl}
if cl.Build.State == delivery.BuildBuilding && len(cl.Build.Asks) > 0 {
n.resumed, n.step = maps.Clone(cl.Build.Asks), cl.Build.Step
n.resumed = maps.Clone(cl.Build.Asks)
}
return n
}
@@ -792,7 +766,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del
return
}
}
step, ask, err := b.answer(ctx, l, ask, asked, module, m.FP2)
step, err := b.Asks.Wait(ctx, ask)
if err != nil {
l.err = fmt.Errorf("%s: %w", l.module, err)
return
@@ -6,10 +6,8 @@ import (
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"strconv"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
@@ -178,7 +176,6 @@ func newRig(t *testing.T, artifacts ...catalogue.Artifact) *rig {
r.b = &Builder{
Claimer: Claimer{Store: r.st, By: "novox.builder"}, Asks: buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 2 * time.Second},
Toolchains: r.tools, Manifests: r.mf, Logf: t.Logf,
Trunk: func(context.Context, uint64, string) (string, error) { return trunkHead, nil },
}
var err error
if r.key, err = delivery.BuilderBuildKey("m", fp1); err != nil {
@@ -618,286 +615,27 @@ func TestAFailedPublishOfACandidatesImageIsWrittenToTheKey(t *testing.T) { // re
}
}
// goneAsks answers Gone from a function of the ask, and counts how often it was asked; it keeps the ids it was asked to
// drop from their stream (Dropper).
type goneAsks struct {
mu sync.Mutex
n int
why func(a delivery.StepAsk) string
dropped []string
}
func (g *goneAsks) Drop(_ context.Context, a delivery.StepAsk) error {
g.mu.Lock()
defer g.mu.Unlock()
g.dropped = append(g.dropped, a.ID)
return nil
}
func (g *goneAsks) Gone(_ context.Context, a delivery.StepAsk) (string, error) {
g.mu.Lock()
g.n++
g.mu.Unlock()
return g.why(a), nil
}
func (g *goneAsks) asked() int {
g.mu.Lock()
defer g.mu.Unlock()
return g.n
}
// waitForAsks waits until n asks were put.
func waitForAsks(t *testing.T, r *rig, n int) []delivery.StepAsk {
t.Helper()
for deadline := time.Now().Add(2 * time.Second); len(r.asks.Asks()) < n; time.Sleep(time.Millisecond) {
if time.Now().After(deadline) {
t.Fatalf("%d asks put, want %d", len(r.asks.Asks()), n)
}
}
return r.asks.Asks()
}
// Review round 2 item 1: a build ask that waits in its stream longer than the step bound is not failed and not asked
// again; the key is done when a node takes it. Issue 1945: an ask that can no longer be answered (its stream gave it up)
// is asked again under a new id, said, at most Askings times; a person asks it again by hand the same way.
// again; the key is done when a node takes it.
func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
t.Run("waits in its stream: never asked again", func(t *testing.T) {
r := newRig(t, goBundle)
r.asks.Answer = nil
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 20 * time.Millisecond}
gone := &goneAsks{why: func(delivery.StepAsk) string { return "" }}
r.b.Gone, r.b.CheckAfter = gone, 5*time.Millisecond
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
waitForAsks(t, r, 1)
time.Sleep(100 * time.Millisecond)
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding {
t.Fatalf("the key = %+v while its ask waits, want building", v)
r := newRig(t, goBundle)
r.asks.Answer = nil
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 20 * time.Millisecond}
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
for deadline := time.Now().Add(2 * time.Second); len(r.asks.Asks()) == 0; time.Sleep(time.Millisecond) {
if time.Now().After(deadline) {
t.Fatal("no ask was put")
}
if gone.asked() == 0 {
t.Fatal("whether the ask can still be answered was never read at the check bound") // 1945
}
r.asks.End(r.answer(r.asks.Asks()[0]))
r.b.Wait()
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(r.asks.Asks()) != 1 {
t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks()))
}
})
for _, c := range []struct {
name string
askings int
// step is the key's step words at start when it names an ask (a key taken over), "" for a fresh build.
step string
byHand bool
always bool // Gone says every ask cannot be answered, not only the first
puts int
done bool
asking int // the asking the event says, 0 when none is said
}{
{name: "given up by its stream: asked again under a new id and done from it", puts: 2, done: true, asking: 2},
{name: "asked again by hand: the same, with no check bound", byHand: true, puts: 2, done: true, asking: 2},
{name: "never answerable: failed after its askings", askings: 2, always: true, puts: 2},
{name: "taken over at its last asking: failed, not asked again", askings: 3, always: true, step: "asked mesh-build-go again (asking 3): x", puts: 0},
} {
t.Run(c.name, func(t *testing.T) { // 1945
r := newRig(t, goBundle)
r.asks.Answer = nil
r.b.Askings = c.askings
var said []AskedAgain
var mu sync.Mutex
r.b.Said = func(e AskedAgain) { mu.Lock(); said = append(said, e); mu.Unlock() }
first := ""
if c.step != "" {
first = "mvl0frv18d1960"
data, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD,
Step: c.step, Asks: map[string]string{delivery.AskGo: first}})
r.st.vals[r.key] = data
}
gone := &goneAsks{}
if !c.byHand {
gone.why = func(a delivery.StepAsk) string {
if c.always || a.ID == first || first == "" {
return "it waits first in line while a node took a later ask"
}
return ""
}
r.b.Gone, r.b.CheckAfter = gone, 5*time.Millisecond
}
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
if c.byHand {
waitForAsks(t, r, 1)
for deadline := time.Now().Add(2 * time.Second); len(r.b.AskAgain(r.key, "stuck since 19:06")) == 0; time.Sleep(time.Millisecond) {
if time.Now().After(deadline) {
t.Fatal("no lane of the key waits")
}
}
}
if c.done {
asks := waitForAsks(t, r, c.puts)
if first == "" && !c.byHand {
first = asks[0].ID
}
last := asks[len(asks)-1]
if v := valueOf(t, r.st, r.key); v.Asks[delivery.AskGo] != last.ID || !strings.Contains(v.Step, "again (asking 2)") {
t.Fatalf("the key = %+v, want building naming the ask asked again %s", v, last.ID)
}
r.asks.End(r.answer(last))
}
r.b.Wait()
v := valueOf(t, r.st, r.key)
if n := len(r.asks.Asks()); n != c.puts {
t.Errorf("%d asks put, want %d", n, c.puts)
}
if c.done && v.State != delivery.BuildDone {
t.Errorf("the key = %+v, want done from the ask asked again", v)
}
if !c.done && (v.State != delivery.BuildFailed || !strings.Contains(v.Error, "could not be answered and was asked")) {
t.Errorf("the key = %+v, want failed saying the ask could not be answered", v)
}
mu.Lock()
defer mu.Unlock()
// Note 62: the old message of an ask asked again because its stream gave it up is dropped from the stream.
if gone.mu.Lock(); c.done && !c.byHand && (len(gone.dropped) != 1 || gone.dropped[0] != first) {
t.Errorf("dropped %v from the stream, want the old ask %s", gone.dropped, first)
}
gone.mu.Unlock()
if c.asking > 0 && (len(said) != 1 || said[0].Asking != c.asking || said[0].New == said[0].Old || said[0].Key != r.key || said[0].Why == "") {
t.Errorf("said %+v, want once at asking %d", said, c.asking)
}
})
}
t.Run("what the bus says of an ask", func(t *testing.T) { // 1945
put := time.Date(2026, 10, 25, 17, 6, 0, 0, time.UTC)
taken := func(kind string, at time.Time) *Taken {
return &Taken{Ask: "later", Kind: kind, At: at, Node: "g14", Started: at}
}
later := taken(delivery.AskDelivery, put.Add(time.Minute))
// ahead is n messages put before the ask, a minute apart, each with step (its state, "" for none).
ahead := func(n int, step string) []Queued {
var qs []Queued
for i := range n {
at := put.Add(-time.Duration(n-i) * time.Minute)
qs = append(qs, Queued{Seq: uint64(i + 1), Ask: fmt.Sprint("a", i), Kind: delivery.AskDelivery, At: at, PutAt: at, Step: step})
}
return qs
}
seen := func(qs []Queued, last *Taken) AskSeen {
return AskSeen{Stream: "S", Kind: delivery.AskDelivery, At: put, OnStream: true, PutAt: put, Ahead: qs, Last: last}
}
for _, c := range []struct {
name string
seen AskSeen
want string // a part of the why, "" when it can be answered
waiting int
}{
{"a node took it", AskSeen{Stream: "S", Step: true}, "", 0},
{"no longer on its stream", AskSeen{Stream: "S"}, "no longer holds it", 0},
{"first in line, a later ask taken", seen(nil, later), "no longer delivers it", 0},
{"first in line, nothing taken since", seen(nil, nil), "", 0},
{"first in line, the ask taken last was asked before it", seen(nil, taken(delivery.AskDelivery, put.Add(-time.Second))), "", 0},
{"two wait ahead, nothing taken since", seen(ahead(2, ""), nil), "", 2},
{"two ahead passed over, the ask not yet", seen(ahead(2, ""), taken(delivery.AskDelivery, put.Add(-time.Second))), "", 0},
// Note 62: seven messages SEAT_NODE_BUILD_GO gave up on sat ahead of a Go ask for five hours.
{"seven given up on ahead, a later ask taken", seen(ahead(7, ""), later), "behind 7 given up on or taken", 0},
{"seven taken ahead (failed as poison), a later ask taken", seen(ahead(7, delivery.StepFailed), later), "no longer delivers it", 0},
{"a check ask taken later proves a delivery ask passed", seen(ahead(7, ""), taken(delivery.AskCheck, put.Add(time.Minute))), "no longer delivers it", 0},
{"a delivery ask taken later does not prove a check ask passed", func() AskSeen { s := seen(ahead(3, ""), later); s.Kind = delivery.AskCheck; return s }(), "", 3},
} {
got := c.seen.Why()
if (c.want == "") != (got == "") || !strings.Contains(got, c.want) || c.seen.Waiting() != c.waiting {
t.Errorf("%s: why %q and %d waiting, want %q and %d", c.name, got, c.seen.Waiting(), c.want, c.waiting)
}
}
if at := time.UnixMilli(1793120000123); !AskedAt(strconv.FormatInt(at.UnixMilli(), 36)+"0a1b2c").Equal(at) || !AskedAt("x").IsZero() {
t.Errorf("AskedAt does not read an ask id's time")
}
})
t.Run("the tool builder_stuck", func(t *testing.T) { // 1945
at := time.Date(2026, 10, 25, 17, 6, 0, 0, time.UTC)
state := func(vals map[string]string) StateRead {
return StateRead{Keys: func() ([]string, error) {
var ks []string
for k := range vals {
ks = append(ks, k)
}
return ks, nil
}, Get: func(k string) (json.RawMessage, time.Time, error) {
if v, ok := vals[k]; ok {
return json.RawMessage(v), at, nil
}
return nil, time.Time{}, nil
}}
}
key, _ := delivery.BuilderBuildKey("m", fp1)
done, _ := delivery.BuilderBuildKey("d", fp1)
building, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD,
Step: "asked mesh-build-go again (asking 2): x", Asks: map[string]string{delivery.AskGo: "mvo53jcm70b5f7", delivery.AskImage: "mvo53jcm70b5f8"}})
finished, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: manifestD, Step: "built"})
running, _ := json.Marshal(delivery.Step{V: delivery.Version, ID: "mvo53jcm70b5f8", Tool: "delivery ask", Node: "g14", State: delivery.StepRunning, Started: at})
_, reask, _ := NewReAsk("m", fp1, "stuck", "shanks.lead", at)
r := StuckReader{Builds: state(map[string]string{key: string(building), done: string(finished)}), ReAsks: state(map[string]string{key: string(reask)}),
Steps: map[string]StateRead{buildrun.GoBuildModule: state(map[string]string{}), ImageBuildModule: state(map[string]string{"g14.mvo53jcm70b5f8": string(running)})},
Now: func() time.Time { return at.Add(time.Hour) }}
out, err := r.Stuck("")
if err != nil {
t.Fatal(err)
}
stuck := out["stuck"].([]Stuck)
if len(stuck) != 1 || stuck[0].Key != key || stuck[0].Age != "1h0m0s" || stuck[0].NoneStep != 1 || stuck[0].ReAsk == nil || stuck[0].ReAsk.Why != "stuck" {
t.Fatalf("stuck = %+v, want the building key alone, an hour old, one ask with no step, with its ask again", stuck)
}
a := stuck[0].Asks
if len(a) != 2 || a[0].Language != delivery.AskGo || a[0].Asking != 2 || a[0].Step != "none" || a[0].Asked.IsZero() ||
a[1].Asking != 1 || !strings.HasPrefix(a[1].Step, "running on g14") {
t.Fatalf("asks = %+v, want go at asking 2 with no step and image at asking 1 running on g14", a)
}
// 1827: on the live shape (3000 build keys over 100 modules, 1 ms a Get, three keys of each module building) a module reads only its
// own keys, the whole answer reads MaxReads values in parallel and says it was cut, and the history (Written) is asked
// only for the rows kept.
var gets, written, inFlight, maxFlight atomic.Int64
vals := map[string]string{}
for i := 0; i < 3000; i++ {
k, _ := delivery.BuilderBuildKey(fmt.Sprintf("mod%02d", i%100), fmt.Sprintf("src2:%064x", i))
vals[k] = string(finished)
if (i/100)%10 == 0 {
vals[k] = string(building)
}
}
slow := state(vals)
get := slow.Get
slow.Get = func(k string) (json.RawMessage, time.Time, error) {
gets.Add(1)
n := inFlight.Add(1)
for m := maxFlight.Load(); n > m && !maxFlight.CompareAndSwap(m, n); m = maxFlight.Load() {
}
time.Sleep(time.Millisecond)
inFlight.Add(-1)
raw, _, err := get(k)
return raw, time.Time{}, err
}
slow.Written = func(string) time.Time { written.Add(1); return at }
for _, row := range []struct {
module string
stuck int
gets int64
cut bool
}{{"mod10", 3, 30, false}, {"", 100, MaxReads, true}} {
gets.Store(0)
written.Store(0)
out, err := StuckReader{Builds: slow, ReAsks: state(map[string]string{}), Steps: map[string]StateRead{}, Now: func() time.Time { return at }}.Stuck(row.module)
if err != nil {
t.Fatal(err)
}
_, cut := out["cut"]
if n := len(out["stuck"].([]Stuck)); n != row.stuck || cut != row.cut || written.Load() != int64(n) || gets.Load() != row.gets {
t.Errorf("%q: %d stuck, %d gets, %d written, cut %v; want %d stuck and gets, written for each, cut %v", row.module, n, gets.Load(), written.Load(), cut, row.stuck, row.cut)
}
}
if maxFlight.Load() < 2 {
t.Errorf("the values were read one at a time")
}
})
time.Sleep(100 * time.Millisecond)
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding {
t.Fatalf("the key = %+v while its ask waits, want building", v)
}
r.asks.End(r.answer(r.asks.Asks()[0]))
r.b.Wait()
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(r.asks.Asks()) != 1 {
t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks()))
}
}
// heldAsks says an ask is held when its id is among held.
@@ -1,290 +0,0 @@
package build
import (
"encoding/json"
"errors"
"fmt"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// The builder's tools (novox/hq issue 1945; the instruction "tools are how a module is spoken to"): builder_stuck reads
// every key that waits on an ask, builder_ask_again asks a key's build again now. The tools bundle serves them through
// the tool runner; the builder's process serves no tool (ADR 0437 §4), so builder_ask_again writes the ask into the
// builder's state reasks, which the builder watches and answers there.
const (
StuckTool = "builder_stuck"
AskAgainTool = "builder_ask_again"
// ReAsksState is the builder's state of asks to ask again: the bucket mesh-delivery-builder_reasks, keyed as the build key.
ReAsksState = "reasks"
ReAsksBucket = delivery.BuilderModule + "_" + ReAsksState
)
// ReAsk is a value of the reasks state: why a person asked the key's build again, who and when, and, once the builder
// read it, its answer.
type ReAsk struct {
V int `json:"v"`
Why string `json:"why"`
By string `json:"by,omitempty"`
At time.Time `json:"at"`
Answer string `json:"answer,omitempty"`
Answered time.Time `json:"answered,omitzero"`
}
// DecodeReAsk reads a reasks value.
func DecodeReAsk(data []byte) (ReAsk, error) {
var r ReAsk
if err := json.Unmarshal(data, &r); err != nil {
return r, err
}
if r.V != 1 {
return r, fmt.Errorf("a reasks value of version %d, not 1", r.V)
}
return r, nil
}
// NewReAsk is the value builder_ask_again writes for a module's fingerprint: its key and the value.
func NewReAsk(module, fp, why, by string, at time.Time) (string, []byte, error) {
key, err := delivery.BuilderBuildKey(module, fp)
if err != nil {
return "", nil, err
}
if strings.TrimSpace(why) == "" {
return "", nil, errors.New("say why the build is asked again (why)")
}
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
return key, data, err
}
// StateRead is a state the tools read: its keys, and a key's value with when it was written (zero when not known).
type StateRead struct {
Keys func() ([]string, error)
Get func(key string) (json.RawMessage, time.Time, error)
// Written is when the key's current value was written, from the state's history: one more round trip, so the tools ask it
// only for a row they keep, and only when Get did not say.
Written func(key string) time.Time
}
// StuckAsk is one ask a key waits on.
type StuckAsk struct {
Language string `json:"language"`
BuildModule string `json:"build_module"`
ID string `json:"id"`
Asking int `json:"asking"`
// Asked is when the ask was put, read from its id; Step is what its build module's steps hold of it: none, or its
// state on its node since its start.
Asked time.Time `json:"asked,omitzero"`
Step string `json:"step"`
}
// Stuck is one build key that waits on an ask.
type Stuck struct {
Key string `json:"key"`
Module string `json:"module"`
FP string `json:"fp"`
State string `json:"state"`
Step string `json:"step"`
By string `json:"by"`
Since time.Time `json:"since,omitzero"`
Age string `json:"age,omitempty"`
Asks []StuckAsk `json:"asks"`
ReAsk *ReAsk `json:"reask,omitempty"`
Unasked bool `json:"unasked,omitempty"`
NoneStep int `json:"asks_with_no_step"`
}
// StuckReader reads what builder_stuck answers.
type StuckReader struct {
Builds StateRead
ReAsks StateRead
// Steps are the build modules' steps states, by build module.
Steps map[string]StateRead
Now func() time.Time
// Logf, when set, gets the one line a call logs: the values it read and the time it took.
Logf func(format string, args ...any)
}
// ReadWorkers is how many values builder_stuck asks the bus for at once, and MaxReads how many it reads in a call (issue 1827:
// every value is a round trip through the runtime, so one after the other a few thousand keys outlast the call's 30 s).
// A copy of the delivery view's and the checker's: a few lines are not worth a group delivery through mesh-sdk.
const (
ReadWorkers = 16
MaxReads = 1000
)
// parallel runs read(i) for i in [0, n), at most ReadWorkers at a time, and answers the first error.
func parallel(n int, read func(i int) error) error {
var (
next atomic.Int64
failed atomic.Bool
once sync.Once
first error
wg sync.WaitGroup
)
for w := 0; w < min(ReadWorkers, n); w++ {
wg.Add(1)
go func() {
defer wg.Done()
for !failed.Load() {
i := int(next.Add(1) - 1)
if i >= n {
return
}
if err := read(i); err != nil {
once.Do(func() { first = err })
failed.Store(true)
}
}
}()
}
wg.Wait()
return first
}
// Stuck is every key building or queued with its asks, oldest first; with module, that module's only.
func (r StuckReader) Stuck(module string) (map[string]any, error) {
started := time.Now()
reads := 0
defer func() {
if r.Logf != nil {
r.Logf("builder_stuck read %d values in %s", reads, time.Since(started).Round(time.Millisecond))
}
}()
keys, err := r.Builds.Keys()
if err != nil {
return nil, err
}
now := time.Now
if r.Now != nil {
now = r.Now
}
// Steps are found by the ask's id, which their key ends in: one index per build module, no value read until an ask names it.
stepKeys := map[string]map[string]string{}
for bm, s := range r.Steps {
ks, err := s.Keys()
if err != nil {
return nil, fmt.Errorf("the steps of %s: %w", bm, err)
}
byID := make(map[string]string, len(ks))
for _, k := range ks {
if _, id, err := delivery.ParseStepKey(k); err == nil {
byID[id] = k
}
}
stepKeys[bm] = byID
}
// Narrowed by the key (the module) before any value is read; the state is in the value, so every matching key is read, in
// parallel, at most MaxReads of them.
type parsed struct{ key, mod, fp string }
var matching []parsed
for _, key := range keys {
mod, fp, err := delivery.ParseBuilderBuildKey(key)
if err != nil || (module != "" && mod != module) {
continue
}
matching = append(matching, parsed{key, mod, fp})
}
sort.Slice(matching, func(i, j int) bool { return matching[i].key < matching[j].key })
cut := len(matching) > MaxReads
total := len(matching)
if cut {
matching = matching[:MaxReads]
}
values := make([]json.RawMessage, len(matching))
ats := make([]time.Time, len(matching))
if err := parallel(len(matching), func(i int) error {
raw, at, err := r.Builds.Get(matching[i].key)
if err == nil {
values[i], ats[i] = raw, at
}
return nil
}); err != nil {
return nil, err
}
reads += len(matching)
out := []Stuck{}
for i, m := range matching {
key, mod, fp := m.key, m.mod, m.fp
raw, at := values[i], ats[i]
if raw == nil {
continue
}
v, err := delivery.DecodeBuilderBuild(raw)
if err != nil || (v.State != delivery.BuildBuilding && v.State != delivery.BuildQueued) {
continue
}
if at.IsZero() && r.Builds.Written != nil {
at = r.Builds.Written(key)
}
s := Stuck{Key: key, Module: mod, FP: fp, State: v.State, Step: v.Step, By: v.By, Since: at, Asks: []StuckAsk{}}
if !at.IsZero() {
s.Age = now().Sub(at).Round(time.Second).String()
}
for language, id := range v.Asks {
bm, _ := delivery.BuildModuleOf(language)
a := StuckAsk{Language: language, BuildModule: bm, ID: id, Asking: askingOf(v.Step, bm), Step: "none"}
if t, ok := idTime(id); ok {
a.Asked = t
}
if st, ok := r.Steps[bm]; ok {
if k, ok := stepKeys[bm][id]; ok {
reads++
if raw, _, err := st.Get(k); err == nil && raw != nil {
if step, err := delivery.DecodeStep(raw); err == nil {
a.Step = fmt.Sprintf("%s on %s since %s", step.State, step.Node, step.Started.UTC().Format(time.RFC3339))
}
}
}
}
if a.Step == "none" {
s.NoneStep++
}
s.Asks = append(s.Asks, a)
}
sort.Slice(s.Asks, func(i, j int) bool { return s.Asks[i].Language < s.Asks[j].Language })
s.Unasked = v.State == delivery.BuildBuilding && len(v.Asks) == 0
if r.ReAsks.Get != nil {
reads++
if raw, _, err := r.ReAsks.Get(key); err == nil && raw != nil {
if ra, err := DecodeReAsk(raw); err == nil {
s.ReAsk = &ra
}
}
}
out = append(out, s)
}
sort.Slice(out, func(i, j int) bool {
if !out[i].Since.Equal(out[j].Since) {
return out[i].Since.Before(out[j].Since)
}
return out[i].Key < out[j].Key
})
answer := map[string]any{"stuck": out, "count": len(out),
"delivery_counts": "not readable by the builder: the build seats' workers' information is granted to the delivery view alone (its backlog); an ask given up on is told by the builder at its check bound and asked again"}
if cut {
answer["cut"], answer["note"] = true, fmt.Sprintf("read %d of %d build keys in key order and stopped; give a module to read its keys alone", MaxReads, total)
}
return answer, nil
}
// StuckBuildModules are the build modules whose steps builder_stuck reads.
var StuckBuildModules = []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, ImageBuildModule}
// idTime is when a step id was made (buildstep.NewID: the time in milliseconds in base 36, then six hex digits).
func idTime(id string) (time.Time, bool) {
if len(id) <= 6 {
return time.Time{}, false
}
ms, err := strconv.ParseInt(id[:len(id)-6], 36, 64)
if err != nil || ms <= 0 {
return time.Time{}, false
}
return time.UnixMilli(ms).UTC(), true
}
+12 -27
View File
@@ -23,21 +23,18 @@ type state struct {
}
type artifact struct {
Name string `json:"name"`
Kind string `json:"kind"`
Language string `json:"language"`
System string `json:"system"`
From string `json:"from"`
Binary string `json:"binary"`
Loads []string `json:"loads"`
Name string `json:"name"`
Kind string `json:"kind"`
Language string `json:"language"`
System string `json:"system"`
From string `json:"from"`
Binary string `json:"binary"`
}
type manifest struct {
Module string `json:"module"`
Version string `json:"version"`
Slug string `json:"slug"`
Tools []string `json:"tools"`
Emits []string `json:"emits"`
Consumes []string `json:"consumes"`
State []state `json:"state"`
Reads []string `json:"reads"`
@@ -87,13 +84,13 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
if m.Module != "mesh-delivery-builder" {
t.Errorf("module = %q", m.Module)
}
if want := []state{{Name: "builds", History: 5}, {Name: "instances", History: 1}, {Name: "queues", History: 1}, {Name: "reasks", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
if want := []state{{Name: "builds", History: 5}, {Name: "instances", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
t.Errorf("state = %+v, want %+v", m.State, want)
}
// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and
// holds no capacity; it puts asks on the three build seats and reads their steps (ADR 0525 §1, §4, §5).
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-controller.assignments",
"mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules"}; !reflect.DeepEqual(sorted(m.Reads), want) {
"mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules"}; !reflect.DeepEqual(sorted(m.Reads), want) {
t.Errorf("reads = %v, want %v", m.Reads, want)
}
if want := []string{"node-build-go", "node-build-image", "node-build-node"}; !reflect.DeepEqual(sorted(m.Uses), want) {
@@ -115,14 +112,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
if m.SecretsOwner != "mesh-delivery-builder" {
t.Errorf("secrets-owner = %q", m.SecretsOwner)
}
// 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again.
if want := []string{"builder_ask_again", "builder_seat_queue", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
if want := []string{"build.asked-again"}; !reflect.DeepEqual(m.Emits, want) {
t.Errorf("emits = %v, want %v", m.Emits, want)
}
for _, k := range []string{"streams", "logs", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} {
for _, k := range []string{"streams", "logs", "emits", "claims", "provides", "requires", "defines-seats", "tools", "seat", "seats", "bus", "config"} {
if _, ok := top[k]; ok {
t.Errorf("the manifest declares %q: the builder has none", k)
}
@@ -137,7 +127,7 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
procs = append(procs, r)
}
if r["type"] == "tool" || r["type"] == "seat" {
t.Errorf("resource %v: the builder's tools are its bundle's, and it holds no seat", r)
t.Errorf("resource %v: the builder serves no tool and holds no seat", r)
}
}
if len(procs) != 1 {
@@ -159,13 +149,8 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
if _, bus := p["bus"]; bus {
t.Errorf("the process declares a bus: it runs on its own broker credential")
}
if len(m.Build.Artifacts) != 2 {
t.Fatalf("artifacts = %+v, want the program and the tools", m.Build.Artifacts)
}
if a := m.Build.Artifacts[1]; a.Name != "tools" || a.Kind != "bundle" || a.Binary != "mesh-delivery-builder-tools" || !reflect.DeepEqual(a.Loads, []string{"mesh-delivery-builder-tools"}) {
t.Errorf("tools artifact = %+v", a) // 1945
} else if st, err := os.Stat(a.From); err != nil || !st.IsDir() {
t.Errorf("the tools' source %s: %v", a.From, err)
if len(m.Build.Artifacts) != 1 {
t.Fatalf("artifacts = %+v, want the program alone", m.Build.Artifacts)
}
a := m.Build.Artifacts[0]
if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-builder" || !strings.HasSuffix(a.From, a.Binary) || a.Kind != "bundle" || a.Language != "go" {
+1 -23
View File
@@ -2,14 +2,6 @@
"module": "mesh-delivery-builder",
"version": "1",
"slug": "builder",
"tools": [
"builder_ask_again",
"builder_seat_queue",
"builder_stuck"
],
"emits": [
"build.asked-again"
],
"consumes": [
"mesh-build-go.toolchain.reported",
"mesh-build-node.toolchain.reported",
@@ -20,9 +12,7 @@
],
"state": [
{ "name": "builds", "history": 5 },
{ "name": "instances", "history": 1 },
{ "name": "queues", "history": 1 },
{ "name": "reasks", "history": 1 }
{ "name": "instances", "history": 1 }
],
"reads": [
"mesh-build-go.steps",
@@ -31,7 +21,6 @@
"mesh-build-node.steps",
"mesh-build-node.toolchains",
"mesh-controller.assignments",
"mesh-delivery-gate.source",
"mesh-delivery-indexer.candidates",
"mesh-delivery-indexer.manifests",
"mesh-delivery-indexer.modules"
@@ -88,17 +77,6 @@
"system": "arch",
"from": "cmd/mesh-delivery-builder",
"binary": "mesh-delivery-builder"
},
{
"name": "tools",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/mesh-delivery-builder-tools",
"binary": "mesh-delivery-builder-tools",
"loads": [
"mesh-delivery-builder-tools"
]
}
]
}
@@ -7,7 +7,6 @@ import (
"encoding/json"
"fmt"
"os"
"sync/atomic"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
@@ -16,35 +15,8 @@ import (
"mesh-delivery-checker/internal/check"
)
// reads counts the values this call read from the bus: the runtime launches the tools for one call, so the count is the call's.
var reads atomic.Int64
// cutAnswer says in the answer that it was cut, and how to narrow it.
func cutAnswer(answer map[string]any, cut string) map[string]any {
if cut != "" {
answer["cut"], answer["note"] = true, cut
}
return answer
}
// logged runs every tool with one line to stderr per call: the tool, the values it read and the time it took.
func logged(tools []stdio.Tool) []stdio.Tool {
for i := range tools {
name, run := tools[i].Name, tools[i].Run
tools[i].Run = func(args map[string]any) (any, error) {
reads.Store(0)
start := time.Now()
out, err := run(args)
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %s read %d values in %s (error: %v)\n", name, reads.Load(), time.Since(start).Round(time.Millisecond), err)
return out, err
}
}
return tools
}
func read(s stdio.KeptState) check.StateRead {
return check.StateRead{Keys: s.Keys, Get: func(key string) (json.RawMessage, error) {
reads.Add(1)
e, err := s.Get(key)
if err != nil || e == nil {
return nil, err
@@ -128,8 +100,8 @@ func Tools(r check.Reader) []stdio.Tool {
"group": map[string]any{"type": "string", "description": "the delivery group's branch name"}}},
Run: func(args map[string]any) (any, error) {
group, _ := args["group"].(string)
vs, cut, err := r.Verdicts(group)
return cutAnswer(map[string]any{"verdicts": vs}, cut), err
vs, err := r.Verdicts(group)
return map[string]any{"verdicts": vs}, err
}},
{Name: check.PullReadTool,
Description: "The verdict that judges one pull request and the full report of its status mesh/check. Read-only.",
@@ -150,8 +122,8 @@ func Tools(r check.Reader) []stdio.Tool {
Run: func(args map[string]any) (any, error) {
level, _ := args["level"].(string)
digest, _ := args["digest"].(string)
kept, cut, err := r.Kept(level, digest)
return cutAnswer(map[string]any{"kept": kept}, cut), err
kept, err := r.Kept(level, digest)
return map[string]any{"kept": kept}, err
}},
{Name: check.PreviewsReadTool,
Description: "The merge previews the checker holds, each as stored under previews.<repository>.<head>.<trunk>: the merge commit, per module directory " +
@@ -166,17 +138,15 @@ func Tools(r check.Reader) []stdio.Tool {
if head != "" && len(head) < 7 {
return nil, fmt.Errorf("the head %q is too short: give at least 7 characters", head)
}
ps, bad, cut, err := r.Previews(repository, head)
return cutAnswer(map[string]any{"previews": ps, "unreadable": bad}, cut), err
ps, bad, err := r.Previews(repository, head)
return map[string]any{"previews": ps, "unreadable": bad}, err
}},
{Name: check.StatusReadTool,
Description: "What runs: the steps in flight on the build modules by node, the candidates without a verdict for their heads, " +
"and the candidates whose check is owed by kind (waiting_by_kind: new, nodes, and trunk, a re-judge pulled after the other two). A key a " +
"build module left running (started past the step bound, or its build module not running on its node) is listed apart under " +
"left_running with why, and not counted. Nothing holds the checker and it has no capacity of its own: a check is an ask on the " +
"build seats' streams, whose backlog the delivery view shows (novox/hq ADR 0525). held_by_turn names the candidates whose check an approved " +
"merge's turn holds off (novox/hq ADR 0548 §3), with the repository and the merge that waits; the gate's turns tool reads every turn and its " +
"hand-now tool steers one. Read-only.",
"build seats' streams, whose backlog the delivery view shows (novox/hq ADR 0525). Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) { return r.Status() }},
}
@@ -195,8 +165,7 @@ func main() {
r := check.Reader{VerdictState: read(stdio.State("verdicts")), KeptState: read(stdio.State("kept")), CandidateState: read(stdio.State("mesh-delivery-indexer.candidates")),
PreviewState: read(stdio.State("previews")),
StepStates: map[string]check.StateRead{"mesh-build-go": read(stdio.State("mesh-build-go.steps")), "mesh-build-node": read(stdio.State("mesh-build-node.steps"))},
HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound(),
TurnState: read(stdio.State("mesh-delivery-gate.turns"))}
HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound()}
tries := stdio.State(check.TriesState)
source := stdio.State("mesh-delivery-gate.source")
trunks := func() (map[string]string, error) {
@@ -215,7 +184,7 @@ func main() {
}},
Trunks: trunks,
}
if err := stdio.Serve("", logged(append(Tools(r), TryTools(trying)...))); err != nil {
if err := stdio.Serve("", append(Tools(r), TryTools(trying)...)); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err)
os.Exit(1)
}
@@ -305,8 +305,7 @@ func startBed(t *testing.T) *bed {
for name, history := range map[string]int{delivery.CheckerVerdictsBucket: delivery.CheckerVerdictsHistory, delivery.CheckerKeptBucket: delivery.CheckerKeptHistory,
delivery.CheckerOwedBucket: delivery.CheckerOwedHistory, delivery.PreviewsBucket: delivery.PreviewsHistory, delivery.CandidatesBucket: delivery.CandidatesHistory,
delivery.ModulesBucket: delivery.ModulesHistory, delivery.ManifestsBucket: delivery.ManifestsHistory, delivery.SourceBucket: delivery.SourceHistory,
delivery.PresenceBucket: delivery.PresenceHistory, delivery.HaveBucket: delivery.HaveHistory, check.TriesBucket: 1,
delivery.TurnsBucket: delivery.TurnsHistory} {
delivery.PresenceBucket: delivery.PresenceHistory, delivery.HaveBucket: delivery.HaveHistory, check.TriesBucket: 1} {
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: uint8(history)})
if err != nil {
t.Fatal(err)
@@ -508,12 +507,9 @@ func TestCheckerJudgesACandidateAndOwesItsStatus(t *testing.T) { // N6/AC5, 1300
}
}
// TestACheckOrACarryOfEqualInputsRunsNothing: a second candidate (another head, the same digests) is checked with every
// package taken from the first's kept verdicts: no test step runs and no second kept key appears. Then, through the real bus
// (novox/hq ADR 0548 §3, §5): a third candidate is not claimed while the gate's merge turn on its repository stands, and is
// claimed when the turn's key is deleted; and a move of the trunk carries the first verdict on the recheck stream by its
// unchanged digests, running no test, the verdict naming the trunk it was carried from.
func TestACheckOrACarryOfEqualInputsRunsNothing(t *testing.T) { // 1287/AC5, 1288/AC6, 1929/AC2, ADR 0548 §3, §5
// TestACheckOfEqualInputsRunsNothing: a second candidate (another head, the same digests) is checked with every package
// taken from the first's kept verdicts: no test step runs and no second kept key appears.
func TestACheckOfEqualInputsRunsNothing(t *testing.T) { // 1287/AC5, 1288/AC6, 1929/AC2
b := startBed(t)
first := b.candidate(t, "feat/x", pull(repo, 7, head1))
b.start(t, nil)
@@ -530,42 +526,6 @@ func TestACheckOrACarryOfEqualInputsRunsNothing(t *testing.T) { // 1287/AC5, 128
if got := keptCount(t, b); got != kept {
t.Errorf("%d kept keys, was %d", got, kept)
}
// §3: the gate's turn on the repository holds a new check off; its delete claims it.
at := time.Now().UTC()
turn, err := delivery.EncodeTurn(delivery.Turn{Repository: repo, Waiting: []delivery.TurnWait{{Group: "fix/other", Heads: "0123456789ab", Approved: at, Since: at}}})
if err != nil {
t.Fatal(err)
}
tk, _ := delivery.TurnsKey(repo)
if _, err := b.buckets[delivery.TurnsBucket].Put(b.ctx, tk, turn); err != nil {
t.Fatal(err)
}
time.Sleep(300 * time.Millisecond) // the checker hears the turn
third := b.candidate(t, "feat/z", pull(repo, 9, "3333333333333333333333333333333333333333"))
time.Sleep(time.Second)
if _, err := b.buckets[delivery.CheckerVerdictsBucket].Get(b.ctx, third); !errors.Is(err, jetstream.ErrKeyNotFound) {
t.Fatalf("verdicts.%s exists (%v) while a merge turn holds its repository", third, err)
}
if err := b.buckets[delivery.TurnsBucket].Delete(b.ctx, tk); err != nil {
t.Fatal(err)
}
b.verdict(t, third, ended)
// §5: the trunk moves; the first verdict is carried by its digests, nothing runs.
tests = b.fake.count("mesh-build-go.build_go_test")
const moved = "abababababababababababababababababababab"
b.setTrunk(t, map[string]string{repo: moved})
c := b.verdict(t, first, func(v delivery.Verdict) bool { return ended(v) && v.Trunk[repo] == moved })
if c.State != delivery.VerdictPass || c.CarriedFrom[repo] != trunk || c.Levels[delivery.LevelUnit].Run != 0 || c.Levels[delivery.LevelIntegration].Run != 0 {
t.Fatalf("the carried verdict = %+v, want a pass carried from %s with no test run", c, trunk[:12])
}
if got := b.fake.count("mesh-build-go.build_go_test"); got != tests {
t.Errorf("build_go_test called %d times on the carry, was %d: a carry of unchanged digests runs nothing", got, tests)
}
if o := b.owed(t, 7, head1, "success"); !strings.HasPrefix(o.Text, "pass 111111111111 on abababababab | carried from aaaaaaaaaaaa") {
t.Errorf("the carried status says %q", o.Text)
}
}
func keptCount(t *testing.T, b *bed) int {
@@ -182,8 +182,7 @@ func (r buildReader) Build(ctx context.Context, module, fp2 string) (delivery.Bu
return b, err == nil, err
}
// CommitOf is the commit the indexer's record of the module names: the commit its fingerprint first appeared at, so the source
// of the build the mesh runs (a later merge that leaves the module alone writes no record, the operator 2026-10-25).
// CommitOf is the commit the indexer's record of the module was computed at: the commit of its repository the mesh runs.
func (r *repoModules) CommitOf(module string) string {
r.mu.Lock()
defer r.mu.Unlock()
@@ -1,50 +0,0 @@
package main
import (
"sync"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-checker/internal/check"
)
// lazyTurns is the gate's merge turns as the checker reads them (novox/hq ADR 0548 §3), opened when the gate's bucket is
// there: before each claim it is opened again until it is, so a checker that started before the gate made the bucket holds
// checks from the gate's first turn on, with no timer and no restart. Until then it holds nothing, and says so once, loudly,
// and once again when it opens.
type lazyTurns struct {
turns *check.Turns
open func() error
logf func(format string, args ...any)
mu sync.Mutex
opened bool
said bool
}
// ensure opens the bucket once it is there; a failure is said once.
func (l *lazyTurns) ensure() {
l.mu.Lock()
defer l.mu.Unlock()
if l.opened || l.open == nil {
return
}
if err := l.open(); err != nil {
if !l.said {
l.said = true
l.logf("NO MERGE TURN HOLDS A CHECK: the gate's bucket %s cannot be read (%v); every check is claimed as if no approved merge waited, "+
"and the bucket is opened again before each claim until it is there", delivery.TurnsBucket, err)
}
return
}
l.opened = true
if l.said {
l.logf("the gate's merge turns are read from %s from now on: an approved merge's turn holds checks off its repositories again", delivery.TurnsBucket)
}
}
// Holding opens the bucket when it is not yet, then answers what the turns hold.
func (l *lazyTurns) Holding(repositories []string, heads string) (string, delivery.TurnWait, bool) {
l.ensure()
return l.turns.Holding(repositories, heads)
}
@@ -152,12 +152,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil {
return err
}
// Its own states, the candidates it checks, and the modules and manifests that say which build modules judge a repository:
// a check without one of them would be wrong or unwritten, so each stops the start, naming its grant (issue 1945).
bucket := func(name string) (jetstream.KeyValue, error) {
kv, err := js.KeyValue(ctx, name)
if err != nil {
return nil, fmt.Errorf("the bucket %s: %w", name, err)
}
return kv, nil
}
buckets := map[string]jetstream.KeyValue{}
for _, name := range []string{delivery.CheckerVerdictsBucket, delivery.CheckerKeptBucket, delivery.CheckerOwedBucket, delivery.PreviewsBucket,
delivery.CandidatesBucket, delivery.ModulesBucket, delivery.ManifestsBucket, check.TriesBucket} {
if buckets[name], err = process.EssentialBucket(ctx, js, "checker", name, "judge or keep a check"); err != nil {
if buckets[name], err = bucket(name); err != nil {
return err
}
}
@@ -193,8 +198,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
tools := &buildrun.ToolchainSet{}
presence := &buildrun.PresenceSet{}
trunks := &check.Trunks{}
turns := &check.Turns{}
lazy := &lazyTurns{turns: turns, logf: logger.Printf}
// A check is one ask per pull request and build module on the build seats' check stream, pulled by a build node with a
// free slot (novox/hq ADR 0525 §1, §3): the checker chooses no node and starts no step.
asks := buildrun.JetStreamAsks{JS: js}
@@ -202,7 +205,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
Verdicts: kvStore{buckets[delivery.CheckerVerdictsBucket]},
Owed: kvStore{buckets[delivery.CheckerOwedBucket]},
Trunks: trunks,
Turns: lazy,
Bound: checkBound,
Link: link,
Logf: logger.Printf,
@@ -210,7 +212,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// The builder's builds say how the candidates' fingerprints stand (issue 1684); where its bucket is not there yet the line
// says nothing of the builds, until the checker starts again.
var builds check.BuildStates
if kv := process.OptionalBucket(ctx, js, delivery.BuilderBuildsBucket, "the line says nothing of the builds until the checker starts again", logger.Printf); kv != nil {
switch kv, err := js.KeyValue(ctx, delivery.BuilderBuildsBucket); {
case errors.Is(err, jetstream.ErrBucketNotFound):
logger.Printf("the bucket %s does not exist yet: the line says nothing of the builds until the checker starts again", delivery.BuilderBuildsBucket)
case err != nil:
return fmt.Errorf("the bucket %s: %w", delivery.BuilderBuildsBucket, err)
default:
builds = buildReader{kvStore{kv}}
}
// The assigner's compositions (ADR 0528 clause 5, issue 1297): their bucket is looked up at each check until it is
@@ -239,7 +246,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
}
repos := &repoModules{manifests: kvStore{buckets[delivery.ManifestsBucket]}, logf: logger.Printf}
ch.Runs = repos // the node-engine a check reads beside at its running commit (ADR 0548 §6)
ch.Judge = newJudge(judgeDeps{
asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: stepBound}, tools: tools, trunks: trunks,
kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]},
@@ -284,75 +290,55 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// the assigner's own presence says whether a composition is being made (compositionWatch, ADR 0537 §4). A check is no
// longer held for a node: its asks wait in the build seats' streams, which redeliver a lost node's ask (ADR 0525 §6,
// issue 1912), so the node-engine's have is not watched here.
if kv := process.OptionalBucket(ctx, js, delivery.PresenceBucket, "no node counts as online until the checker starts again", logger.Printf); kv != nil {
if pw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { presence.Put(e.Key(), e.Value(), !opOf(e)) }); err != nil {
process.NotWatched(logger.Printf, delivery.PresenceBucket, err, "no node counts as online until the checker starts again")
} else {
go func() {
errs <- follow(ctx, pw, func(e jetstream.KeyValueEntry) {
if presence.Put(e.Key(), e.Value(), !opOf(e)) {
ch.NodesChanged(ctx) // a node came online or went: a check whose forge was not heard may now be heard
}
})
}()
switch kv, err := js.KeyValue(ctx, delivery.PresenceBucket); {
case errors.Is(err, jetstream.ErrBucketNotFound):
logger.Printf("the bucket %s does not exist yet: no node counts as online until the checker starts again", delivery.PresenceBucket)
case err != nil:
return fmt.Errorf("the bucket %s: %w", delivery.PresenceBucket, err)
default:
pw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { presence.Put(e.Key(), e.Value(), !opOf(e)) })
if err != nil {
return err
}
go func() {
errs <- follow(ctx, pw, func(e jetstream.KeyValueEntry) {
if presence.Put(e.Key(), e.Value(), !opOf(e)) {
ch.NodesChanged(ctx) // a node came online or went: a check whose forge was not heard may now be heard
}
})
}()
}
// The gate's source says the trunk head of each repository, which a pull request is merged onto. Where it is not there
// yet no trunk is known, so nothing is checked.
if kv := process.OptionalBucket(ctx, js, delivery.SourceBucket, "no trunk is known, so nothing is checked until the checker starts again", logger.Printf); kv != nil {
switch kv, err := js.KeyValue(ctx, delivery.SourceBucket); {
case errors.Is(err, jetstream.ErrBucketNotFound):
logger.Printf("the bucket %s does not exist yet: no trunk is known until the checker starts again", delivery.SourceBucket)
case err != nil:
return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err)
default:
sw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) {
if e.Key() == delivery.SourceMainKey {
trunks.Put(e.Value(), !opOf(e))
}
})
if err != nil {
process.NotWatched(logger.Printf, delivery.SourceBucket, err, "no trunk is known, so nothing is checked until the checker starts again")
} else {
go func() {
errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) {
if e.Key() == delivery.SourceMainKey && trunks.Put(e.Value(), !opOf(e)) {
ch.TrunkMoved(ctx) // the trunk moved: its statuses go pending, and a verdict for the old trunk is judged again
}
})
}()
}
}
// The gate's merge turns (novox/hq ADR 0548 §3): no check is claimed of a repository an approved merge waits on, and the
// delete of its turn claims it. Read before the candidates, so no check starts past a turn written before this start.
// Where the gate's bucket is not there yet it is read as empty, and opened again before each claim until it is there
// (lazyTurns), so a gate delivered after the checker started holds checks from its first turn on.
lazy.open = func() error {
kv, err := js.KeyValue(ctx, delivery.TurnsBucket)
if err != nil {
return err
}
tw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { turns.Put(e.Key(), e.Value(), !opOf(e)) })
if err != nil {
return err
}
go func() {
errs <- follow(ctx, tw, func(e jetstream.KeyValueEntry) {
if turns.Put(e.Key(), e.Value(), !opOf(e)) {
ch.Reconsider(ctx) // a turn went or changed: a check it held is claimed now
errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) {
if e.Key() == delivery.SourceMainKey && trunks.Put(e.Value(), !opOf(e)) {
ch.TrunkMoved(ctx) // the trunk moved: its statuses go pending, and a verdict for the old trunk is judged again
}
})
}()
return nil
}
lazy.ensure()
// The modules say which repository holds a TypeScript module, so the Node build module judges it too.
mw, err := settle(ctx, buckets[delivery.ModulesBucket], func(e jetstream.KeyValueEntry) { repos.Put(ctx, e.Key(), e.Value(), !opOf(e)) })
if err != nil {
return err
}
go func() {
errs <- follow(ctx, mw, func(e jetstream.KeyValueEntry) {
before := repos.CommitOf(check.NodeEngineModule)
repos.Put(ctx, e.Key(), e.Value(), !opOf(e))
if after := repos.CommitOf(check.NodeEngineModule); after != before {
ch.TrunkMoved(ctx) // the node-engine the mesh runs moved: a verdict that read it beside is carried (ADR 0548 §6)
}
})
errs <- follow(ctx, mw, func(e jetstream.KeyValueEntry) { repos.Put(ctx, e.Key(), e.Value(), !opOf(e)) })
}()
each := func(e jetstream.KeyValueEntry) {
if opOf(e) {
@@ -28,11 +28,8 @@ import (
//
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
// key <node>.<module>) hold an open assignment of its build module on its node, and only once it was written since that
// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report
// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node
// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran
// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot
// key <node>.<module>) hold an open assignment of its build module on its node; it is passed on as deleted otherwise,
// and passed on again when the module is assigned there again. Until the assignments are settled (or where they cannot
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
// refusal is the guard of novox/hq ADR 0465, and it stays.
type toolchainGate struct {
@@ -41,14 +38,14 @@ type toolchainGate struct {
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
logf func(format string, a ...any)
known bool // the assignments are settled
standing map[string]time.Time // <node>.<module> with an open assignment -> since when it stands
standing map[string]bool // <node>.<module> with an open assignment
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
left map[string]string // <module>/<key> -> why it was said left out, so each reason is said once
left map[string]bool // <module>/<key> said as left out, so it is said once
}
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{},
raw: map[string]map[string][]byte{}, left: map[string]string{}}
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]bool{},
raw: map[string]map[string][]byte{}, left: map[string]bool{}}
for _, m := range modules {
g.modules[m] = true
}
@@ -70,31 +67,20 @@ func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool
return g.pass(module, key)
}
// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers
// whether what it passed on changed.
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool {
// Assignment takes one change of the controller's assignments in and answers whether what it passed on changed.
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool) bool {
node, module, ok := strings.Cut(key, ".")
if !ok || !g.modules[module] {
return false
}
assigned, open := openAssignment(value)
open = open && !deleted
open := !deleted && openAssignment(value)
g.mu.Lock()
defer g.mu.Unlock()
was, stood := g.standing[key]
since := assigned
if since.IsZero() {
// A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it.
since = at
if stood {
since = was
}
}
if open == stood && (!open || since.Equal(was)) {
if g.standing[key] == open {
return false
}
if open {
g.standing[key] = since
g.standing[key] = true
} else {
delete(g.standing, key)
}
@@ -125,58 +111,37 @@ func (g *toolchainGate) Settled() bool {
return changed
}
// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not
// known), deleted otherwise. The caller holds mu.
// pass hands one key on as it stands: its value when its node is assigned (or the assignments are not known), deleted
// otherwise. The caller holds mu.
func (g *toolchainGate) pass(module, key string) bool {
value, held := g.raw[module][key]
if !held {
return g.forward(module, key, nil, true)
}
node, _, err := delivery.ParseToolchainKey(key)
why := ""
if err == nil && g.known {
why = g.leftOut(module, node, value)
}
if why == "" {
if err != nil || !g.known || g.standing[node+"."+module] {
delete(g.left, module+"/"+key)
return g.forward(module, key, value, false)
}
if g.left[module+"/"+key] != why {
g.left[module+"/"+key] = why
g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why)
if !g.left[module+"/"+key] {
g.left[module+"/"+key] = true
g.logf("%s.%s is left out of the agreed toolchain: %s has no assignment on %s, so it is no builder (issue 1940)",
delivery.ToolchainsBucket(module), key, module, node)
}
return g.forward(module, key, nil, true)
}
// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module,
// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse.
// The caller holds mu.
func (g *toolchainGate) leftOut(module, node string, value []byte) string {
since, open := g.standing[node+"."+module]
if !open {
return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node)
}
tc, err := delivery.DecodeToolchain(value)
if err == nil && tc.At.Before(since) {
return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again",
tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node)
}
return ""
}
// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not
// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when
// it was assigned, zero where the value does not say.
func openAssignment(value []byte) (time.Time, bool) {
// openAssignment is whether a value of the controller's assignments holds an assignment that stands: one that has not
// ended and has no state (a pending one is not on the machine yet; mesh-controller internal/chain Assignment).
func openAssignment(value []byte) bool {
var a struct {
Assigned time.Time `json:"assigned"`
Ended time.Time `json:"ended"`
State string `json:"state"`
Ended time.Time `json:"ended"`
State string `json:"state"`
}
if json.Unmarshal(value, &a) != nil {
return time.Time{}, false
return false
}
return a.Assigned, a.Ended.IsZero() && a.State == ""
return a.Ended.IsZero() && a.State == ""
}
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
@@ -190,19 +155,17 @@ func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate,
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
return errs, nil
}
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e)) }
w, err := settle(ctx, kv, each)
if err != nil {
// Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read.
logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
return errs, nil
return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err)
}
if gate.Settled() {
changed(ctx)
}
go func() {
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) {
if gate.Assignment(e.Key(), e.Value(), !opOf(e)) {
changed(ctx)
}
})
@@ -3,19 +3,10 @@ package main
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
import (
"bytes"
"context"
"errors"
"fmt"
"log"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
@@ -53,10 +44,10 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
}
for _, node := range []string{"g14", "novox", "shanks"} {
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{})
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false)
}
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{})
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false)
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false) // another module's: no matter
if !gate.Settled() {
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
}
@@ -72,20 +63,20 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
t.Fatal("an unassigned node's report changed what was passed on")
}
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) {
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false) {
t.Fatal("assigning ace again passed nothing on")
}
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
t.Fatalf("builders that run different versions still disagree; got %q", none)
}
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) {
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false) {
t.Fatal("a pending assignment is no assignment on the machine")
}
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
t.Fatalf("got %q, %q", v, none)
}
gate.Assignment("ace.mesh-build-node", nil, true, time.Time{})
gate.Assignment("ace.mesh-build-node", nil, true)
if v, _ := agreedOf(set); v != "v26.12.1" {
t.Fatalf("got %q", v)
}
@@ -97,122 +88,25 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
if n := len(set.Entries("typescript")); n != 2 {
t.Fatalf("novox and shanks are left; got %d entries", n)
}
// 1945 (a part exits at start for a bucket it works without): a bus that refuses the assignments' watch, or every
// toolchains bucket, leaves the watches unstarted and the part running, said; a refused bucket it cannot work without
// is an error naming the bucket and its grant.
var logs bytes.Buffer
logger := log.New(&logs, "", 0)
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments")}
if _, err := watchAssignments(context.Background(), refused, newToolchainGate(nil, set.Put, t.Logf), func(context.Context) {}, logger); err != nil {
t.Errorf("a refused watch of the assignments stopped the start: %v", err)
}
if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil {
t.Errorf("a refused toolchains bucket stopped the start: %v", err)
}
if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") {
t.Errorf("the refusals were not said: %q", logs.String())
}
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "checker", "mesh-delivery-indexer_modules", "work"); err == nil ||
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
}
}
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
type refusingBus struct {
refuseOpen bool
err error
}
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
if r.refuseOpen {
return nil, r.err
}
return refusedWatch{err: r.err}, nil
}
type refusedWatch struct {
jetstream.KeyValue
err error
}
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
return nil, r.err
}
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
return nil, r.err
}
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript
// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node
// writes it again. An assignment whose value names no time stands since the revision that opened it.
// A newly assigned build node that has not reported yet changes nothing; its report counts as it comes.
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again
report := func(t *testing.T, node, version string, at time.Time) (string, []byte) {
key, _ := gateReport(t, node, version)
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
if err != nil {
t.Fatal(err)
}
return key, value
set := &buildrun.ToolchainSet{}
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(string, ...any) {})
gate.Settled()
if gate.Assignment("g14.mesh-build-node", []byte(`{}`), false) {
t.Fatal("an assignment with no report passed something on")
}
for _, c := range []struct {
name string
assignment string // ace's assignment value
revision time.Time // when the bus wrote it
at time.Time // when ace wrote its report
counts bool
}{
{"no report yet, then one", `{}`, time.Time{}, time.Now(), true},
{"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false},
{"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false},
{"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true},
} {
t.Run(c.name, func(t *testing.T) {
set := &buildrun.ToolchainSet{}
var said []string
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
for _, node := range []string{"g14", "novox", "shanks"} {
gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{})
key, value := gateReport(t, node, "v26.12.1")
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
}
gate.Settled()
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) {
t.Fatal("an assignment with no report passed something on")
}
key, value := report(t, "ace", "v26.12.0", c.at)
if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts {
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
}
// A rewrite of the assignment that does not move it changes nothing, and nothing is said again.
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) {
t.Fatal("the same assignment written again changed what was passed on")
}
v, none := agreedOf(set)
if c.counts {
if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none)
}
return
}
if v != "v26.12.1" || none != "" {
t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none)
}
if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") {
t.Fatalf("said %q, want once why ace's report is left out", said)
}
// ace writes its report again since it was assigned: it counts.
key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute))
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
t.Fatal("ace's new report was not passed on")
}
if n := len(set.Entries("typescript")); n != 4 {
t.Fatalf("four builders report; got %d", n)
}
})
key, value := gateReport(t, "g14", "v26.12.1")
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
t.Fatal("an assigned node's report was not passed on")
}
key, value = gateReport(t, "ace", "v26.12.0")
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
t.Fatal("a report of a node with no assignment key was passed on after the assignments settled")
}
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
t.Fatalf("got %q, %q", v, none)
}
}
@@ -8,8 +8,6 @@ import (
"strings"
"sync"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
@@ -65,14 +63,11 @@ func (w *toolchainWatch) ensure(module string) error {
w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module)
return nil
case err != nil:
// Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the module's next report.
w.logger.Printf("the bucket %s cannot be read (%v): %s's toolchains are not heard until its next report", name, err, module)
return nil
return fmt.Errorf("the bucket %s: %w", name, err)
}
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.put(module, e.Key(), e.Value(), !opOf(e)) })
if err != nil {
process.NotWatched(w.logger.Printf, name, err, fmt.Sprintf("%s's toolchains are not heard until its next report", module))
return nil
return err
}
w.watched[module] = true
go func() {
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-checker
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026111253-9c61918ac36f
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232922-99407c37df95
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+8 -2
View File
@@ -1,5 +1,11 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026111253-9c61918ac36f h1:9+TZpNq8Ifj1+qmbidE6l/7qkbua4yLdzRMrezNvf1A=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026111253-9c61918ac36f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024214154-bc16740feb0d h1:oJREw0ggKqJWJ5DEISSL0i2ZC5XqdXAidxo9NJ74TCc=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024214154-bc16740feb0d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024231716-571ac521f053 h1:8Kbp3aJG3jGndDOfmGdMSDgJVEcc1hyu921hVnnlnlQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024231716-571ac521f053/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232250-9b408db24a58 h1:4Damtdqlf6u6nJhvMA99peDy1jmjixtcwFXJ4bWEkLk=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232250-9b408db24a58/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232922-99407c37df95 h1:I8zbgGzRjmgYydPTWo8hmaZW/P/g+qGDUfu3E32/pkU=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232922-99407c37df95/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -1,355 +0,0 @@
package check
import (
"context"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// One check per delivery (novox/hq ADR 0548): the checker's part. A merge turn holds new checks off its repository (§3);
// a trunk or sibling move never ends a running check and carries its verdict once it ended (§1, §5, §6); only a head git
// refused gets a whole check after a move (§7); the line of a carried verdict says so (§8).
// turnOf is a turn's value on a repository for a delivery of the heads hash.
func turnOf(t testing.TB, repo, group, heads string) []byte {
t.Helper()
at := time.Date(2026, 10, 26, 9, 0, 0, 0, time.UTC)
data, err := delivery.EncodeTurn(delivery.Turn{Repository: repo, Waiting: []delivery.TurnWait{{Group: group, Heads: heads, Approved: at, Since: at}}})
if err != nil {
t.Fatal(err)
}
return data
}
func turnKey(t testing.TB, repo string) string {
t.Helper()
k, err := delivery.TurnsKey(repo)
if err != nil {
t.Fatal(err)
}
return k
}
// waitFor polls a condition the checker's goroutines make true, failing after two seconds.
func waitFor(t testing.TB, what string, ok func() bool) {
t.Helper()
for deadline := time.Now().Add(2 * time.Second); !ok(); time.Sleep(time.Millisecond) {
if time.Now().After(deadline) {
t.Fatalf("%s did not happen", what)
}
}
}
// pullWithFP is a pull request touching the module m at the fingerprint `src2:` and 64 times the hex digit d.
func pullWithFP(repo string, number int, head string, d string) delivery.CandidatePull {
p := pullOf(repo, number, head)
p.Modules = []string{"m"}
p.Fps = map[string]delivery.CandidateModule{"m": {FP2: "src2:" + strings.Repeat(d, 64)}}
return p
}
func TestOneCheckPerDelivery(t *testing.T) {
ctx := context.Background()
for name, run := range map[string]func(t *testing.T){
// §3: no claim while a turn names one of its repositories; a claim on the turn's delete event.
"a turn on its repository holds a new check off until the turn goes": func(t *testing.T) {
j := passJudge()
r := newRig(t, j)
turns := &Turns{}
r.ch.Turns = turns
turns.Put(turnKey(t, repoA), turnOf(t, repoA, "fix/other", "0123456789ab"), false)
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
if j.count() != 0 || len(r.verdicts.vals) != 0 {
t.Fatalf("judged %d times with a turn on %s, verdicts %v", j.count(), repoA, r.verdicts.vals)
}
if held := r.ch.Held()[keyOf(t, "feat/x", cand)]; !strings.Contains(held, repoA) || !strings.Contains(held, "fix/other") {
t.Errorf("the hold is said as %q, want the turn's repository and group", held)
}
if !turns.Put(turnKey(t, repoA), nil, true) {
t.Fatal("the turn's delete changed nothing")
}
r.ch.Reconsider(ctx) // what the watch of turns does on the delete
r.ch.Wait()
if j.count() != 1 || r.verdict(t, keyOf(t, "feat/x", cand)).State != delivery.VerdictPass {
t.Errorf("judged %d times after the turn went, want once", j.count())
}
},
"a turn on another repository, or of its own heads, holds nothing": func(t *testing.T) {
j := passJudge()
r := newRig(t, j)
turns := &Turns{}
r.ch.Turns = turns
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
c, _ := delivery.DecodeCandidate(cand)
turns.Put(turnKey(t, repoSDK), turnOf(t, repoSDK, "fix/other", "0123456789ab"), false)
turns.Put(turnKey(t, repoA), turnOf(t, repoA, "feat/x", GateHeads(c)), false)
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
if j.count() != 1 {
t.Errorf("judged %d times, want once: the turns hold another repository and its own merge", j.count())
}
},
// §3, the race: a check claimed before the gate's turn is judged on the commit it read, never restarted.
"a check claimed before a turn was written runs to its verdict": func(t *testing.T) {
release, started := make(chan struct{}), make(chan struct{})
j := &judgeFunc{f: func(context.Context, *Case, int) (Outcome, error) {
close(started)
<-release
return Outcome{Verdict: passing()}, nil
}}
r := newRig(t, j)
turns := &Turns{}
r.ch.Turns = turns
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
<-started
turns.Put(turnKey(t, repoA), turnOf(t, repoA, "fix/other", "0123456789ab"), false)
r.ch.Reconsider(ctx)
close(release)
r.ch.Wait()
if j.count() != 1 || r.verdict(t, keyOf(t, "feat/x", cand)).State != delivery.VerdictPass {
t.Errorf("judged %d times; the running check is not ended by the turn", j.count())
}
},
// §1, §5: a trunk move during a running check calls the judge once, writes its verdict, keeps the status pending (a pass
// on an old trunk is never shown), then asks one carry, on the recheck stream, and no check.
"a trunk move during a check ends nothing and asks one carry after it": func(t *testing.T) {
var r *rig
var kinds []string
var from []map[string]string
judged := make(chan struct{})
j := &judgeFunc{f: func(ctx context.Context, c *Case, call int) (Outcome, error) {
kinds, from = append(kinds, c.Kind), append(from, c.CarriedFrom)
if call == 1 {
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) // the trunk moves while it runs
r.ch.TrunkMoved(ctx) // as the watch of the source says it
close(judged)
} else if s := owedState(t, r, repoA, 7, head1); s != "pending" {
t.Errorf("the status during the carry is %q, want pending: the pass on the old trunk is never shown", s)
}
return Outcome{Verdict: passing()}, nil
}}
r = newRig(t, j)
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
<-judged
r.ch.Wait()
v := r.verdict(t, keyOf(t, "feat/x", cand))
if j.count() != 2 || kinds[0] != KindNew || kinds[1] != KindTrunk || from[0] != nil || from[1][repoA] != trunkA {
t.Fatalf("judged %d times as %v carried from %v; want a whole check, then one carry from %s", j.count(), kinds, from, trunkA[:12])
}
if v.State != delivery.VerdictPass || v.Trunk[repoA] != trunkA2 || v.CarriedFrom[repoA] != trunkA {
t.Errorf("the verdict is %s on %v carried from %v; want a pass on the new trunk carried from the old", v.State, v.Trunk, v.CarriedFrom)
}
if s := owedState(t, r, repoA, 7, head1); s != "success" {
t.Errorf("the status after the carry is %q, want success", s)
}
if r.verdicts.creates[keyOf(t, "feat/x", cand)] != 1 {
t.Error("the verdict key was made again: a carry rewrites it")
}
},
// §5: the composition is kept when no fingerprint of a module it composed changed, and runs again otherwise.
"a carry keeps a passed composition while the fingerprints stay the same": func(t *testing.T) {
var kept []bool
j := &judgeFunc{f: func(_ context.Context, c *Case, _ int) (Outcome, error) {
kept = append(kept, c.Composition != nil)
v := passing()
v.Composition = &delivery.Composition{State: delivery.CompositionPass, SDK: "v0.1.14-x", FPSet: c.FPSet}
return Outcome{Verdict: v}, nil
}}
r := newRig(t, j)
cand := candidateOf(t, "feat/x", pullWithFP(repoA, 7, head1, "a"))
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS})
r.ch.TrunkMoved(ctx) // the indexer's fingerprints of the candidate stay the same
r.ch.Wait()
r.ch.Candidate(ctx, "feat%2Fx", candidateOf(t, "feat/x", pullWithFP(repoA, 7, head1, "b"))) // a fingerprint changed with the move
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS})
r.ch.TrunkMoved(ctx)
r.ch.Wait()
if len(kept) != 3 || kept[0] || !kept[1] || kept[2] {
t.Errorf("the composition was kept in the checks %v; want composed, kept on the carry, composed again once a fingerprint changed", kept)
}
},
// §6: a sibling's move puts the verdict out of date, its status pending saying it is carried, and the carry's verdict
// names the sibling's old commit; the status follows the carry's verdict. Which packages run is the digest's (only
// those that name MESH_CHECK_BESIDE change with a sibling), the build module's part.
"a sibling's move is carried and the status follows the carry": func(t *testing.T) {
var r *rig
var from map[string]string
j := &judgeFunc{f: func(_ context.Context, c *Case, call int) (Outcome, error) {
if call == 2 {
from = c.CarriedFrom
if s := owedState(t, r, repoA, 7, head1); s != "pending" {
t.Errorf("the status during the sibling's carry is %q, want pending", s)
}
}
return Outcome{Verdict: passing(), Beside: map[string]string{catalogueRepository: r.trunks.Heads()[catalogueRepository]}}, nil
}}
r = newRig(t, j)
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS, catalogueRepository: trunkS})
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
if b := r.verdict(t, keyOf(t, "feat/x", cand)).Beside; b[catalogueRepository] != trunkS {
t.Fatalf("the verdict names the siblings %v, want the catalogue at %s", b, trunkS[:12])
}
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS, catalogueRepository: trunkS2})
r.ch.TrunkMoved(ctx)
r.ch.Wait()
v := r.verdict(t, keyOf(t, "feat/x", cand))
if j.count() != 2 || from[catalogueRepository] != trunkS || v.CarriedFrom[catalogueRepository] != trunkS || v.Beside[catalogueRepository] != trunkS2 {
t.Errorf("judged %d times, carried from %v, verdict beside %v carried from %v; want one carry over the catalogue's move", j.count(), from, v.Beside, v.CarriedFrom)
}
if s := owedState(t, r, repoA, 7, head1); s != "success" {
t.Errorf("the status after the carry is %q, want success", s)
}
},
// §7: a carry whose merge conflicts fails naming the files and asks no check; a later move onto which the head merges
// cleanly asks a whole check, not a carry.
"a carry git refuses fails naming the files, and the next move asks a whole check": func(t *testing.T) {
var kinds []string
var from []map[string]string
j := &judgeFunc{f: func(_ context.Context, c *Case, call int) (Outcome, error) {
kinds, from = append(kinds, c.Kind), append(from, c.CarriedFrom)
if call == 2 {
return Outcome{Verdict: delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "ace",
Words: repoA + "#7 " + ConflictWords + "go.mod"}}}, nil
}
return Outcome{Verdict: passing()}, nil
}}
r := newRig(t, j)
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS})
r.ch.TrunkMoved(ctx)
r.ch.Wait()
r.ch.Reconsider(ctx) // anything else that looks again asks nothing of a conflict
r.ch.Wait()
if v := r.verdict(t, keyOf(t, "feat/x", cand)); j.count() != 2 || !IsConflictFailure(v) || !strings.Contains(v.Failed.Words, "go.mod") {
t.Fatalf("judged %d times, verdict %+v; want the carry failed naming go.mod and nothing asked after it", j.count(), v.Failed)
}
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS})
r.ch.TrunkMoved(ctx)
r.ch.Wait()
if j.count() != 3 || kinds[1] != KindTrunk || from[1] == nil || kinds[2] != KindNew || from[2] != nil {
t.Errorf("checks %v carried from %v; want a whole check, a carry, then a whole check after the conflict", kinds, from)
}
},
// §8, §6: the line of a carried verdict names the head on the new trunk commit first, as the gate and the merge tool
// read a line, then where it was carried from and what the carry ran and kept; while it is carried it is pending and says so.
"the line of a carry says where it was carried from": func(t *testing.T) {
cs, p := lineCase(t, pullOf(repoA, 7, head1))
v := passVerdict()
v.Trunk = map[string]string{repoA: trunkA}
v.Beside = map[string]string{catalogueRepository: trunkS2}
v.CarriedFrom = map[string]string{repoA: trunkA2, catalogueRepository: trunkS}
line := Line(v, cs, p, nil, nil, nil)
if !strings.HasPrefix(line, "pass 111111111111 on aaaaaaaaaaaa | carried from abababababab, mesh-catalog 555555555555 | unit 12 run") {
t.Errorf("the carried line = %q", line)
}
if head, trunk := checkWordsOf(line); head != head1[:12] || trunk != trunkA[:12] {
t.Errorf("the line names %s on %s first, want the head on the new trunk", head, trunk)
}
cs.CarriedFrom = map[string]string{repoA: trunkA2}
pending := Line(delivery.Verdict{State: delivery.VerdictRunning}, cs, p, nil, nil, nil)
if !strings.HasPrefix(pending, "being carried to 111111111111 on aaaaaaaaaaaa | from abababababab") {
t.Errorf("the pending line of a carry = %q", pending)
}
},
// §3 (review round 1): two approved merges in one turn whose trunk moved each get their carry; neither holds the other's.
"two merges waiting in one turn each get their carry": func(t *testing.T) {
var r *rig
j := &judgeFunc{f: func(context.Context, *Case, int) (Outcome, error) { return Outcome{Verdict: passing()}, nil }}
r = newRig(t, j)
candA, candC := candidateOf(t, "feat/a", pullOf(repoA, 7, head1)), candidateOf(t, "feat/c", pullOf(repoA, 8, head2))
r.ch.Candidate(ctx, "feat%2Fa", candA)
r.ch.Candidate(ctx, "feat%2Fc", candC)
r.ch.Wait()
a, _ := delivery.DecodeCandidate(candA)
c, _ := delivery.DecodeCandidate(candC)
at := time.Date(2026, 10, 26, 9, 0, 0, 0, time.UTC)
turn, err := delivery.EncodeTurn(delivery.Turn{Repository: repoA, Waiting: []delivery.TurnWait{
{Group: "feat/a", Heads: GateHeads(a), Approved: at, Since: at}, {Group: "feat/c", Heads: GateHeads(c), Approved: at.Add(time.Minute), Since: at}}})
if err != nil {
t.Fatal(err)
}
turns := &Turns{}
turns.Put(turnKey(t, repoA), turn, false)
r.ch.Turns = turns
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) // a third merge moves the trunk under both
r.ch.TrunkMoved(ctx)
r.ch.Wait()
for _, cand := range [][]byte{candA, candC} {
group := map[bool]string{true: "feat/a", false: "feat/c"}[string(cand) == string(candA)]
if v := r.verdict(t, keyOf(t, group, cand)); v.Trunk[repoA] != trunkA2 || v.CarriedFrom[repoA] != trunkA {
t.Errorf("%s: verdict on %v carried from %v; want its carry claimed and done", group, v.Trunk, v.CarriedFrom)
}
}
},
// §6 (review round 1): the node-engine read at the commit the mesh runs is named, and a new running commit is carried.
"a new running node-engine is carried": func(t *testing.T) {
runs := runsAt{nodeEngineModule: trunkS}
j := &judgeFunc{f: func(_ context.Context, c *Case, _ int) (Outcome, error) {
return Outcome{Verdict: passing(), Beside: map[string]string{nodeEngineRepo: runs[nodeEngineModule]}}, nil
}}
r := newRig(t, j)
r.ch.Runs = runs
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
r.ch.TrunkMoved(ctx) // nothing moved: nothing is carried
r.ch.Wait()
runs[nodeEngineModule] = trunkS2
r.ch.TrunkMoved(ctx) // what the indexer's watch does when mesh-host's running commit changes
r.ch.Wait()
v := r.verdict(t, keyOf(t, "feat/x", cand))
if j.count() != 2 || v.CarriedFrom[nodeEngineRepo] != trunkS || v.Beside[nodeEngineRepo] != trunkS2 {
t.Errorf("judged %d times, verdict beside %v carried from %v; want one carry over the node-engine's new running commit", j.count(), v.Beside, v.CarriedFrom)
}
},
// §5: a carry that fails names the move it failed on.
"a carry that fails names the move": func(t *testing.T) {
j := &judgeFunc{f: func(_ context.Context, c *Case, call int) (Outcome, error) {
if call == 1 {
return Outcome{Verdict: passing()}, nil
}
return Outcome{Verdict: delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Test: "unit: m/a.TestX", Words: "want 4"}}}, nil
}}
r := newRig(t, j)
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS})
r.ch.TrunkMoved(ctx)
r.ch.Wait()
v := r.verdict(t, keyOf(t, "feat/x", cand))
if v.State != delivery.VerdictFail || v.CarriedFrom != nil || !strings.HasPrefix(v.Failed.Words, "want 4") ||
!strings.Contains(v.Failed.Words, repoA+" "+trunkA[:12]+" to "+trunkA2[:12]) {
t.Errorf("the failed carry is %s carried from %v with words %q; want the test's words, then the move", v.State, v.CarriedFrom, v.Failed.Words)
}
},
} {
t.Run(name, run)
}
}
// checkWordsOf is the head and trunk a line names first, as the gate reads it (mesh-delivery-gate onestatus.go checkLine).
func checkWordsOf(line string) (string, string) {
for _, part := range strings.Split(line, " | ") {
f := strings.Fields(part)
for i := 0; i+2 < len(f); i++ {
if f[i+1] == "on" && len(f[i]) == 12 && len(f[i+2]) == 12 {
return f[i], f[i+2]
}
}
}
return "", ""
}
@@ -0,0 +1,96 @@
package check
import (
"context"
"fmt"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// countingAsks puts asks through inner, and answers no wait until want asks are put (or a bound passes, when it says
// how many were): a group whose asks are put one after another, each waited for, never gets past its first.
type countingAsks struct {
inner Asks
want int
mu sync.Mutex
puts int
all chan struct{}
}
func (c *countingAsks) Put(ctx context.Context, a delivery.StepAsk) error {
if err := c.inner.Put(ctx, a); err != nil {
return err
}
c.mu.Lock()
c.puts++
if c.puts == c.want {
close(c.all)
}
c.mu.Unlock()
return nil
}
func (c *countingAsks) Wait(ctx context.Context, a delivery.StepAsk) (delivery.Step, error) {
select {
case <-c.all:
case <-time.After(time.Second):
c.mu.Lock()
defer c.mu.Unlock()
return delivery.Step{}, fmt.Errorf("%d of %d asks were put before the first was waited for", c.puts, c.want)
}
return c.inner.Wait(ctx, a)
}
// 1921/S8: a group's asks are all put before any is waited for, so its members run side by side on the build nodes;
// the verdict is judged in member order as before.
func TestAGroupPutsEveryMembersAskBeforeTheFirstAnswer(t *testing.T) {
b := newBench(t)
counting := &countingAsks{inner: b.ex.Asks, want: 3, all: make(chan struct{})}
b.ex.Asks = counting
cs := caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2), pullOf("novox/mesh-controller", 3, head1))
v := b.judge(t, cs).Verdict
if v.State != delivery.VerdictPass {
t.Fatalf("verdict = %+v", v)
}
if counting.puts != 3 {
t.Errorf("%d asks were put, want 3", counting.puts)
}
}
// 1921/S10: the levels of an ask run side by side, so what of the ask's time is its fetch (Took.Clone) is the ask's time
// less its longest level, not less the sum of its levels.
func TestTheCloneTimeTakesTheLongestLevelOffNotTheSum(t *testing.T) {
b := newBench(t)
b.test = func(c stepCall) (any, error) {
r := captured[Tested](t, "go-test-slow")
r.Level, _ = c.args["level"].(string)
r.Elapsed, r.Setup, r.Lint = 1.5, 0, 0 // the ask's step takes 2 s (bench.answer)
return r, nil
}
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
if v.State != delivery.VerdictPass || v.Took.Clone != 500 {
t.Errorf("clone = %d ms, want 500 (2 s less the longest level's 1.5 s): %+v", v.Took.Clone, v)
}
}
// ADR 0542 §2: a lint finding, which the Go build module marks as the lint step's, is reported as the lint step's failure,
// not as the unit level's.
func TestALintFindingIsTheLintStepsFailure(t *testing.T) {
b := newBench(t)
b.test = func(c stepCall) (any, error) {
r := captured[Tested](t, "go-test-slow")
r.Level, _ = c.args["level"].(string)
if r.Level == delivery.LevelUnit {
r.Passed = false
r.Failed = []Failure{{Package: ".", Test: "gofmt", Step: "lint", Output: "not as gofmt writes them (run gofmt -w): a.go"}}
}
return r, nil
}
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
if v.State != delivery.VerdictFail || v.Failed.Test != "lint: ..gofmt" {
t.Errorf("failed = %+v, want the lint step's failure", v.Failed)
}
}
@@ -127,9 +127,6 @@ func (ch *Checker) Composed(ctx context.Context, comp delivery.CompositionVerdic
return
}
withComposition(&v, comp)
if v.Composition != nil && v.Composition.State != delivery.CompositionNotRun {
v.Composition.FPSet = cs.FPSet
}
data, err := delivery.EncodeVerdict(v)
if err != nil {
ch.logf("%s: the verdict with its composition cannot be written: %v", cs.Key, err)
@@ -12,18 +12,8 @@ type BesideRef struct {
Repository string `json:"repository"`
Commit string `json:"commit"`
Ref string `json:"ref,omitempty"`
// atTrunk says the sibling was read at its trunk's head, so a move of that trunk puts the verdict out of date and is
// carried (novox/hq ADR 0548 §6). Never sent: the node-engine at the commit the mesh runs, mesh-sdk at a pin and a
// group's member at its head do not move with a trunk.
atTrunk bool
// runs says the sibling was read at the commit the mesh runs (the node-engine, the indexer's record of mesh-host): a new
// running commit puts the verdict out of date and is carried (ADR 0548 §6). Never sent.
runs bool
}
// NodeEngineModule is the module whose running commit the node-engine sibling is read at.
const NodeEngineModule = nodeEngineModule
const (
catalogueRepository = "novox/mesh-catalog"
nodeEngineRepo = "novox/mesh-host"
@@ -61,21 +51,18 @@ func (j *Examiner) besideFor(ctx context.Context, c *Case, p Pull) ([]BesideRef,
switch {
case repo == catalogueRepository:
if head := heads[repo]; head != "" {
out = append(out, BesideRef{Repository: repo, Commit: head, atTrunk: c.member(repo) == nil})
out = append(out, BesideRef{Repository: repo, Commit: head})
}
case repo == nodeEngineRepo:
commit := ""
if j.Runs != nil {
commit = j.Runs.CommitOf(nodeEngineModule)
}
atTrunk := false
if commit == "" {
commit, atTrunk = heads[repo], true
commit = heads[repo]
}
if commit != "" {
// Named in the verdict either way (ADR 0548 §6, issue 568's own example): at the commit the mesh runs, a new
// running commit puts the verdict out of date; at the trunk, a trunk move does (Checker.besideNow).
out = append(out, BesideRef{Repository: repo, Commit: commit, atTrunk: atTrunk, runs: !atTrunk})
out = append(out, BesideRef{Repository: repo, Commit: commit})
}
case repo == SDKRepository && p.Repository == controllerRepo && j.Forge != nil:
text, err := j.Forge.File(ctx, p.Repository, "go.mod", p.Head)
@@ -95,7 +82,7 @@ func (j *Examiner) besideFor(ctx context.Context, c *Case, p Pull) ([]BesideRef,
if member := c.member(repo); member != nil {
out = append(out, BesideRef{Repository: repo, Commit: member.Head, Ref: refOf(member.Number)})
} else if head := heads[repo]; head != "" {
out = append(out, BesideRef{Repository: repo, Commit: head, atTrunk: true})
out = append(out, BesideRef{Repository: repo, Commit: head})
} else if repo == controllerRepo && p.Repository == nodeEngineRepo {
// The node-engine's step-nine test judges the controller's own manifest and fails without it (issue 1295): a
// checker that cannot place it says so, where placing none left that test to skip.
@@ -121,13 +108,3 @@ func refOf(number int) string {
}
return "refs/pull/" + strconv.Itoa(number) + "/head"
}
// besideAtTrunk are the siblings of refs read at their trunk's head that the case does not merge, by repository: what a
// verdict names as Beside (ADR 0548 §6).
func besideAtTrunk(c *Case, refs []BesideRef, into map[string]string) {
for _, r := range refs {
if _, merged := c.Trunk[r.Repository]; (r.atTrunk || r.runs) && !merged {
into[r.Repository] = r.Commit
}
}
}
@@ -1,79 +0,0 @@
package check
import (
"fmt"
"sort"
"sync"
"sync/atomic"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// ReadWorkers is how many values a read tool asks the bus for at once. Each value is a round trip from the tool through the
// runtime to the bus, so reading a bucket of thousands of keys one after the other outlasts the 30 s a tool call is given
// (issue 1827, as the delivery view's issue 1945). It is a copy of the view's: a few lines are not worth a group delivery
// through mesh-sdk (rules-are-weighed).
const ReadWorkers = 16
// MaxReads bounds the values one call of a read tool reads; past it the answer says it was cut and how to narrow it.
const MaxReads = 1000
// parallel runs read(i) for i in [0, n), at most ReadWorkers at a time, and answers the first error; once one failed no
// further read starts.
func parallel(n int, read func(i int) error) error {
var (
next atomic.Int64
failed atomic.Bool
once sync.Once
first error
wg sync.WaitGroup
)
for w := 0; w < min(ReadWorkers, n); w++ {
wg.Add(1)
go func() {
defer wg.Done()
for !failed.Load() {
i := int(next.Add(1) - 1)
if i >= n {
return
}
if err := read(i); err != nil {
once.Do(func() { first = err })
failed.Store(true)
}
}
}()
}
wg.Wait()
return first
}
// capKeys keeps the first MaxReads keys and answers a note when it cut the rest, "" otherwise.
func capKeys(keys []string, narrow string) ([]string, string) {
if len(keys) <= MaxReads {
return keys, ""
}
return keys[:MaxReads], fmt.Sprintf("read %d of %d matching keys and stopped; %s", MaxReads, len(keys), narrow)
}
// newestFirst sorts step keys (<node>.<id>) by id, newest first: an id is the time in base 36, then six hex digits, so the
// longer id is the later and equal lengths compare as text. A key that is no step key goes last.
func newestFirst(keys []string) {
id := func(k string) (string, bool) {
_, id, err := delivery.ParseStepKey(k)
return id, err == nil
}
sort.Slice(keys, func(i, j int) bool {
a, aok := id(keys[i])
b, bok := id(keys[j])
switch {
case aok != bok:
return aok
case len(a) != len(b):
return len(a) > len(b)
case a != b:
return a > b
}
return keys[i] > keys[j]
})
}
@@ -46,17 +46,9 @@ type Case struct {
notes []string
// Progress says what the check is doing, so a check that ran past its bound names the step it was stuck at.
Progress *Progress
// Kind is what asked the check (KindNew, KindNodes, KindTrunk): a carry after a trunk or sibling move is asked on the
// build seats' recheck stream, every other check on check (ADR 0536 §4, ADR 0548 §5).
// Kind is what asked the check (KindNew, KindNodes, KindTrunk): a re-judge after a trunk move is asked on the build
// seats' recheck stream, every other check on check (ADR 0536 §4).
Kind string
// CarriedFrom are, for the carry of a verdict that passed, the trunk and sibling commits it was judged on (ADR 0548 §5);
// nil for a whole check.
CarriedFrom map[string]string
// FPSet is the candidate's fingerprint set (Candidate.FPSet): a composition names the set it judged.
FPSet string
// Composition is, for a carry, the composition that passed on the same fingerprint set: it is kept and not asked again,
// since no fingerprint of a module it composed changed (ADR 0548 §5). Nil composes.
Composition *delivery.Composition
}
// Progress is the step a check is at, written by the check and read when its bound passes.
@@ -88,8 +80,11 @@ func CaseOf(c delivery.Candidate, trunk map[string]string) *Case {
if err != nil {
return nil
}
cs := &Case{Key: key, Group: c.Group, HeadsHash: c.HeadsHash, Heads: GateHeads(c), Trunk: map[string]string{}, Progress: &Progress{}, FPSet: c.FPSet}
cs := &Case{Key: key, Group: c.Group, HeadsHash: c.HeadsHash, Heads: GateHeads(c), Trunk: map[string]string{}, Progress: &Progress{}}
for _, p := range c.Pulls {
if p.Merged {
continue // merged while the group waits for its other members: judged by the verdict the group had (issue 1791)
}
switch head, known := trunk[p.Repository]; {
case !known || head == "":
cs.Left = append(cs.Left, fmt.Sprintf("%s#%d: the trunk head of %s is not known", p.Repository, p.Number, p.Repository))
@@ -127,7 +122,18 @@ func GateHeads(c delivery.Candidate) string {
}
// sameTrunk says whether two verdicts' trunk heads are the same.
func sameTrunk(a, b map[string]string) bool { return maps.Equal(a, b) }
// sameTrunk says whether a verdict judged on the trunk heads judged stands for a case on the trunk heads now: every
// repository of the case at the head it was judged on. A repository the verdict names and the case does not is a member
// that merged since (issue 1791): its own merge moved that trunk, and the members left are judged by the verdict the
// group had.
func sameTrunk(judged, now map[string]string) bool {
for repo, head := range now {
if judged[repo] != head {
return false
}
}
return len(now) > 0 || len(judged) == 0
}
// Trunks holds the trunk head of each repository, as the gate's source.main says (a watch puts it in).
type Trunks struct {
@@ -4,7 +4,6 @@ import (
"context"
"errors"
"fmt"
"sort"
"strings"
"sync"
"time"
@@ -43,9 +42,6 @@ type Outcome struct {
// Judged says, by pull request (pullKey), which program judged a change to a build module (ADR 0535 §5); a pull request
// that changes none is not named.
Judged map[string]string
// Beside are the siblings the check read at their trunk's head, by repository (ADR 0548 §6): the verdict names them, and
// a move of one is carried.
Beside map[string]string
}
// BuildNote is the state of a candidate's module build, by module: built, failed or waiting (ADR 0484 clause 7).
@@ -69,14 +65,6 @@ type Checker struct {
Owed KV
Judge Judge
Trunks interface{ Heads() map[string]string }
// Runs says which commit the mesh runs of a module (the indexer's modules): the node-engine a verdict read beside at its
// running commit is out of date when that commit moves (ADR 0548 §6); nil reads every sibling at its trunk.
Runs interface{ CommitOf(module string) string }
// Turns are the gate's merge turns (ADR 0548 §3): no check is claimed of a repository a turn names, but the check of the
// waiting delivery's own heads; nil holds none.
Turns interface {
Holding(repositories []string, heads string) (string, delivery.TurnWait, bool)
}
// Bound is how long one check may run; none when 0 (DefaultBound).
Bound time.Duration
// Link is the base of the address a status points at; the verdict key follows.
@@ -95,13 +83,24 @@ type Checker struct {
tried map[string]bool // by verdict key: looked at since this process started
epoch uint64 // changes of the nodes seen since this process started
nodesFail map[string]uint64 // by verdict key: the epoch a failure of the nodes was written at
held map[string]string // by verdict key: the turn that holds its check off, as last said
// current is, by candidates key, the verdict key of the heads the group holds now; running holds, by candidates key,
// the stop of each check that runs, by its verdict key (issue 1791; issue 1921, try speed S9).
current map[string]string
running map[string]map[string]context.CancelCauseFunc
}
// superseded is why a check whose heads the group no longer holds was stopped: the heads it holds now.
type superseded struct{ by string }
func (s superseded) Error() string {
return "superseded: the group holds other heads now (" + s.by + "), whose check judges it"
}
func (ch *Checker) init() {
ch.once.Do(func() {
ch.known, ch.inflight, ch.again, ch.tried = map[string]delivery.Candidate{}, map[string]bool{}, map[string]bool{}, map[string]bool{}
ch.nodesFail, ch.held = map[string]uint64{}, map[string]string{}
ch.nodesFail = map[string]uint64{}
ch.current, ch.running = map[string]string{}, map[string]map[string]context.CancelCauseFunc{}
})
}
@@ -139,6 +138,15 @@ func (ch *Checker) Candidate(ctx context.Context, key string, value []byte) {
}
ch.mu.Lock()
ch.known[key] = c
if vk, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash); err == nil {
ch.current[key] = vk
// A check of heads the group no longer holds is stopped: its asks are not put, and it ends superseded.
for running, stop := range ch.running[key] {
if running != vk {
stop(superseded{by: c.HeadsHash})
}
}
}
ch.mu.Unlock()
ch.consider(ctx, key, c)
}
@@ -204,7 +212,7 @@ func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidat
// Every check is asked at once: the build seats' streams hold the backlog, a new head's check and a check after a
// failure of the nodes on check, a re-judge after a trunk move on recheck, pulled after them (ADR 0525 §4, §9;
// ADR 0536 §4). The checker holds no place and no capacity of its own.
ch.prepare(ctx, cs)
cs.Kind = ch.kindOf(ctx, cs)
ch.try(ctx, key, cs)
ch.mu.Lock()
more := ch.again[cs.Key]
@@ -224,123 +232,29 @@ func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidat
}
// The kinds of a check (ADR 0536 §4): a head never judged and a head asked again after a failure of the nodes are asked on
// the build seats' check stream; a carry, a verdict moved to a newer trunk or sibling commit by its package digests (ADR 0548
// §5), on the recheck stream, pulled after.
// the build seats' check stream, a head asked again only because its trunk moved on the recheck stream, pulled after.
const (
KindNew = "new"
KindNodes = "nodes"
KindTrunk = "trunk"
)
// prepare reads the case's verdict key and says what its check is (ADR 0548 §1, §5, §7): none (or one left running) is a
// head never judged, a whole check; a failure of the nodes is asked again after the nodes changed; a verdict judged on trunk
// or sibling commits that moved since is carried, its packages run only where the move changed their digest, and a pass's
// commits are named as the carry's CarriedFrom. A verdict that stands is ranked with the carries: its claim finds nothing
// to do.
func (ch *Checker) prepare(ctx context.Context, cs *Case) {
cs.Kind, cs.CarriedFrom, cs.Composition = KindNew, nil, nil
// kindOf is the kind of a case's check, read from its verdict key: none (or one left running) is a head never judged; a
// failure of the nodes is asked again after the nodes changed; a verdict on other trunk heads is a re-judge after a trunk
// move. A verdict that stands is ranked with the re-judges: its claim finds nothing to do.
func (ch *Checker) kindOf(ctx context.Context, cs *Case) string {
value, _, found, err := ch.Verdicts.Get(ctx, cs.Key)
if err != nil || !found {
return
return KindNew
}
v, err := delivery.DecodeVerdict(value)
switch {
case err != nil || v.State == delivery.VerdictRunning:
return
return KindNew
case IsNodesFailure(v):
cs.Kind = KindNodes
return
case IsConflictFailure(v):
return // a head git refused: its next check is whole (ADR 0548 §7)
return KindNodes
}
cs.Kind = KindTrunk
if v.State == delivery.VerdictPass && ch.stale(v, cs) {
cs.CarriedFrom = judgedOn(v)
if c := v.Composition; c != nil && c.State == delivery.CompositionPass && c.FPSet != "" && c.FPSet == cs.FPSet {
kept := *c
cs.Composition = &kept
}
}
}
// stale says whether a verdict was judged on commits that are no longer the case's: a trunk it merged moved, or a sibling it
// read at its trunk's head (ADR 0548 §6).
func (ch *Checker) stale(v delivery.Verdict, cs *Case) bool {
return !sameTrunk(v.Trunk, cs.Trunk) || besideMoved(v.Beside, ch.besideNow())
}
// besideNow are the commits a sibling is read at now, by repository: its trunk head, and for the node-engine the commit the
// mesh runs when the indexer names one, as besideFor places them.
func (ch *Checker) besideNow() map[string]string {
now := ch.heads()
if ch.Runs != nil {
if c := ch.Runs.CommitOf(nodeEngineModule); c != "" {
if now == nil {
now = map[string]string{}
}
now[nodeEngineRepo] = c
}
}
return now
}
// besideMoved says whether a sibling read at these commits moved: the commit it is read at now is known and another.
func besideMoved(beside, now map[string]string) bool {
for r, c := range beside {
if h := now[r]; h != "" && h != c {
return true
}
}
return false
}
// judgedOn are the trunk and sibling commits a verdict was judged on, by repository.
func judgedOn(v delivery.Verdict) map[string]string {
out := make(map[string]string, len(v.Trunk)+len(v.Beside))
for r, c := range v.Beside {
out[r] = c
}
for r, c := range v.Trunk {
out[r] = c
}
return out
}
// carried is what a carry's verdict names as carried from (ADR 0548 §5): the commits it was judged on before, of the
// repositories the new verdict still judges or reads; nil when none of them moved.
func carried(from map[string]string, v delivery.Verdict) map[string]string {
out, moved := map[string]string{}, false
for r, c := range from {
now, ok := v.Trunk[r]
if !ok {
now, ok = v.Beside[r]
}
if ok && now != "" && c != "" {
out[r] = c
moved = moved || now != c
}
}
if !moved {
return nil
}
return out
}
// moveWords say a carry's moves, for the words of a failure: "carried from <old12> to <new12> of <repository>", each moved
// repository, sorted.
func moveWords(from map[string]string, v delivery.Verdict) string {
var moves []string
for r, c := range from {
now, ok := v.Trunk[r]
if !ok {
now = v.Beside[r]
}
if now != "" && now != c {
moves = append(moves, fmt.Sprintf("%s %s to %s", r, short12(c), short12(now)))
}
}
sort.Strings(moves)
return strings.Join(moves, ", ")
return KindTrunk
}
func (ch *Checker) candidate(key string, fallback delivery.Candidate) delivery.Candidate {
@@ -372,9 +286,6 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64
return nil, fmt.Errorf("%s: %w", cs.Key, err)
}
if !found {
if ch.heldByTurn(cs) {
return nil, nil
}
rev, err := ch.Verdicts.Create(ctx, cs.Key, data)
if errors.Is(err, ErrConflict) {
return nil, nil // another checker made it first
@@ -395,7 +306,7 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64
return &claimed{rev}, nil
}
return nil, nil
case ch.stale(cur, cs):
case !sameTrunk(cur.Trunk, cs.Trunk):
retake = true
case cur.State == delivery.VerdictFail && IsNodesFailure(cur):
ch.mu.Lock()
@@ -403,7 +314,7 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64
ch.mu.Unlock()
retake = !ok || epoch > at
}
if !retake || ch.heldByTurn(cs) {
if !retake {
return nil, nil
}
rev, err = ch.Verdicts.Update(ctx, cs.Key, data, rev)
@@ -416,59 +327,6 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64
return &claimed{rev}, nil
}
// heldByTurn says whether a merge turn holds the case's check off (ADR 0548 §3): an approved merge waits on one of the
// repositories it merges heads into, and it is not the merge of the case's own heads. The check is claimed when the turn's
// key goes (Reconsider on the turns watch). Said once per turn in the log. A check already claimed is never held: one
// claimed in the moment before the gate wrote its turn is judged on the commit it read and carried once after.
func (ch *Checker) heldByTurn(cs *Case) bool {
if ch.Turns == nil {
return false
}
repos := make([]string, 0, len(cs.Trunk))
for r := range cs.Trunk {
repos = append(repos, r)
}
repo, w, held := ch.Turns.Holding(repos, cs.Heads)
why := ""
if held {
why = repo + " " + w.Group + " " + w.Heads
}
ch.mu.Lock()
said := ch.held[cs.Key]
if why == "" {
delete(ch.held, cs.Key)
} else {
ch.held[cs.Key] = why
}
ch.mu.Unlock()
if held && said != why {
ch.logf("%s: not claimed: the approved merge of %s (heads %s) holds a turn on %s, so its check waits for that merge (ADR 0548 §3)",
cs.Key, w.Group, w.Heads, repo)
}
return held
}
// Held is every check a merge turn holds off now, by verdict key, with the turn's repository, group and heads.
func (ch *Checker) Held() map[string]string {
ch.init()
ch.mu.Lock()
defer ch.mu.Unlock()
out := make(map[string]string, len(ch.held))
for k, v := range ch.held {
out[k] = v
}
return out
}
// ConflictWords are what the words of a head that git would not merge onto its trunk say before the files in conflict.
const ConflictWords = "does not merge into the trunk: files in conflict: "
// IsConflictFailure says whether a verdict failed because its head did not merge onto its trunk (ADR 0548 §7): it has no
// pass to carry, so its next check, at a push or at a later trunk move, is a whole check of its head.
func IsConflictFailure(v delivery.Verdict) bool {
return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, ConflictWords)
}
// IsNodesFailure says whether a verdict failed because a part of the mesh the check reads was not heard (UnheardWords):
// it is judged again when the nodes change. No check fails for want of a build node any more: its asks wait in the build
// seats' streams (ADR 0525 §6, issue 1912).
@@ -492,33 +350,50 @@ func (ch *Checker) try(ctx context.Context, key string, cs *Case) {
if cl == nil {
return
}
run, stop := context.WithCancelCause(ctx)
defer stop(nil)
ch.mu.Lock()
now := ch.current[key]
if ch.running[key] == nil {
ch.running[key] = map[string]context.CancelCauseFunc{}
}
ch.running[key][cs.Key] = stop
ch.mu.Unlock()
defer func() {
ch.mu.Lock()
delete(ch.running[key], cs.Key)
if len(ch.running[key]) == 0 {
delete(ch.running, key)
}
ch.mu.Unlock()
}()
if now != "" && now != cs.Key {
stop(superseded{by: now}) // the group moved on before the check began
}
began := ch.now()
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
out := ch.judge(ctx, cs)
if run.Err() == nil {
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
}
out := ch.judge(run, cs)
if ctx.Err() != nil {
return
}
var sup superseded
if errors.As(context.Cause(run), &sup) {
// Stopped for newer heads: the verdict ends failed, not running, and no status is put, as the newer check owns the
// status of the heads they share.
v := delivery.Verdict{Group: cs.Group, Heads: cs.Heads, Trunk: cs.Trunk, State: delivery.VerdictFail,
Failed: delivery.FailedFirst{Machine: "checker", Words: sup.Error()}}
if data, err := delivery.EncodeVerdict(v); err == nil {
if _, err := ch.Verdicts.Update(ctx, cs.Key, data, cl.rev); err != nil {
ch.logf("%s: the superseded verdict could not be written: %v", cs.Key, err)
}
}
ch.logf("%s: %s", cs.Key, sup.Error())
return
}
v := out.Verdict
v.Group, v.Heads, v.Trunk, v.Beside = cs.Group, cs.Heads, cs.Trunk, nil
for r, c := range out.Beside {
if _, merged := cs.Trunk[r]; !merged && c != "" {
if v.Beside == nil {
v.Beside = map[string]string{}
}
v.Beside[r] = c
}
}
if cs.CarriedFrom != nil {
// A carry (ADR 0548 §5): a pass names the commits it was carried from; a failure names the move it failed on.
switch v.State {
case delivery.VerdictPass:
v.CarriedFrom = carried(cs.CarriedFrom, v)
case delivery.VerdictFail:
if moves := moveWords(cs.CarriedFrom, v); moves != "" {
v.Failed.Words = strings.TrimSpace(v.Failed.Words + "\nfailed on the carry over the move of " + moves + ": the move changed what these tests read")
}
}
}
v.Group, v.Heads, v.Trunk = cs.Group, cs.Heads, cs.Trunk
if v.Took.Queue == 0 && !cs.Queued.IsZero() && began.After(cs.Queued) {
v.Took.Queue = began.Sub(cs.Queued).Milliseconds()
}
@@ -540,11 +415,11 @@ func (ch *Checker) try(ctx context.Context, key string, cs *Case) {
ch.nodesFail[cs.Key] = epoch
ch.mu.Unlock()
}
if ch.movedSince(cs) || besideMoved(v.Beside, ch.besideNow()) {
if ch.movedSince(cs) {
// Judged on a trunk that moved while it ran: the verdict is kept as judged, and the status stays pending until the
// check of the new trunk, which Reconsider starts, ends.
ch.owePending(ctx, cs, v)
ch.logf("%s: %s on a trunk that moved while it ran; the status stays pending until it is carried", cs.Key, v.State)
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
ch.logf("%s: %s on a trunk that moved while it ran; the status stays pending", cs.Key, v.State)
return
}
ch.writeOwed(ctx, cs, v, out)
@@ -684,37 +559,14 @@ func (ch *Checker) TrunkMoved(ctx context.Context) {
if err != nil || !found {
continue // never judged: no status to put out of date
}
v, err := delivery.DecodeVerdict(value)
if err == nil && !ch.stale(v, cs) {
if v, err := delivery.DecodeVerdict(value); err == nil && sameTrunk(v.Trunk, cs.Trunk) {
continue
}
if err == nil && v.State == delivery.VerdictPass {
cs.CarriedFrom = judgedOn(v) // the status says it is being carried to the new commit (ADR 0548 §6)
}
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
}
ch.Reconsider(ctx)
}
// owePending owes mesh/check pending on the heads of a verdict judged on commits that moved while it ran: a pass says it is
// being carried to the trunk's head now (ADR 0548 §6), anything else that it is checked again.
func (ch *Checker) owePending(ctx context.Context, judged *Case, v delivery.Verdict) {
next := judged
ckey, err := delivery.CandidatesKey(v.Group)
ch.mu.Lock()
c, ok := ch.known[ckey]
ch.mu.Unlock()
if err == nil && ok {
if now := CaseOf(c, ch.heads()); now != nil && now.Key == judged.Key {
next = now
}
}
if v.State == delivery.VerdictPass {
next.CarriedFrom = judgedOn(v)
}
ch.writeOwed(ctx, next, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
}
// movedSince says whether a trunk the case was judged on is no longer the trunk's head.
func (ch *Checker) movedSince(cs *Case) bool {
now := ch.heads()
@@ -28,7 +28,6 @@ func TestAPassingCheckTakesTheAssignersComposition(t *testing.T) {
f := &fakeCompositions{v: delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionPass, SDK: "v0.1.14-x", Machines: []string{"g14"}, Took: 40}}
b.ex.Compositions = f
cs := caseOf(t, pullOf(repoA, 7, head1))
cs.FPSet = "fps-of-the-candidate"
v := b.judge(t, cs).Verdict
if v.State != delivery.VerdictPass || v.Composition == nil || v.Composition.State != delivery.CompositionPass || v.Composition.SDK != "v0.1.14-x" || v.Took.Composition != 40 {
t.Fatalf("verdict = %+v, composition %+v", v, v.Composition)
@@ -39,15 +38,6 @@ func TestAPassingCheckTakesTheAssignersComposition(t *testing.T) {
if !compositionMentions(v, "passed") {
t.Errorf("the line says %q", compositionWords(v.Composition, v.Took.Composition))
}
if v.Composition.FPSet == "" || v.Composition.FPSet != cs.FPSet {
t.Errorf("the composition names the fingerprint set %q, want the candidate's %q (ADR 0548 §5)", v.Composition.FPSet, cs.FPSet)
}
// ADR 0548 §5: a carry whose fingerprints did not change keeps the composition it passed and asks the assigner nothing.
carry := caseOf(t, pullOf(repoA, 7, head1))
carry.CarriedFrom, carry.Composition = map[string]string{repoA: trunkA2}, v.Composition
if kept := b.judge(t, carry).Verdict; kept.Composition == nil || kept.Composition.SDK != "v0.1.14-x" || len(f.asked) != 1 {
t.Errorf("the carry's composition is %+v and the assigner was asked %d times; want the kept one and no second ask", kept.Composition, len(f.asked))
}
}
func compositionMentions(v delivery.Verdict, word string) bool {
@@ -74,6 +74,18 @@ func (f *fakeSteps) tool(name string) []stepCall {
return out
}
// toolAt is tool's calls of one level: a check's levels run side by side (issue 1921, try speed S10), so only the calls of
// one level keep an order.
func (f *fakeSteps) toolAt(name, level string) []stepCall {
var out []stepCall
for _, c := range f.tool(name) {
if c.args["level"] == level {
out = append(out, c)
}
}
return out
}
type entries []delivery.NodeToolchain
func (e entries) Entries(language string) []delivery.NodeToolchain {
@@ -269,9 +281,9 @@ func TestOnlyThePackageWithoutAKeptVerdictRuns(t *testing.T) { // 1288/AC1
t.Fatal(err)
}
out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
tests := b.steps.tool("test")
if len(tests) != 2 || strings.Join(packagesOf(tests[0]), ",") != "fixture/slow" || tests[0].args["level"] != "unit" {
t.Fatalf("test calls = %+v, want the unit level of fixture/slow alone, then the integration level", tests)
tests := b.steps.toolAt("test", "unit")
if len(tests) != 1 || strings.Join(packagesOf(tests[0]), ",") != "fixture/slow" || len(b.steps.toolAt("test", "integration")) != 1 {
t.Fatalf("test calls = %+v, want the unit level of fixture/slow alone, and the integration level", b.steps.tool("test"))
}
if u := out.Verdict.Levels[delivery.LevelUnit]; u.Run != 2 || u.Kept != 40 {
t.Errorf("unit = %+v, want 2 run and 40 kept", u)
@@ -302,7 +314,7 @@ func TestAReAskRunsOnlyWhatFailedAndWhatChanged(t *testing.T) { // 1288/AC2, AC3
// Asked again with nothing changed: only the failed package runs; the good one is taken.
b.steps.calls = nil
b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
tests := b.steps.tool("test")
tests := b.steps.toolAt("test", "unit")
if len(tests) == 0 || strings.Join(packagesOf(tests[0]), ",") != "fixture/failing/bad" {
t.Fatalf("the re-ask ran %+v, want only fixture/failing/bad", tests)
}
@@ -310,7 +322,7 @@ func TestAReAskRunsOnlyWhatFailedAndWhatChanged(t *testing.T) { // 1288/AC2, AC3
b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"fixture/failing/bad": digA, "fixture/failing/good": digD}}
b.steps.calls = nil
b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
tests = b.steps.tool("test")
tests = b.steps.toolAt("test", "unit")
if got := strings.Join(packagesOf(tests[0]), ","); got != "fixture/failing/bad,fixture/failing/good" && got != "fixture/failing/good,fixture/failing/bad" {
t.Errorf("after a digest changed the run was %q, want the failed package and the changed one", got)
}
@@ -342,7 +354,7 @@ func TestANodePackageIsNeverKeptAndRunsEveryCheck(t *testing.T) { // 1288/AC4, 1
}
// 1289/AC1, AC2 as issue 1921 changed it: the change's own failing test runs first, the level's other packages still run
// after it (so one check names every failure), and no later level runs.
// after it (so one check names every failure); the later level ran beside it (S10) and is not judged.
func TestTheChangesOwnFailingTestRunsFirstThenTheRestAndNoLaterLevel(t *testing.T) {
b := newBench(t)
b.fetch.Changed = []string{"modules/m/internal/b/b_test.go"}
@@ -358,12 +370,12 @@ func TestTheChangesOwnFailingTestRunsFirstThenTheRestAndNoLaterLevel(t *testing.
if out.Verdict.State != delivery.VerdictFail || !strings.Contains(out.Verdict.Failed.Test, "TestThreeIsFour") {
t.Fatalf("verdict = %+v", out.Verdict)
}
tests := b.steps.tool("test")
tests := b.steps.toolAt("test", "unit")
if len(tests) != 2 || strings.Join(packagesOf(tests[0]), ",") != "m/internal/b" || strings.Join(packagesOf(tests[1]), ",") != "m/internal/a" {
t.Fatalf("test calls = %+v, want m/internal/b, then m/internal/a, each once", tests)
}
if n := len(b.steps.tool("digest")); n != 1 {
t.Errorf("%d digest calls, want the unit level's alone (the check stopped)", n)
if out.Verdict.Levels[delivery.LevelIntegration].Run != 0 {
t.Errorf("the integration level was judged after the unit level failed: %+v", out.Verdict.Levels)
}
if out.Verdict.Composition == nil || out.Verdict.Composition.State != delivery.CompositionNotRun {
t.Errorf("composition = %+v", out.Verdict.Composition)
@@ -4,6 +4,7 @@ import (
"context"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -87,3 +88,29 @@ func TestATrunkMoveMarksEveryOpenPullRequestIntoItPendingAtOnce(t *testing.T) {
t.Errorf("mesh-sdk#13, alone into a trunk that did not move, is %q, want its success to stand", s)
}
}
func TestACheckThatEndsAfterItsTrunkMovedOwesPendingNotItsVerdict(t *testing.T) { // gate (a): a pass on an old trunk is never shown
ctx := context.Background()
var r *rig
release, judged := make(chan struct{}), make(chan struct{})
j := &judgeFunc{f: func(ctx context.Context, c *Case, call int) (Outcome, error) {
if call == 1 {
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) // the trunk moves while it runs
defer close(judged)
} else {
<-release // the check asked again for the new trunk waits in the stream meanwhile
}
return Outcome{Verdict: passing()}, nil
}}
r = newRig(t, j)
r.ch.Candidate(ctx, "feat%2Fx", candidateOf(t, "feat/x", pullOf(repoA, 7, head1)))
<-judged
for deadline := time.Now().Add(2 * time.Second); owedState(t, r, repoA, 7, head1) == "" && time.Now().Before(deadline); time.Sleep(time.Millisecond) {
}
time.Sleep(20 * time.Millisecond)
if s := owedState(t, r, repoA, 7, head1); s != "pending" {
t.Errorf("the status is %q, want pending: the pass was judged on a trunk that moved", s)
}
close(release)
r.ch.Wait()
}
@@ -159,6 +159,9 @@ type target struct {
runner string
// judged is what the line says of the program that judged a change to the build module; empty for any other change.
judged string
// ask is the check ask put for it, and failed why it could not be asked (issue 1921, try speed S8).
ask delivery.StepAsk
failed *delivery.Verdict
}
// judgedBy takes what a step under runner workspace answered of the program that ran it (judged_by): the candidate, or the
@@ -203,15 +206,12 @@ type state struct {
nodes map[string]bool
clone time.Duration
slow []string
// beside are the siblings read at their trunk's head, by repository (ADR 0548 §6).
beside map[string]string
}
// Judge runs the case (it is the Judge of a Checker). A failure of the change, of a merge or of a step is a verdict; an error is only a case that cannot
// be judged at all.
func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) {
st := &state{j: j, c: c, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{},
beside: map[string]string{}}
st := &state{j: j, c: c, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{}}
if v := st.judgeAll(ctx); v != nil {
return st.finish(ctx, *v), nil
}
@@ -233,13 +233,6 @@ func (st *state) finish(ctx context.Context, v delivery.Verdict) Outcome {
// fails the check, naming the machine it failed at. A try is not composed (it has no candidate); a composition that
// cannot be had leaves it not run, said; with no assigner running it is not judged, said at once.
func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
if st.c.Composition != nil && st.c.Try == nil {
// A carry whose candidate's fingerprints did not change keeps the composition it passed (ADR 0548 §5).
kept := *st.c.Composition
v.Composition = &kept
st.c.note("the composition is kept: no fingerprint of a module it composed changed")
return
}
if st.j.Compositions == nil || st.c.Try != nil {
return
}
@@ -253,9 +246,6 @@ func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
return
}
withComposition(v, comp)
if v.Composition != nil && v.Composition.State != delivery.CompositionNotRun {
v.Composition.FPSet = st.c.FPSet // a carry keeps it while the fingerprints stay the same (ADR 0548 §5)
}
}
// outcome fills the verdict's levels, times, machines and composition from what ran.
@@ -295,9 +285,6 @@ func (st *state) outcome(v delivery.Verdict) Outcome {
}
}
out := Outcome{Verdict: v, Slowest: slow}
if len(st.beside) > 0 {
out.Beside = st.beside
}
for _, t := range st.targets {
if t.judged != "" {
if out.Judged == nil {
@@ -323,12 +310,13 @@ func (st *state) levelsOf() []string {
return []string{delivery.LevelUnit, delivery.LevelIntegration}
}
// judgeAll puts one check ask per pull request and build module the repository needs, the mesh-sdk member first, and
// judges each answer. A head that does not merge, a pin that is not allowed or a level that fails ends the check at once
// (ADR 0484 clause 13).
// judgeAll puts one check ask per pull request and build module the repository needs, every ask of the group at once so
// the members run side by side on the build nodes (issue 1921, try speed S8), and judges the answers in member order, the
// mesh-sdk member first. A head that does not merge, a pin that is not allowed or a level that fails ends the check at once
// (ADR 0484 clause 13): the verdict is the first failure in member order, as when the asks were put one after another.
func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
// The mesh-sdk member first: whether its merge changes the TypeScript package's paths decides whether the other members'
// Node fetches take the group's package (ADR 0500 §5).
// Node fetches take the group's package (ADR 0500 §5). Those Node asks alone wait for the mesh-sdk member's answer.
pulls := slices.Clone(st.c.Pulls)
slices.SortStableFunc(pulls, func(a, b Pull) int {
if (a.Repository == SDKRepository) != (b.Repository == SDKRepository) {
@@ -339,35 +327,32 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
}
return 0
})
var groupPackage map[string]any
for _, p := range pulls {
var merged []Fetched
sdkGroup := len(st.c.Pulls) > 1 && len(pulls) > 0 && pulls[0].Repository == SDKRepository
members := make([][]*target, len(pulls))
var waiting []*target // Node asks of the other members, put once the mesh-sdk member is judged
for i, p := range pulls {
for _, bm := range st.j.Modules.BuildModules(p.Repository) {
t := &target{pull: p, module: bm}
if tc, none := delivery.AgreedToolchain(st.j.Toolchains.Entries(buildrun.LanguageOf[bm])); none == "" {
t.toolchain = tc.Version
}
args := map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number), "base": p.Trunk}
if bm == buildrun.NodeBuildModule && groupPackage != nil && p.Repository != SDKRepository {
args["package"] = groupPackage
}
if bm == buildrun.GoBuildModule {
beside, err := st.j.besideFor(ctx, st.c, p)
if err != nil {
return fail(forgeMachine(err, bm), err.Error()+unheardWords(err))
}
if len(beside) > 0 {
args["beside"] = beside
t.beside = besideLines(beside)
besideAtTrunk(st.c, beside, st.beside)
}
if p.Repository == catalogueRepository {
t.runner = RunnerWorkspace // used by the build module when the merge changes it (ADR 0535 §1)
}
}
t.fetchArgs = args
t := st.target(ctx, p, bm)
members[i] = append(members[i], t)
st.targets = append(st.targets, t)
if v := st.ask(ctx, t, &merged); v != nil {
if t.failed != nil {
continue // said when the judging reaches it, so the verdict stays the first failure in member order
}
if sdkGroup && bm == buildrun.NodeBuildModule && p.Repository != SDKRepository {
waiting = append(waiting, t)
continue
}
t.failed = st.put(ctx, t)
}
}
var groupPackage map[string]any
for i, p := range pulls {
var merged []Fetched
for _, t := range members[i] {
if t.failed != nil {
return t.failed
}
if v := st.judgeAsk(ctx, t, &merged); v != nil {
return v
}
}
@@ -377,14 +362,51 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
if p.Repository == SDKRepository && len(st.c.Pulls) > 1 && st.j.changesPackage(p, merged) {
groupPackage = map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number)}
}
if i == 0 && sdkGroup {
for _, t := range waiting {
if groupPackage != nil {
t.fetchArgs["package"] = groupPackage
}
t.failed = st.put(ctx, t)
}
}
}
return nil
}
// ask puts the target's check ask, waits for the step of the node that pulled it and judges its answer: the fetch, the
// pins of its go.mod files, then each level's digest, kept verdicts and tests.
func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *delivery.Verdict {
// target is the check of one pull request by one build module: its fetch's arguments, its toolchain, the siblings its
// tests read and the runner it is asked with; failed says why it cannot be asked.
func (st *state) target(ctx context.Context, p Pull, bm string) *target {
t := &target{pull: p, module: bm}
if tc, none := delivery.AgreedToolchain(st.j.Toolchains.Entries(buildrun.LanguageOf[bm])); none == "" {
t.toolchain = tc.Version
}
args := map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number), "base": p.Trunk}
if bm == buildrun.GoBuildModule {
beside, err := st.j.besideFor(ctx, st.c, p)
if err != nil {
t.failed = fail(forgeMachine(err, bm), err.Error()+unheardWords(err))
return t
}
if len(beside) > 0 {
args["beside"] = beside
t.beside = besideLines(beside)
}
if p.Repository == catalogueRepository {
t.runner = RunnerWorkspace // used by the build module when the merge changes it (ADR 0535 §1)
}
}
t.fetchArgs = args
return t
}
// put puts the target's check ask on its build seat's stream; a check that was stopped (its group holds other heads now,
// issue 1791) puts none.
func (st *state) put(ctx context.Context, t *target) *delivery.Verdict {
p := t.pull
if err := context.Cause(ctx); err != nil {
return fail(t.module, fmt.Sprintf("the check of %s#%d was not asked: %v", p.Repository, p.Number, err))
}
job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: st.levelsOf(), Kept: true}
if t.runner != "" {
job.Runner, job.RunnerWhen = t.runner, buildGoDir
@@ -396,13 +418,20 @@ func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *deliver
if st.c.Kind == KindTrunk {
kind = delivery.AskRecheck // a re-judge after a trunk move goes after every new head's check (ADR 0536 §4)
}
a := delivery.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: askLanguage[t.module], Asker: delivery.CheckerModule,
t.ask = delivery.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: askLanguage[t.module], Asker: delivery.CheckerModule,
At: st.j.now().UTC(), Check: job}
st.c.Progress.Set(fmt.Sprintf("the %s ask %s of %s#%d", kind, a.ID, p.Repository, p.Number), "")
if err := st.j.Asks.Put(ctx, a); err != nil {
st.c.Progress.Set(fmt.Sprintf("the %s ask %s of %s#%d", kind, t.ask.ID, p.Repository, p.Number), "")
if err := st.j.Asks.Put(ctx, t.ask); err != nil {
return fail(t.module, fmt.Sprintf("the check of %s#%d could not be asked: %v", p.Repository, p.Number, err))
}
step, err := st.j.Asks.Wait(ctx, a)
return nil
}
// judgeAsk waits for the step of the node that pulled the target's ask and judges its answer: the fetch, the pins of its
// go.mod files, then each level's digest, kept verdicts and tests.
func (st *state) judgeAsk(ctx context.Context, t *target, merged *[]Fetched) *delivery.Verdict {
p := t.pull
step, err := st.j.Asks.Wait(ctx, t.ask)
if err != nil {
if ctx.Err() != nil {
return fail(t.module, ctx.Err().Error())
@@ -431,7 +460,7 @@ func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *deliver
if st.c.Try == nil {
st.j.writePreview(ctx, p, append(*merged, t.fetched))
}
return fail(step.Node, fmt.Sprintf("%s#%d %s%s", p.Repository, p.Number, ConflictWords, strings.Join(t.fetched.Conflicts, ", ")))
return fail(step.Node, fmt.Sprintf("%s#%d does not merge into the trunk: files in conflict: %s", p.Repository, p.Number, strings.Join(t.fetched.Conflicts, ", ")))
}
t.changed = t.fetched.Changed
if !changesBuildGo(p.Repository, t.changed) {
@@ -444,7 +473,7 @@ func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *deliver
ran := 0.0
for _, la := range answer.Levels {
v, elapsed := st.level(ctx, t, la)
ran += elapsed
ran = max(ran, elapsed) // the levels run side by side (issue 1921, try speed S10): the longest is what they took
if v != nil {
st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
return v
@@ -553,98 +582,23 @@ const MaxWords = 4096
// failureWords is a failure's words: its output's last lines up to MaxWords, with the line the build module told apart as
// the test's error first when the output does not start with it, so the first line of the words is the cause and the
// status line, which is cut short, names it. **A failure's words are never "PASS" or "ok"** (issue 1901, its third
// meeting: a package that failed after its tests passed was worded "PASS"): go test's summary lines are dropped, and with
// no cause told apart (an older build module) the cause is the output's first "panic:", "fatal error:" or "signal:"
// line, read before the output is cut, or the words say the tests passed and the program failed after them.
// status line, which is cut short, names it.
func failureWords(f Failure) string {
out, passed := dropSummary(strings.TrimSpace(f.Output))
cause := strings.TrimSpace(f.Cause)
if cause == "" {
cause = causeIn(out)
}
switch {
case cause == "" && out == "" && passed:
cause = "the tests passed, then the test program exited with a failure: an os.Exit or a panic in TestMain or after the tests"
case cause == "" && out == "":
// The build module on main sends a package that stopped (a timeout, an exit, a kill) as its "FAIL" line alone.
cause = "the package failed with no test to name and no panic, signal or build error in its output (an older build module sent no cause)"
}
if len(cause) > MaxCause {
cause = strings.ToValidUTF8(cause[:MaxCause], "") + "..."
}
// The words keep MaxWords exactly: the cause and its newline, the marker and the kept lines (review round 1).
budget := MaxWords
if cause != "" {
budget -= len(cause) + 1
}
if len(out) > budget {
const marker = "... "
tail := out[len(out)-(budget-len(marker)):]
out := strings.TrimSpace(f.Output)
if len(out) > MaxWords {
out = out[len(out)-MaxWords:]
// Start on a whole line, and a whole character.
if i := strings.IndexByte(tail, '\n'); i >= 0 {
tail = strings.TrimLeft(tail[i:], "\n")
if i := strings.IndexByte(out, '\n'); i >= 0 {
out = "... " + strings.TrimLeft(out[i:], "\n")
}
out = marker + strings.ToValidUTF8(tail, "")
out = strings.ToValidUTF8(out, "")
}
if cause != "" && !strings.HasPrefix(firstLine(out), cause) {
out = strings.TrimRight(cause+"\n"+out, "\n")
if c := strings.TrimSpace(f.Cause); c != "" && !strings.HasPrefix(firstLine(out), c) {
out = c + "\n" + out
}
return out
}
// MaxCause is the most a cause in a failure's words holds: one long line (the 1901 review: a cause of a kilobyte could
// push the words past MaxWords).
const MaxCause = 1024
// dropSummary drops go test's summary lines from a failure's output ("PASS", "FAIL", "ok <pkg>", "FAIL <pkg>"), and says
// whether a "PASS" was among them: the package's tests passed before it failed.
func dropSummary(out string) (string, bool) {
passed := false
lines := strings.Split(out, "\n")
kept := lines[:0]
for _, l := range lines {
// go's exact forms only (review round 1): "PASS" or "FAIL" alone, "ok \t<pkg>\t<time>", "FAIL\t<pkg>\t<time>".
switch t := strings.TrimRight(l, "\r"); {
case t == "PASS":
passed = true
case t == "FAIL", strings.HasPrefix(t, "ok \t"), strings.HasPrefix(t, "FAIL\t"):
default:
kept = append(kept, l)
}
}
return strings.TrimSpace(strings.Join(kept, "\n")), passed
}
// causeIn is a race the race detector reported, as its heading, its access and the frame that made it ("WARNING: DATA
// RACE: Write at 0x... by goroutine 9 in pkg.F.func1()"); else the first "panic:", "fatal error:" or "signal:" line of a
// failure's output, in that order; or "". **Only lines at column 0**, where go prints the runtime's: a test's indented log
// line holding the same words is not the cause (review round 1).
func causeIn(out string) string {
lines := strings.Split(out, "\n")
for i, l := range lines {
if strings.TrimRight(l, "\r") != "WARNING: DATA RACE" {
continue
}
words := "WARNING: DATA RACE"
if i+1 < len(lines) && strings.TrimSpace(lines[i+1]) != "" {
words += ": " + strings.TrimSuffix(strings.TrimSpace(lines[i+1]), ":")
if i+2 < len(lines) && strings.HasPrefix(lines[i+2], " ") {
words += " in " + strings.TrimSpace(lines[i+2])
}
}
return words
}
for _, prefix := range []string{"panic: ", "fatal error: ", "signal: "} {
for _, l := range lines {
if l = strings.TrimRight(l, "\r"); strings.HasPrefix(l, prefix) {
return l
}
}
}
return ""
}
// firstLine is the first non-empty line of a failure's words.
func firstLine(s string) string {
for _, l := range strings.Split(s, "\n") {
@@ -679,7 +633,11 @@ func levelFailure(t *target, level string, results ...Tested) *delivery.Verdict
if t.module == buildrun.NodeBuildModule && f.Package == "" {
name = res.Module
}
all = append(all, delivery.FailedFirst{Test: level + ": " + name, Words: failureWords(f)})
step := level
if f.Step != "" {
step = f.Step // the lint step's finding, not the level's (ADR 0542 §2)
}
all = append(all, delivery.FailedFirst{Test: step + ": " + name, Words: failureWords(f)})
}
}
if len(all) == 0 { // no result: a level said failed with nothing to name
@@ -177,8 +177,8 @@ func TestACandidateThatDoesNotCompileFailsTheCheckWithTheCompilersWords(t *testi
if out.Verdict.State != delivery.VerdictFail || !strings.Contains(out.Verdict.Failed.Words, "undefined: x") {
t.Fatalf("verdict = %+v, want a failure naming the compile error", out.Verdict)
}
if n := len(b.steps.tool("test")); n != 1 {
t.Errorf("%d test steps, want the first one's failure to end the check", n)
if n := len(b.steps.toolAt("test", "unit")); n != 1 {
t.Errorf("%d unit test steps, want the first one's failure to end the level", n)
}
}
@@ -85,20 +85,18 @@ type pieces struct {
head, commits, counts, shortCounts, times, shortTimes, over, builds, composition, shortComposition, group string
// judged names the program that judged a change to a build module (ADR 0535 §5); never dropped.
judged string
// carried says, of a carried verdict, the commits it was carried from (ADR 0548 §8); never dropped.
carried string
}
// pullKey names a pull request of a case: `<owner/repo>#<number>`.
func pullKey(p Pull) string { return fmt.Sprintf("%s#%d", p.Repository, p.Number) }
func (p pieces) join(drop int) string {
parts := []string{p.head, p.commits, p.carried, p.counts, p.judged}
parts := []string{p.head, p.commits, p.counts, p.judged}
switch {
case drop < 3 && p.times != "":
parts = append(parts, p.times)
case drop == 3 && p.shortTimes != "":
parts = []string{p.head, p.commits, p.carried, p.shortCounts, p.judged, p.shortTimes}
parts = []string{p.head, p.commits, p.shortCounts, p.judged, p.shortTimes}
}
if p.over != "" {
parts = append(parts, p.over)
@@ -173,39 +171,6 @@ func buildWords(builds []BuildNote) string {
return "builds: " + strings.Join(parts, ", ")
}
// carriedWords say the commits a verdict was carried from, for the line of pull request p (ADR 0548 §8): its own trunk's
// commit as twelve characters, a sibling's with its name before it; only those that moved, sorted. Empty when none did.
func carriedWords(from map[string]string, v delivery.Verdict, cs *Case, p Pull) string {
var own string
var others []string
for r, old := range from {
now, ok := v.Trunk[r]
if cs != nil && !ok {
now, ok = cs.Trunk[r]
}
if !ok {
now = v.Beside[r]
}
if now == "" || now == old {
continue
}
if strings.EqualFold(r, p.Repository) {
own = short12(old)
continue
}
name := r
if i := strings.LastIndex(r, "/"); i >= 0 {
name = r[i+1:]
}
others = append(others, name+" "+short12(old))
}
slices.Sort(others)
if own != "" {
others = append([]string{own}, others...)
}
return strings.Join(others, ", ")
}
// parts makes the pieces of the line of a pull request of a case.
func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string, judged map[string]string) pieces {
var pc pieces
@@ -214,18 +179,8 @@ func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []s
switch v.State {
case delivery.VerdictRunning:
pc.head = "checking " + commits
if cs != nil && cs.CarriedFrom != nil {
// Out of date and being carried to the new commit (ADR 0548 §6): pending, and saying so.
pc.head = "being carried to " + commits
if w := carriedWords(cs.CarriedFrom, v, cs, p); w != "" {
pc.carried = "from " + w
}
}
case delivery.VerdictPass:
pc.head = "pass " + commits
if w := carriedWords(v.CarriedFrom, v, cs, p); w != "" {
pc.carried = "carried from " + w // the counts that follow say what the carry ran and kept
}
default:
pc.head = "failed: " + firstFailed(v)
if n := failedCount(v); n > 1 {
@@ -1,92 +0,0 @@
package check
import (
"strings"
"testing"
)
// What the build module before this fix sent for two packages that failed after their tests passed (issue 1901, its third
// meeting: a try of mesh-controller kept "PASS" as its failure's words). Each is that build module's Output for the real
// `go test -json` of a throwaway package (mesh-catalog modules/mesh-build-go testdata/captured/go-test-json-pkg-*.jsonl):
// TestMain panicking in a goroutine after m.Run, TestMain exiting 2 after m.Run, and TestMain racing after m.Run.
const (
oldPassThenPanic = "PASS\npanic: the bus closed after the tests: nats: connection closed\ngoroutine 20 [running]:\n" +
"example.com/passthenfail.TestMain.func1()\n\t/tmp/fix-1901b/passthenfail/p_test.go:13 +0x25\n" +
"created by example.com/passthenfail.TestMain in goroutine 1\n\t/tmp/fix-1901b/passthenfail/p_test.go:13 +0x25\n" +
"FAIL\texample.com/passthenfail\t0.005s"
oldPassThenExit = "PASS\nFAIL\texample.com/mainexit\t0.004s"
// TestMain racing after m.Run passed, under go test -race (go-test-json-pkg-racemain.jsonl).
oldPassThenRace = "PASS\n==================\nWARNING: DATA RACE\nWrite at 0x000000836428 by goroutine 9:\n example.com/racemain.TestMain.func1()\n /tmp/fix-1901b/racemain/p_test.go:15 +0x24\nPrevious write at 0x000000836428 by main goroutine:\n example.com/racemain.TestMain()\n /tmp/fix-1901b/racemain/p_test.go:16 +0x44\n main.main()\n _testmain.go:48 +0x171\nGoroutine 9 (running) created at:\n example.com/racemain.TestMain()\n /tmp/fix-1901b/racemain/p_test.go:15 +0x37\n main.main()\n _testmain.go:48 +0x171\n==================\nFound 1 data race(s)\nFAIL\texample.com/racemain\t1.031s"
)
// A failure with no cause told apart never has "PASS" or "ok" as its words: the checker takes the panic line from the
// output, or says that the tests passed and the program failed after them (1901/AC5).
func TestAPackageFailureIsNeverWordedPass(t *testing.T) { // 1901/AC5
cases := []struct {
name, output, want string
}{
{"panic after the tests", oldPassThenPanic, "panic: the bus closed after the tests: nats: connection closed"},
{"exit after the tests", oldPassThenExit, "the tests passed, then the test program exited with a failure"},
{"race after the tests", oldPassThenRace, "WARNING: DATA RACE: Write at 0x000000836428 by goroutine 9 in example.com/racemain.TestMain.func1()"},
{"a panic deep in a long output", "PASS\npanic: the seat row vanished\n" + strings.Repeat("goroutine 7 [chan receive]:\n\tsome/frame.go:1 +0x1\n", 300) + "FAIL\tp\t2.0s", "panic: the seat row vanished"},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
v := levelFailure(&target{}, "integration", Tested{Failed: []Failure{{Package: "github.com/novox/mesh-controller/cmd/mesh-controller", Output: c.output}}})
for _, l := range strings.Split(v.Failed.Words, "\n") {
if l = strings.TrimSpace(l); l == "PASS" || strings.HasPrefix(l, "ok ") || strings.HasPrefix(l, "ok\t") {
t.Errorf("1901/AC5: the words keep %q: %q", l, v.Failed.Words)
}
}
if !strings.HasPrefix(v.Failed.Words, c.want) {
t.Errorf("1901/AC5: words start %q, want %q", firstLine(v.Failed.Words), c.want)
}
if got := firstFailed(*v); !strings.Contains(got, c.want) || strings.HasSuffix(got, ": PASS") {
t.Errorf("1901/AC5: status = %q", got)
}
})
}
}
// A cause the build module told apart is cut to a line's bound, so the words keep their bound (the 1901 review's note).
func TestALongCauseIsCutToALinesBound(t *testing.T) { // 1901/AC5
f := Failure{Package: "p", Test: "TestX", Cause: strings.Repeat("c", 5000), Output: "x_test.go:1: something"}
v := levelFailure(&target{}, "unit", Tested{Failed: []Failure{f}})
if len(v.Failed.Words) > MaxWords+200 {
t.Errorf("1901/AC5: words are %d bytes, bound %d", len(v.Failed.Words), MaxWords)
}
}
// The build module on main sends a package that stopped (a timeout, an exit, a kill) as only go test's last line, with no
// cause: the words then say so, never nothing (review round 1). Each output is what that build module sends for the
// capture of the same name (go-test-json-pkg-timeout, -exiting, -killed): the package's own output, its "FAIL" line.
func TestAPackageStoppedWithOnlyItsFailLineIsStillWorded(t *testing.T) { // 1901/AC5
for _, out := range []string{"FAIL\texample.com/timeout\t2.017s", "FAIL\texample.com/exiting\t0.006s", "FAIL\texample.com/killed\t0.006s"} {
v := levelFailure(&target{}, "integration", Tested{Failed: []Failure{{Package: "example.com/x", Output: out}}})
if !strings.HasPrefix(v.Failed.Words, "the package failed with no test to name and no panic, signal or build error in its output") {
t.Errorf("1901/AC5: %q is worded %q", out, v.Failed.Words)
}
}
}
// A line of the test's own that starts as go's summary does, or that holds a runtime's words indented, is not taken for
// either (review round 1): go prints its summary in exact forms and the runtime's lines at column 0.
func TestOnlyGosExactFormsAreSummaryOrCause(t *testing.T) { // 1901/AC5
out := " x_test.go:3: ok so far\nok then: the seat is free\n panic: child retried and recovered\npanic: the real one\nFAIL\tp\t1.0s"
v := levelFailure(&target{}, "unit", Tested{Failed: []Failure{{Package: "p", Output: out}}})
if !strings.HasPrefix(v.Failed.Words, "panic: the real one") || !strings.Contains(v.Failed.Words, "ok then: the seat is free") {
t.Errorf("1901/AC5: words = %q", v.Failed.Words)
}
}
// The words keep MaxWords exactly, counting the cause, the marker and the newlines (review round 1).
func TestTheWordsKeepTheirBoundExactly(t *testing.T) { // 1901/AC5
for _, cause := range []string{"", "a cause", strings.Repeat("c", 3000)} {
for _, n := range []int{100, 170, 171, 172, 500} {
f := Failure{Package: "p", Test: "TestX", Cause: cause, Output: strings.Repeat("a log line of twenty\n", n) + "the end"}
if w := failureWords(f); len(w) > MaxWords {
t.Errorf("1901/AC5: cause %d bytes, %d lines: words are %d bytes, bound %d", len(cause), n, len(w), MaxWords)
}
}
}
}
@@ -118,8 +118,10 @@ func (st *state) checkPinsOf(ctx context.Context, t *target) verdictPtr {
return nil
}
// sameBranchPin names the branch of the tried repository that points at the tried head and that mesh-sdk also has, at the
// commit the pin names; empty when there is none.
// sameBranchPin names the mesh-sdk branch whose head the pin is, for a try: the branch of the tried repository that points
// at the tried head and that mesh-sdk also has, at the commit the pin names, first; else any branch of mesh-sdk but its
// trunk whose head the pin is, for a try asked by a bare commit that no branch of the tried repository points at (issue
// 1624; try speed S12). Empty when there is none.
func (st *state) sameBranchPin(ctx context.Context, t *target, pin string) (string, error) {
mine, err := st.j.Forge.BranchHeads(ctx, t.pull.Repository)
if err != nil {
@@ -131,9 +133,6 @@ func (st *state) sameBranchPin(ctx context.Context, t *target, pin string) (stri
same = append(same, name)
}
}
if len(same) == 0 {
return "", nil
}
sdk, err := st.j.Forge.BranchHeads(ctx, SDKRepository)
if err != nil {
return "", err
@@ -144,6 +143,16 @@ func (st *state) sameBranchPin(ctx context.Context, t *target, pin string) (stri
return name, nil
}
}
var any []string
for name, h := range sdk {
if name != "main" && strings.HasPrefix(h, pin) {
any = append(any, name)
}
}
sort.Strings(any)
if len(any) > 0 {
return any[0], nil
}
return "", nil
}
@@ -240,12 +240,7 @@ func TestTheSiblingsAreWhereTheTestsStateThem(t *testing.T) { // review D2, ADR
catalogue + " go.mod " + head2: "module git.novox.be/novox/mesh-catalog\n"}, onTrunk: map[string]bool{"cb54756c3865": true}}
b.ex.Forge = forge
// A group of the controller and the catalogue: the controller's tests read the catalogue's trunk, not the group's head.
out := b.judge(t, caseOf(t, pullOf(controller, 7, head1), pullOf(catalogue, 12, head2)))
// ADR 0548 §6, issue 568's own example: the verdict names the node-engine at the commit the mesh runs, and mesh-sdk at the
// trunk the catalogue's tests read it at; the catalogue, a member here, is not named.
if out.Beside[host] != hostRuns || out.Beside[catalogue] != "" || out.Beside[SDKRepository] != "9999999999999999999999999999999999999999" {
t.Errorf("the siblings named for the verdict = %v, want the node-engine at %s and mesh-sdk at its trunk", out.Beside, hostRuns[:12])
}
b.judge(t, caseOf(t, pullOf(controller, 7, head1), pullOf(catalogue, 12, head2)))
got := besideOf(t, b, controller)
if c := got[catalogue]; c.Commit != trunkS || c.Ref != "" {
t.Errorf("the catalogue beside the controller = %+v, want its trunk %s", c, trunkS)
@@ -329,11 +324,7 @@ func TestTheChangesOwnRepositoryIsBesideItAtItsTrunk(t *testing.T) {
func TestACatalogueChangeHasTheCatalogueBesideItAtItsTrunk(t *testing.T) {
b := newBench(t)
b.ex.Trunks = trunksOf(t, map[string]string{catalogueRepository: trunkA, SDKRepository: trunkS, controllerRepo: trunkA2})
out := b.judge(t, caseOf(t, pullOf(catalogueRepository, 12, head1)))
// ADR 0548 §6: the outcome names the siblings read at their trunk's head, never the repository it merges.
if out.Beside[SDKRepository] != trunkS || out.Beside[controllerRepo] != trunkA2 || out.Beside[catalogueRepository] != "" {
t.Errorf("the siblings named for the verdict = %v, want mesh-sdk and mesh-controller at their trunks and not the catalogue", out.Beside)
}
b.judge(t, caseOf(t, pullOf(catalogueRepository, 12, head1)))
got := besideOf(t, b, catalogueRepository)
if c := got[catalogueRepository]; c.Commit != trunkA || c.Ref != "" {
t.Fatalf("the catalogue beside its own change = %+v, want its trunk %s", c, trunkA)
@@ -417,3 +408,22 @@ func TestAMergeCheckWithTheSameBranchPinIsStillRefusedWithoutAGroupPullRequest(t
t.Fatalf("verdict = %+v", v)
}
}
// Issue 1624 (met again on 2026-10-25; try speed S12): a try asked by a bare commit, which no branch of the tried
// repository points at, still takes a pin that is the head of a mesh-sdk branch, naming that branch; a pin that is the head
// of no mesh-sdk branch is still refused.
func TestATryByCommitTakesAPinThatIsTheHeadOfAnSDKBranch(t *testing.T) {
b, c := sameBranchBench(t, head2[:12])
b.ex.Forge.(*fakeForge).branches[repoA] = map[string]string{"main": trunkA, "feat/r6": trunkA2} // the branch moved on
if v := b.judge(t, c).Verdict; v.State != delivery.VerdictPass {
t.Fatalf("verdict = %+v", v)
}
if n := c.Notes(); len(n) != 1 || n[0] != "pin accepted: head of mesh-sdk feat/r6" {
t.Errorf("notes = %q", n)
}
b, c = sameBranchBench(t, "9aaa5c80ac5c")
b.ex.Forge.(*fakeForge).branches[repoA] = map[string]string{"main": trunkA}
if v := b.judge(t, c).Verdict; v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "neither on mesh-sdk's trunk") {
t.Errorf("a pin no mesh-sdk branch holds: verdict = %+v", v)
}
}
@@ -39,9 +39,7 @@ type Reader struct {
StepBound time.Duration // how long the checker waits for one step; DefaultStepBound when 0
// Trunks are the trunk heads of every repository (the gate's source); nil names no re-judge after a trunk move.
Trunks func() (map[string]string, error)
// TurnState is the gate's merge turns (ADR 0548 §3); without it no check is named as held by a turn.
TurnState StateRead
Now func() time.Time
Now func() time.Time
}
// VerdictRead is a verdict as stored with the full report of the status of each pull request it judged (Report), when the
@@ -91,46 +89,37 @@ func (r Reader) read(key string, raw json.RawMessage) (VerdictRead, error) {
return out, nil
}
// Verdicts are the verdicts the checker holds, by key; with a group only that group's. The keys are narrowed by the group
// before any value is read, at most MaxReads are read (sixteen at a time), and cut says when it stopped there.
func (r Reader) Verdicts(group string) (out []VerdictRead, cut string, err error) {
// Verdicts are the verdicts the checker holds, by key; with a group only that group's.
func (r Reader) Verdicts(group string) ([]VerdictRead, error) {
if r.VerdictState.Keys == nil || r.VerdictState.Get == nil {
return nil, "", fmt.Errorf("the verdicts state cannot be read here")
return nil, fmt.Errorf("the verdicts state cannot be read here")
}
keys, err := r.VerdictState.Keys()
if err != nil {
return nil, "", err
return nil, err
}
sort.Strings(keys)
var matching []string
var out []VerdictRead
for _, k := range keys {
if group != "" {
if g, _, err := delivery.ParseCheckerVerdictKey(k); err != nil || g != group {
continue
}
}
matching = append(matching, k)
}
matching, cut = capKeys(matching, "give a group to narrow the answer")
rows := make([]VerdictRead, len(matching))
found := make([]bool, len(matching))
if err := parallel(len(matching), func(i int) error {
raw, err := r.VerdictState.Get(matching[i])
if err != nil || len(raw) == 0 {
return err
raw, err := r.VerdictState.Get(k)
if err != nil {
return nil, err
}
rows[i], err = r.read(matching[i], raw)
found[i] = err == nil
return err
}); err != nil {
return nil, "", err
}
for i, row := range rows {
if found[i] {
out = append(out, row)
if len(raw) == 0 {
continue
}
vr, err := r.read(k, raw)
if err != nil {
return nil, err
}
out = append(out, vr)
}
return out, cut, nil
return out, nil
}
// Pull answers for one pull request: the verdict of the candidate that holds it, with its report; for one the indexer holds
@@ -144,95 +133,74 @@ func (r Reader) Pull(repository string, number int) (VerdictRead, error) {
return VerdictRead{}, err
}
sort.Strings(keys)
keys, cut := capKeys(keys, "")
// The candidates are read in batches, each in parallel and taken in key order, so the read stops at the batch that holds it.
for from := 0; from < len(keys); from += ReadWorkers * 4 {
batch := keys[from:min(from+ReadWorkers*4, len(keys))]
cands := make([]delivery.Candidate, len(batch))
ok := make([]bool, len(batch))
_ = parallel(len(batch), func(i int) error {
raw, err := r.CandidateState.Get(batch[i])
if err != nil || len(raw) == 0 {
return nil
}
c, err := delivery.DecodeCandidate(raw)
cands[i], ok[i] = c, err == nil
return nil
})
for i, c := range cands {
if !ok[i] {
for _, k := range keys {
raw, err := r.CandidateState.Get(k)
if err != nil || len(raw) == 0 {
continue
}
c, err := delivery.DecodeCandidate(raw)
if err != nil {
continue
}
for _, p := range c.Pulls {
if p.Repository != repository || p.Number != number {
continue
}
for _, p := range c.Pulls {
if p.Repository != repository || p.Number != number {
continue
}
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
if err != nil {
return VerdictRead{}, err
}
vraw, err := r.VerdictState.Get(vkey)
if err != nil {
return VerdictRead{}, err
}
if len(vraw) == 0 {
return VerdictRead{Key: vkey, Note: "no verdict yet for the heads of its candidate"}, nil
}
return r.read(vkey, vraw)
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
if err != nil {
return VerdictRead{}, err
}
vraw, err := r.VerdictState.Get(vkey)
if err != nil {
return VerdictRead{}, err
}
if len(vraw) == 0 {
return VerdictRead{Key: vkey, Note: "no verdict yet for the heads of its candidate"}, nil
}
return r.read(vkey, vraw)
}
}
if cut != "" {
return VerdictRead{}, fmt.Errorf("no candidate among the first %d holds %s#%d", MaxReads, repository, number)
}
return VerdictRead{}, fmt.Errorf("no candidate holds %s#%d", repository, number)
}
// Kept is the kept verdicts: one key's value as stored, or with an empty key the keys of a level (or every level) and their values.
func (r Reader) Kept(level, digest string) (map[string]json.RawMessage, string, error) {
func (r Reader) Kept(level, digest string) (map[string]json.RawMessage, error) {
if r.KeptState.Keys == nil || r.KeptState.Get == nil {
return nil, "", fmt.Errorf("the kept state cannot be read here")
return nil, fmt.Errorf("the kept state cannot be read here")
}
if digest != "" {
key, err := delivery.KeptKey(level, digest)
if err != nil {
return nil, "", err
return nil, err
}
raw, err := r.KeptState.Get(key)
if err != nil {
return nil, "", err
return nil, err
}
if len(raw) == 0 {
return nil, "", fmt.Errorf("no kept verdict %s", key)
return nil, fmt.Errorf("no kept verdict %s", key)
}
return map[string]json.RawMessage{key: raw}, "", nil
return map[string]json.RawMessage{key: raw}, nil
}
keys, err := r.KeptState.Keys()
if err != nil {
return nil, "", err
return nil, err
}
sort.Strings(keys)
var matching []string
for _, k := range keys {
if l, _, err := delivery.ParseKeptKey(k); err == nil && (level == "" || l == level) {
matching = append(matching, k)
}
}
matching, cut := capKeys(matching, "give a level and a digest to read one")
values := make([]json.RawMessage, len(matching))
if err := parallel(len(matching), func(i int) (err error) {
values[i], err = r.KeptState.Get(matching[i])
return err
}); err != nil {
return nil, "", err
}
out := map[string]json.RawMessage{}
for i, k := range matching {
if len(values[i]) > 0 {
out[k] = values[i]
for _, k := range keys {
if l, _, err := delivery.ParseKeptKey(k); err != nil || (level != "" && l != level) {
continue
}
raw, err := r.KeptState.Get(k)
if err != nil {
return nil, err
}
if len(raw) > 0 {
out[k] = raw
}
}
return out, cut, nil
return out, nil
}
// PreviewRead is one preview as stored, with the parts of its key.
@@ -246,18 +214,17 @@ type PreviewRead struct {
// Previews are the previews the checker holds, each as stored. With a repository only that repository's; with a head (a
// prefix of at least 7 characters is enough) only previews of that head. A key that is no preview key is listed as unreadable.
func (r Reader) Previews(repository, head string) ([]PreviewRead, []string, string, error) {
func (r Reader) Previews(repository, head string) ([]PreviewRead, []string, error) {
if r.PreviewState.Keys == nil || r.PreviewState.Get == nil {
return nil, nil, "", fmt.Errorf("the previews state cannot be read here")
return nil, nil, fmt.Errorf("the previews state cannot be read here")
}
keys, err := r.PreviewState.Keys()
if err != nil {
return nil, nil, "", err
return nil, nil, err
}
sort.Strings(keys)
type part struct{ repo, head, trunk string }
parts := map[string]part{}
var matching, bad []string
out := []PreviewRead{}
var bad []string
for _, k := range keys {
repo, h, trunk, err := delivery.ParsePreviewKey(k)
if err != nil {
@@ -267,25 +234,15 @@ func (r Reader) Previews(repository, head string) ([]PreviewRead, []string, stri
if (repository != "" && repo != repository) || (head != "" && !strings.HasPrefix(h, head)) {
continue
}
parts[k] = part{repo, h, trunk}
matching = append(matching, k)
}
matching, cut := capKeys(matching, "give a repository and a head to narrow the answer")
values := make([]json.RawMessage, len(matching))
if err := parallel(len(matching), func(i int) (err error) {
values[i], err = r.PreviewState.Get(matching[i])
return err
}); err != nil {
return nil, nil, "", err
}
out := []PreviewRead{}
for i, k := range matching {
if len(values[i]) > 0 {
p := parts[k]
out = append(out, PreviewRead{Key: k, Repository: p.repo, Head: p.head, Trunk: p.trunk, Preview: values[i]})
raw, err := r.PreviewState.Get(k)
if err != nil {
return nil, nil, err
}
if len(raw) > 0 {
out = append(out, PreviewRead{Key: k, Repository: repo, Head: h, Trunk: trunk, Preview: raw})
}
}
return out, bad, cut, nil
return out, bad, nil
}
// RunningStep is a step in flight on a build module.
@@ -314,46 +271,7 @@ type Status struct {
// Waiting are the candidates whose check waits, by kind (ADR 0536 §4): new (no verdict for its heads) and nodes (failed
// for the nodes, asked again when they change) start before trunk (judged on trunk heads that moved since).
Waiting map[string][]string `json:"waiting_by_kind"`
// HeldByTurn are the candidates whose check an approved merge's turn holds off (ADR 0548 §3), by group: the repository and
// the group of the merge that waits. Their check is claimed when that turn goes.
HeldByTurn map[string]string `json:"held_by_turn"`
Notes []string `json:"notes,omitempty"`
}
// turns reads the gate's merge turns, by repository; nil when they cannot be read, said in a note.
func (r Reader) turns(st *Status) *Turns {
if r.TurnState.Keys == nil || r.TurnState.Get == nil {
st.Notes = append(st.Notes, "the gate's merge turns are not readable here, so no check is named as held by one")
return nil
}
keys, err := r.TurnState.Keys()
if err != nil {
st.Notes = append(st.Notes, "the gate's merge turns cannot be listed, so no check is named as held by one: "+err.Error())
return nil
}
t := &Turns{}
for _, k := range keys {
if raw, err := r.TurnState.Get(k); err == nil && len(raw) > 0 {
t.Put(k, raw, false)
}
}
return t
}
// heldBy says the turn that holds a candidate's check off, or "".
func heldBy(t *Turns, c delivery.Candidate, heads map[string]string) string {
cs := CaseOf(c, heads)
if t == nil || cs == nil {
return ""
}
repos := make([]string, 0, len(cs.Trunk))
for r := range cs.Trunk {
repos = append(repos, r)
}
if repo, w, ok := t.Holding(repos, cs.Heads); ok {
return repo + ": the approved merge of " + w.Group + " waits on it"
}
return ""
Notes []string `json:"notes,omitempty"`
}
// Status reads the build modules' steps and the candidates against their verdicts.
@@ -382,28 +300,16 @@ func (r Reader) Status() (Status, error) {
st.Notes = append(st.Notes, "the steps of "+m+" cannot be listed: "+err.Error())
continue
}
// A step that runs started lately: the newest ids are read first, at most MaxReads of them.
newestFirst(keys)
keys, cut := capKeys(keys, "")
if cut != "" {
st.Notes = append(st.Notes, "the steps of "+m+": read the newest "+fmt.Sprint(MaxReads)+" of its keys; a step older than those is not shown")
}
steps := make([]delivery.Step, len(keys))
isRunning := make([]bool, len(keys))
_ = parallel(len(keys), func(i int) error {
raw, err := sr.Get(keys[i])
sort.Strings(keys)
for _, k := range keys {
raw, err := sr.Get(k)
if err != nil || len(raw) == 0 {
return nil
}
step, err := delivery.DecodeStep(raw)
steps[i], isRunning[i] = step, err == nil && step.State == delivery.StepRunning
return nil
})
for i := range keys {
if !isRunning[i] {
continue
}
step := steps[i]
step, err := delivery.DecodeStep(raw)
if err != nil || step.State != delivery.StepRunning {
continue
}
rs := RunningStep{Module: m, Node: step.Node, ID: step.ID, Tool: step.Tool}
if !step.Started.IsZero() {
rs.Since = step.Started.UTC().Format("2006-01-02T15:04:05Z")
@@ -424,8 +330,6 @@ func (r Reader) Status() (Status, error) {
}
st.Unjudged = []string{}
st.Waiting = map[string][]string{KindNew: {}, KindNodes: {}, KindTrunk: {}}
st.HeldByTurn = map[string]string{}
turns := r.turns(&st)
var heads map[string]string
if r.Trunks == nil {
st.Notes = append(st.Notes, "the trunk heads are not readable here, so no check is named as waiting for a trunk move")
@@ -440,47 +344,26 @@ func (r Reader) Status() (Status, error) {
return st, err
}
sort.Strings(keys)
total := len(keys)
keys, cut := capKeys(keys, "")
if cut != "" {
st.Notes = append(st.Notes, fmt.Sprintf("read the first %d candidates of %d", MaxReads, total))
}
type judged struct {
c delivery.Candidate
raw json.RawMessage
ok bool
}
rows := make([]judged, len(keys))
_ = parallel(len(keys), func(i int) error {
raw, err := r.CandidateState.Get(keys[i])
for _, k := range keys {
raw, err := r.CandidateState.Get(k)
if err != nil || len(raw) == 0 {
return nil
continue
}
c, err := delivery.DecodeCandidate(raw)
if err != nil {
return nil
continue
}
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
if err != nil {
return nil
}
rows[i].c = c
if vraw, err := r.VerdictState.Get(vkey); err == nil {
rows[i].raw, rows[i].ok = vraw, true
}
return nil
})
for _, row := range rows {
if !row.ok {
continue
}
c, raw := row.c, row.raw
raw, err = r.VerdictState.Get(vkey)
if err != nil {
continue
}
if len(raw) == 0 {
st.Unjudged = append(st.Unjudged, c.Group)
st.Waiting[KindNew] = append(st.Waiting[KindNew], c.Group)
if why := heldBy(turns, c, heads); why != "" {
st.HeldByTurn[c.Group] = why
}
continue
}
v, err := delivery.DecodeVerdict(raw)
@@ -489,11 +372,8 @@ func (r Reader) Status() (Status, error) {
case IsNodesFailure(v):
st.Waiting[KindNodes] = append(st.Waiting[KindNodes], c.Group)
case heads != nil:
if cs := CaseOf(c, heads); cs != nil && (!sameTrunk(v.Trunk, cs.Trunk) || besideMoved(v.Beside, heads)) {
if cs := CaseOf(c, heads); cs != nil && !sameTrunk(v.Trunk, cs.Trunk) {
st.Waiting[KindTrunk] = append(st.Waiting[KindTrunk], c.Group)
if why := heldBy(turns, c, heads); why != "" {
st.HeldByTurn[c.Group] = why
}
}
}
}
@@ -3,9 +3,7 @@ package check
import (
"context"
"encoding/json"
"fmt"
"strings"
"sync/atomic"
"testing"
"time"
@@ -53,7 +51,7 @@ func readerOf(t *testing.T) (Reader, string) {
func TestTheVerdictToolsAnswerTheStoredVerdictWithTheFullReportOfEachPullRequest(t *testing.T) { // N9/AC1, 1686
rd, key := readerOf(t)
vs, _, err := rd.Verdicts("")
vs, err := rd.Verdicts("")
if err != nil || len(vs) != 1 || vs[0].Key != key {
t.Fatalf("verdicts = %+v, %v", vs, err)
}
@@ -63,7 +61,7 @@ func TestTheVerdictToolsAnswerTheStoredVerdictWithTheFullReportOfEachPullRequest
if got := vs[0].Reports["novox/mesh-delivery#7"]; !strings.HasPrefix(got, "pass 111111111111 on aaaaaaaaaaaa") || !strings.Contains(got, "composition not yet run") {
t.Errorf("reports = %v", vs[0].Reports)
}
if only, _, _ := rd.Verdicts("feat/other"); len(only) != 0 {
if only, _ := rd.Verdicts("feat/other"); len(only) != 0 {
t.Errorf("another group's verdicts = %+v", only)
}
}
@@ -87,11 +85,11 @@ func TestAnHqPullRequestIsAnsweredAsAnyOther(t *testing.T) {
func TestTheKeptToolAnswersTheKeysAsStoredByLevelAndByDigest(t *testing.T) { // N9/AC1
rd, _ := readerOf(t)
all, _, err := rd.Kept("", "")
all, err := rd.Kept("", "")
if err != nil || len(all) != 1 {
t.Fatalf("kept = %v, %v", all, err)
}
one, _, err := rd.Kept("unit", digA)
one, err := rd.Kept("unit", digA)
if err != nil || len(one) != 1 {
t.Fatalf("one kept = %v, %v", one, err)
}
@@ -100,10 +98,10 @@ func TestTheKeptToolAnswersTheKeysAsStoredByLevelAndByDigest(t *testing.T) { //
t.Errorf("kept value = %+v, %v", k, err)
}
}
if none, _, _ := rd.Kept("integration", ""); len(none) != 0 {
if none, _ := rd.Kept("integration", ""); len(none) != 0 {
t.Errorf("integration = %v", none)
}
if _, _, err := rd.Kept("unit", digB); err == nil {
if _, err := rd.Kept("unit", digB); err == nil {
t.Error("a digest nothing kept was answered")
}
}
@@ -117,17 +115,17 @@ func TestThePreviewsToolShowsTheStoredPreviewOfAPullRequest(t *testing.T) {
}
key, _ := delivery.PreviewKey(repoA, head1, head2)
rd := Reader{PreviewState: fakeState{key: data, "junk": []byte("{}")}.read()}
got, bad, _, err := rd.Previews(repoA, head1[:8])
got, bad, err := rd.Previews(repoA, head1[:8])
if err != nil || len(got) != 1 || got[0].Key != key || got[0].Head != head1 || got[0].Trunk != head2 || string(got[0].Preview) != string(data) {
t.Fatalf("previews = %+v, %v", got, err)
}
if len(bad) != 1 || bad[0] != "junk" {
t.Errorf("unreadable = %v", bad)
}
if got, _, _, _ := rd.Previews("novox/other", ""); len(got) != 0 {
if got, _, _ := rd.Previews("novox/other", ""); len(got) != 0 {
t.Errorf("another repository's = %+v", got)
}
if _, _, _, err := (Reader{}).Previews("", ""); err == nil {
if _, _, err := (Reader{}).Previews("", ""); err == nil {
t.Error("answered with no state to read")
}
}
@@ -234,134 +232,4 @@ func TestTheStatusCountsAnHQOnlyGroupAsUnjudged(t *testing.T) {
if len(st.Unjudged) != 1 || st.Unjudged[0] != "pb-x" {
t.Errorf("unjudged = %v, want pb-x", st.Unjudged)
}
// ADR 0548 §3: a candidate whose check an approved merge's turn holds off is named with the turn.
held := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
hkey, _ := delivery.CandidatesKey("feat/x")
st, err = Reader{CandidateState: fakeState{hkey: held}.read(), VerdictState: fakeState{}.read(),
Trunks: func() (map[string]string, error) { return map[string]string{repoA: trunkA}, nil },
TurnState: fakeState{turnKey(t, repoA): turnOf(t, repoA, "fix/other", "0123456789ab")}.read()}.Status()
if err != nil {
t.Fatal(err)
}
if why := st.HeldByTurn["feat/x"]; !strings.Contains(why, repoA) || !strings.Contains(why, "fix/other") {
t.Errorf("held_by_turn = %v, want feat/x held by the merge of fix/other on %s", st.HeldByTurn, repoA)
}
}
// slowBucket wraps a state with a Get that takes delay and counts the values read, the lists and the most Gets in flight.
type slowBucket struct {
vals fakeState
reads, lists atomic.Int64
inFlight, maxFlight atomic.Int64
}
func (b *slowBucket) get(key string) (json.RawMessage, error) {
b.reads.Add(1)
now := b.inFlight.Add(1)
for m := b.maxFlight.Load(); now > m && !b.maxFlight.CompareAndSwap(m, now); m = b.maxFlight.Load() {
}
time.Sleep(time.Millisecond)
b.inFlight.Add(-1)
return b.vals.read().Get(key)
}
func (b *slowBucket) read() StateRead {
return StateRead{Keys: func() ([]string, error) { b.lists.Add(1); return b.vals.read().Keys() }, Get: b.get}
}
func (b *slowBucket) Keys() ([]string, error) { return b.read().Keys() }
func (b *slowBucket) Get(key string) (json.RawMessage, error) { return b.get(key) }
func (b *slowBucket) Create(string, []byte) error { return nil }
// 1827: a read tool lists the keys of a bucket of thousands and reads only the values its answer needs, sixteen at a time, at
// most MaxReads, saying when it cut (on the live shape of 2026-10-26: kept 7990 keys, steps 10034, tries 1000s).
func TestReadToolsReadOnlyWhatTheyNeedInParallelAndSayWhenCut(t *testing.T) { // 1827
kept := fakeState{}
for i := 0; i < 1800; i++ {
level := map[bool]string{true: "unit", false: "integration"}[i%9 != 0]
k, _ := delivery.KeptKey(level, fmt.Sprintf("%064x", i))
kept[k] = []byte(`{"v":1}`)
}
previews := fakeState{}
for i := 0; i < 1500; i++ {
repo := map[bool]string{true: repoA, false: repoSDK}[i < 300]
k, _ := delivery.PreviewKey(repo, fmt.Sprintf("%08x%032x", i, 0), trunkA)
previews[k] = []byte(`{"v":1}`)
}
for _, row := range []struct {
name string
call func(*slowBucket) (rows int, cut string, err error)
rows int
reads, lists int64
cut bool
}{
{"kept of one key", func(b *slowBucket) (int, string, error) {
m, c, err := Reader{KeptState: b.read()}.Kept("unit", fmt.Sprintf("%064x", 1))
return len(m), c, err
}, 1, 1, 0, false},
{"kept of a level", func(b *slowBucket) (int, string, error) {
m, c, err := Reader{KeptState: b.read()}.Kept("integration", "")
return len(m), c, err
}, 200, 200, 1, false},
{"kept of everything", func(b *slowBucket) (int, string, error) {
m, c, err := Reader{KeptState: b.read()}.Kept("", "")
return len(m), c, err
}, MaxReads, MaxReads, 1, true},
{"previews of a repository", func(b *slowBucket) (int, string, error) {
p, _, c, err := Reader{PreviewState: b.read()}.Previews(repoSDK, "")
return len(p), c, err
}, 1200, 1000, 1, true},
{"previews of a repository and a head", func(b *slowBucket) (int, string, error) {
p, _, c, err := Reader{PreviewState: b.read()}.Previews(repoA, fmt.Sprintf("%08x", 7))
return len(p), c, err
}, 1, 1, 1, false},
} {
b := &slowBucket{}
b.vals = kept
if strings.HasPrefix(row.name, "previews") {
b.vals = previews
}
start := time.Now()
rows, cut, err := row.call(b)
if err != nil {
t.Fatalf("%s: %v", row.name, err)
}
if rows != min(row.rows, MaxReads) || b.reads.Load() != row.reads || b.lists.Load() != row.lists || (cut != "") != row.cut {
t.Errorf("%s: %d rows, %d reads, %d lists, cut %q; want %d, %d, %d, cut %v", row.name, rows, b.reads.Load(), b.lists.Load(), cut, min(row.rows, MaxReads), row.reads, row.lists, row.cut)
}
if row.reads > 20 && b.maxFlight.Load() < 2 {
t.Errorf("%s: values were read one at a time", row.name)
}
t.Logf("%s: %d values in %v (one after the other: %v)", row.name, b.reads.Load(), time.Since(start), time.Duration(b.reads.Load())*time.Millisecond)
}
// The status reads the newest steps of a bucket of 1500 first, at most MaxReads, and finds the one that runs.
now := time.Date(2026, 10, 26, 7, 0, 0, 0, time.UTC)
steps := fakeState{}
for i := 0; i < 1500; i++ {
id := fmt.Sprintf("mvo5%04dabcdef", i)
step := delivery.Step{ID: id, Tool: "t", Node: "g14", State: delivery.StepPassed}
if i == 1499 {
step.State, step.Started = delivery.StepRunning, now.Add(-time.Minute)
}
raw, _ := delivery.EncodeStep(step)
steps["g14."+id] = raw
}
sb := &slowBucket{vals: steps}
st, err := Reader{StepStates: map[string]StateRead{"mesh-build-go": sb.read()}, HaveState: fakeState{"g14.mesh-build-go": haveOf(t, delivery.PhaseRunning)}.read(),
Now: func() time.Time { return now }}.Status()
if err != nil || sb.reads.Load() != MaxReads || len(st.Running) != 1 || sb.maxFlight.Load() < 2 {
t.Errorf("status: %d reads (want %d), running %+v, in flight %d, %v", sb.reads.Load(), MaxReads, st.Running, sb.maxFlight.Load(), err)
}
// A new try reads the newest TryScan of 1500 ended tries, not all, and still finds the same head asked twice.
tries := fakeState{}
for i := 0; i < 1500; i++ {
raw, _ := encodeTry(Try{ID: fmt.Sprintf("try-%d", 1700000000000000000+i), Repository: repoA, Head: head1, Trunk: trunkA, State: TryPass})
tries[fmt.Sprintf("try-%d", 1700000000000000000+i)] = raw
}
tb := &slowBucket{vals: tries}
if _, err := AskTry(tb, tryTrunks, TryAsk{Repository: repoA, Head: head2, Trunk: trunkA}, now); err != nil || tb.reads.Load() != TryScan {
t.Errorf("AskTry read %d of 1500 tries (want %d), %v", tb.reads.Load(), TryScan, err)
}
}
@@ -61,6 +61,9 @@ type Failure struct {
Output string `json:"output,omitempty"`
// Cause is the first line of what the test reported as its error, when the build module told it apart (issue 1901).
Cause string `json:"cause,omitempty"`
// Step is "lint" for a finding of the lint step (gofmt, go vet), which runs beside the unit level and is reported as
// its own failure, not the level's (ADR 0542 §2).
Step string `json:"step,omitempty"`
}
// TestTime is one test and the seconds it took.
@@ -0,0 +1,102 @@
package check
import (
"context"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
const repoC = "novox/mesh-controller"
func statusOf(t *testing.T, r *rig, repo string, n int, head string) string {
t.Helper()
key, _ := delivery.CheckerOwedKey(repo, n, head)
raw, _, found, _ := r.owed.Get(context.Background(), key)
if !found {
return ""
}
o, err := delivery.DecodeCheckerOwed(raw)
if err != nil {
t.Fatal(err)
}
return o.State
}
// 1791 note 9 (issue 1921, try speed S9): two of a group's three members merge. The candidate keeps them, marked merged,
// with the same heads; their trunks moved by their own merges. No new check is asked, and the member left keeps its
// passed status, so the merge of the rest is not refused.
func TestTwoMembersMergingAskNoNewCheckAndTheThirdKeepsItsPass(t *testing.T) {
ctx, j := context.Background(), passJudge()
r := newRig(t, j)
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS, repoC: trunkA})
a, s, c := pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2), pullOf(repoC, 3, head1)
r.ch.Candidate(ctx, "feat%2Fx", candidateOf(t, "feat/x", a, s, c))
r.ch.Wait()
if j.count() != 1 || statusOf(t, r, repoC, 3, head1) != "success" {
t.Fatalf("judged %d times, the third's status %q", j.count(), statusOf(t, r, repoC, 3, head1))
}
a.Merged, s.Merged = true, true
r.ch.Candidate(ctx, "feat%2Fx", candidateOf(t, "feat/x", a, s, c))
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS2, repoC: trunkA})
r.ch.TrunkMoved(ctx)
r.ch.Wait()
if j.count() != 1 {
t.Errorf("a new check was asked after two members merged: judged %d times", j.count())
}
if got := statusOf(t, r, repoC, 3, head1); got != "success" {
t.Errorf("the third member's status is %q, want success", got)
}
// The trunk of the member left moves: that is a re-judge as before.
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS2, repoC: trunkA2})
r.ch.TrunkMoved(ctx)
r.ch.Wait()
if j.count() != 2 {
t.Errorf("the member left's trunk moved and it was judged %d times, want again", j.count())
}
}
// 1791 note 8 (issue 1921, try speed S9): a group opened member by member writes a candidate for each partial set. When
// the group's candidate moves to new heads, the check of the older heads is stopped and ends failed, superseded, instead
// of running on or staying running; it puts no status, as the newer check owns the heads they share.
func TestACheckWhoseHeadsTheGroupNoLongerHoldsIsStoppedAndEndsSuperseded(t *testing.T) {
ctx := context.Background()
started := make(chan struct{})
j := &judgeFunc{f: func(ctx context.Context, c *Case, call int) (Outcome, error) {
if len(c.Pulls) == 1 {
close(started)
select {
case <-ctx.Done():
return Outcome{Verdict: failure("checker", ctx.Err().Error())}, nil
case <-time.After(3 * time.Second):
return Outcome{Verdict: passing()}, nil // never stopped
}
}
return Outcome{Verdict: passing()}, nil
}}
r := newRig(t, j)
partial := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
whole := candidateOf(t, "feat/x", pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))
r.ch.Candidate(ctx, "feat%2Fx", partial)
<-started
r.ch.Candidate(ctx, "feat%2Fx", whole)
done := make(chan struct{})
go func() { r.ch.Wait(); close(done) }()
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("the check of the older heads was not stopped")
}
old := r.verdict(t, keyOf(t, "feat/x", partial))
if old.State != delivery.VerdictFail || !strings.Contains(old.Failed.Words, "superseded") {
t.Errorf("the older heads' verdict = %+v, want failed, superseded", old)
}
if v := r.verdict(t, keyOf(t, "feat/x", whole)); v.State != delivery.VerdictPass {
t.Errorf("the newer heads' verdict = %+v", v)
}
if got := statusOf(t, r, repoA, 7, head1); got != "success" {
t.Errorf("the shared head's status is %q, want the newer check's success", got)
}
}
@@ -29,8 +29,6 @@ const (
TriesBucket = delivery.CheckerModule + "_" + TriesState
// MaxTries is how many tries may wait or run at once, mesh-wide: each holds a build node for minutes.
MaxTries = 4
// TryScan is how many of the newest tries a new ask reads to find the ones that wait or run (issue 1827).
TryScan = 64
// TryKeep is how long an ended try's key is kept before the checker removes it.
TryKeep = 7 * 24 * time.Hour
)
@@ -159,33 +157,15 @@ func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (
if err != nil {
return "", fmt.Errorf("the tries cannot be listed: %w", err)
}
// An ended try's key stays TryKeep, so the bucket holds thousands; one that waits or runs was asked lately. Only the
// newest TryScan keys are read, in parallel (a try id is the time in nanoseconds, so the newest sorts last).
sort.Slice(keys, func(i, j int) bool {
if len(keys[i]) != len(keys[j]) {
return len(keys[i]) > len(keys[j])
}
return keys[i] > keys[j]
})
if len(keys) > TryScan {
keys = keys[:TryScan]
}
tries := make([]Try, len(keys))
live := make([]bool, len(keys))
if err := parallel(len(keys), func(i int) error {
raw, err := store.Get(keys[i])
sort.Strings(keys)
var open []string
for _, k := range keys {
raw, err := store.Get(k)
if err != nil || len(raw) == 0 {
return nil
continue
}
cur, err := DecodeTry(raw)
tries[i], live[i] = cur, err == nil && !cur.Over()
return nil
}); err != nil {
return "", err
}
var open []string
for i, cur := range tries {
if !live[i] {
if err != nil || cur.Over() {
continue
}
if cur.Repository == t.Repository && cur.Head == t.Head && cur.Trunk == t.Trunk && cur.Tests == t.Tests && cur.Level == t.Level {
@@ -1,78 +0,0 @@
package check
import (
"sort"
"strings"
"sync"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// Turns holds the gate's merge turns (novox/hq ADR 0548 §3), by repository, as a watch of the gate's turns bucket puts them
// in: the checker claims no check of a repository a turn names, except the check of a waiting delivery's own heads, so an
// approved merge goes before a check not yet started. A bucket not there is read as empty.
type Turns struct {
mu sync.Mutex
byRepo map[string]delivery.Turn
}
// Put takes a value of the turns bucket in, or its delete, and answers whether what the checker holds changed. A value that
// is not a turn is dropped as a delete would be: a turn the checker cannot read holds nothing.
func (t *Turns) Put(key string, value []byte, deleted bool) bool {
repository, err := delivery.ParseTurnsKey(key)
if err != nil {
return false
}
t.mu.Lock()
defer t.mu.Unlock()
if t.byRepo == nil {
t.byRepo = map[string]delivery.Turn{}
}
cur, had := t.byRepo[repository]
if !deleted {
if turn, err := delivery.DecodeTurn(value); err == nil {
t.byRepo[repository] = turn
return !had || !delivery.SameTurn(cur, turn)
}
}
delete(t.byRepo, repository)
return had
}
// Holding names the first of the repositories (by name) a turn holds off the check of the heads, and the delivery that
// waits first in it; false when none does.
func (t *Turns) Holding(repositories []string, heads string) (string, delivery.TurnWait, bool) {
if t == nil {
return "", delivery.TurnWait{}, false
}
sorted := make([]string, 0, len(repositories))
for _, r := range repositories {
sorted = append(sorted, strings.ToLower(r))
}
sort.Strings(sorted)
t.mu.Lock()
defer t.mu.Unlock()
for _, r := range sorted {
turn, ok := t.byRepo[r]
if !ok || !delivery.TurnHolds(turn, heads) {
continue
}
for _, w := range turn.Waiting {
if w.Heads != heads {
return r, w, true
}
}
}
return "", delivery.TurnWait{}, false
}
// All is every turn held now, by repository (a copy).
func (t *Turns) All() map[string]delivery.Turn {
t.mu.Lock()
defer t.mu.Unlock()
out := make(map[string]delivery.Turn, len(t.byRepo))
for r, turn := range t.byRepo {
out[r] = turn
}
return out
}
@@ -97,7 +97,7 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
t.Errorf("state = %+v, want %+v", m.State, want)
}
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-controller.assignments",
"mesh-delivery-assigner.compositions", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-gate.turns", "mesh-delivery-indexer.candidates",
"mesh-delivery-assigner.compositions", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates",
"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) {
t.Errorf("reads = %v, want %v", m.Reads, want)
}
@@ -25,7 +25,6 @@
"mesh-delivery-assigner.compositions",
"mesh-delivery-builder.builds",
"mesh-delivery-gate.source",
"mesh-delivery-gate.turns",
"mesh-delivery-indexer.candidates",
"mesh-delivery-indexer.manifests",
"mesh-delivery-indexer.modules",
@@ -22,8 +22,6 @@ import (
"syscall"
"time"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
@@ -190,18 +188,18 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
call := callerOn(conn, cred.User)
// No forge event is heard while the open pull requests are read and said (ADR 0443 §5): events and syncs take turns.
var turn sync.Mutex
kv, err := process.EssentialBucket(ctx, js, "bridge", headsBucket, "keep the heads it wrote on the forge stream")
kv, err := js.KeyValue(ctx, headsBucket)
if err != nil {
return err
return fmt.Errorf("the bucket %s: %w", headsBucket, err)
}
heads := kvHeads{kv}
// Everything written on FORGE, an event's and a sync's, keeps the head in the bucket.
rec := &recorder{pub: js, heads: heads, logger: logger}
said := func(ctx context.Context) error { return syncOpen(ctx, call, rec, heads, &turn, logger) }
// What the gate asks of the forge (ADR 0437 §8): its owed acts and its merges, each performed once per revision.
performedKV, err := process.EssentialBucket(ctx, js, "bridge", performedBucket, "perform an act of the gate only once")
performedKV, err := js.KeyValue(ctx, performedBucket)
if err != nil {
return err
return fmt.Errorf("the bucket %s: %w", performedBucket, err)
}
perf := &performer{call: call, pub: rec, done: kvRevisions{performedKV}, logger: logger}
watching := make(chan error, 3)
@@ -209,12 +207,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
bucket string
react func(ctx context.Context, key string, rev uint64, value []byte)
}{{delivery.OwedBucket, perf.owed}, {delivery.MergesBucket, perf.merges}} {
// The gate's buckets come with the gate and its grant; the bridge does not wait for them (no timer), and is started
// again by its delivery when the gate arrives. A bucket refused or not answered is the same (issue 1945).
kv := process.OptionalBucket(ctx, js, w.bucket, "nothing of it is performed until the bridge starts again", logger.Printf)
if kv == nil {
kv, err := js.KeyValue(ctx, w.bucket)
if errors.Is(err, jetstream.ErrBucketNotFound) {
// The gate's buckets come with the gate; the bridge does not wait for them (no timer), and is started again
// by its delivery when the gate arrives.
logger.Printf("the bucket %s does not exist yet: nothing of it is performed until the bridge starts again", w.bucket)
continue
}
if err != nil {
return fmt.Errorf("the bucket %s: %w", w.bucket, err)
}
go func() {
watching <- watch(ctx, kv, func(key string, rev uint64, value []byte) {
hctx, cancel := context.WithTimeout(ctx, syncDeadline)
@@ -229,8 +231,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
go onReconnect(ctx, again, said, logger)
cons, err := js.Consumer(ctx, eventsStream, cred.Node+"_"+cred.Module)
if err != nil {
// Essential: the forge's events are what the bridge carries.
return fmt.Errorf("the consumer %s_%s on %s cannot be read (%v): the bridge hears no forge event without it, so it stops; where the bus refused it or did not answer, this module's consumes (made by the controller with its grant) have not reached the bus yet", cred.Node, cred.Module, eventsStream, err)
return fmt.Errorf("the consumer %s_%s on %s: %w", cred.Node, cred.Module, eventsStream, err)
}
cc, err := cons.Consume(func(m jetstream.Msg) {
hear(&turn, func() {
@@ -5,7 +5,6 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"slices"
"strings"
@@ -13,8 +12,6 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
@@ -69,16 +66,6 @@ func body(m map[string]any) []byte {
}
func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit part; the bus is the integration test)
t.Run("1945: a refused bucket is said, or stops the start naming its grant", func(t *testing.T) {
refusal := errors.New("nats: permissions violation for publish to $JS.API.STREAM.INFO.KV_x")
var said []string
if kv := process.OptionalBucket(context.Background(), refusingBus{refusal}, "x", "x is not heard", func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }); kv != nil || len(said) != 1 || !strings.Contains(said[0], "permissions violation") {
t.Errorf("an optional bucket refused: %v, said %v", kv, said)
}
if _, err := process.EssentialBucket(context.Background(), refusingBus{refusal}, "bridge", "x", "work"); err == nil || !strings.Contains(err.Error(), "grant of x") {
t.Errorf("an essential bucket refused: %v, want its grant named", err)
}
})
var logs bytes.Buffer
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{}
handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0))
@@ -243,8 +230,3 @@ func someFiles(_, _, path, _ string) ([]byte, error) {
}
return nil, bridge.ErrNotFound
}
// refusingBus refuses every bucket with err (issue 1945).
type refusingBus struct{ err error }
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { return nil, r.err }
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
@@ -1,7 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -7,10 +7,8 @@ import (
"encoding/json"
"fmt"
"io"
"maps"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"time"
@@ -440,9 +438,8 @@ func TestIndexerGivesACandidateItsFingerprintWhenTheCheckerCreatesThePreviewsBuc
// TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual: two merged messages of mesh-sdk (its module.json at 9cb4d59,
// whose lib names its paths) differ in the root tree, as a change to go/ alone makes them, and carry the same objects of the
// named paths; the module's kept tree, and so its fingerprint's tree input, is the same after both, its new form
// (fingerprint or reason) does not move, and so the second writes no value at all (issue 1758, I9, ADR 0499 check clause 6
// in miniature; the operator, 2026-10-25: a module's value moves only with the module).
// named paths; the module's kept tree, and so its fingerprint's tree input, is the same after both, and its new form
// (fingerprint or reason) does not move (issue 1758, I9, ADR 0499 check clause 6 in miniature).
func TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual(t *testing.T) {
manifest, err := os.ReadFile("../../internal/index/testdata/sdk-9cb4d59-mesh-sdk.module.json")
if err != nil {
@@ -451,7 +448,7 @@ func TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual(t *testing.T) {
objects := map[string]string{"src": "1111111111111111111111111111111111111111", "test": "2222222222222222222222222222222222222222",
"package.json": "3333333333333333333333333333333333333333", "package-lock.json": "4444444444444444444444444444444444444444", "tsconfig.json": "5555555555555555555555555555555555555555"}
js, ctx, _, modules := startIndexer(t, func(context.Context, jetstream.KeyValue) {})
publish := func(merge, root string, objects map[string]string) {
merged := func(merge, root string) delivery.Module {
data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-sdk", Branch: "feat/x", Number: 70,
Head: "0123456789abcdef0123456789abcdef01234567", Merge: merge, Root: root, Trees: map[string]string{"": root},
Manifests: map[string]string{"": string(manifest)}, Paths: map[string]map[string]string{"": objects}})
@@ -461,8 +458,6 @@ func TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual(t *testing.T) {
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-sdk.merged", data); err != nil {
t.Fatal(err)
}
}
at := func(merge string) delivery.Module {
var m delivery.Module
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err := modules.Get(ctx, "mesh-sdk"); err == nil {
@@ -474,35 +469,13 @@ func TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual(t *testing.T) {
t.Fatalf("modules.mesh-sdk = %+v within 5 s, want the merge %s", m, merge)
return m
}
const firstMerge, secondMerge, thirdMerge = "89abcdef0123456789abcdef0123456789abcdef", "99abcdef0123456789abcdef0123456789abcdef", "a9abcdef0123456789abcdef0123456789abcdef"
publish(firstMerge, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", objects)
first := at(firstMerge)
if first.Inputs2 == nil || !strings.HasPrefix(first.Inputs2.Tree, delivery.NamedTreePrefix) {
t.Fatalf("kept inputs %+v, want a named tree", first.Inputs2)
first := merged("89abcdef0123456789abcdef0123456789abcdef", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
second := merged("99abcdef0123456789abcdef0123456789abcdef", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")
if first.Inputs2 == nil || second.Inputs2 == nil || first.Inputs2.Tree != second.Inputs2.Tree || !strings.HasPrefix(first.Inputs2.Tree, delivery.NamedTreePrefix) {
t.Fatalf("kept trees %+v and %+v, want one named tree", first.Inputs2, second.Inputs2)
}
// The go/-only merge, then one that changes src/: the stream is taken in order, so once the third is written the second
// was heard, and it wrote nothing: the module's value stays at the commit its fingerprint first appeared at (the
// operator, 2026-10-25).
publish(secondMerge, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", objects)
changed := maps.Clone(objects)
changed["src"] = "6666666666666666666666666666666666666666"
publish(thirdMerge, "cccccccccccccccccccccccccccccccccccccccc", changed)
at(thirdMerge)
history, err := modules.History(ctx, "mesh-sdk")
if err != nil {
t.Fatal(err)
}
var commits []string
for _, e := range history {
if m, err := delivery.DecodeModule(e.Value()); err == nil {
commits = append(commits, m.Commit)
if m.Commit == firstMerge && (m.Inputs2 == nil || m.Inputs2.Tree != first.Inputs2.Tree || m.FP2 != first.FP2 || m.None2 != first.None2) {
t.Errorf("the value at the first merge moved: %+v, was %+v", m, first)
}
}
}
if slices.Contains(commits, secondMerge) {
t.Errorf("the go/-only merge wrote a value: the commits written are %v", commits)
if first.FP2 != second.FP2 || first.None2 != second.None2 {
t.Errorf("the new form moved: %q/%q, then %q/%q", first.FP2, first.None2, second.FP2, second.None2)
}
}
@@ -642,33 +615,17 @@ func TestIndexerCountsOnlyTheAssignedBuildNodesToolchains(t *testing.T) {
return got
}
agreed := func(m delivery.Module) bool { return m.FP2 != "" && m.Toolchains["go"].Version == "go1.27.3" }
held := read(agreed)
if !agreed(held) {
t.Fatalf("within 5 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27.3 alone", held.FP2, held.None2, held.Toolchains)
if m := read(agreed); !agreed(m) {
t.Fatalf("within 5 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27.3 alone", m.FP2, m.None2, m.Toolchains)
}
if _, err := assignments.Put(ctx, "n2.mesh-build-go", []byte(`{"node":"n2","module":"mesh-build-go","assigned":"2026-10-24T08:00:00Z"}`)); err != nil {
t.Fatal(err)
}
// n2's report, written since its assignment, counts: the builders disagree. A fingerprint already computed is kept
// (issue 1940), and only a new one needs the agreement: a merge moving avahi gets none until they agree.
time.Sleep(500 * time.Millisecond)
if e, err := modules.Get(ctx, "avahi"); err != nil {
t.Fatal(err)
} else if m, err := delivery.DecodeModule(e.Value()); err != nil || m.FP2 != held.FP2 {
t.Fatalf("after n2's assignment avahi has fp2 %q, none2 %q: want its fingerprint %q kept", m.FP2, m.None2, held.FP2)
}
moved := strings.Replace(string(data), `"newtree"`, `"movedtree"`, 1)
if moved == string(data) {
t.Fatal("the merge names no tree to move")
}
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-catalog.merged", []byte(moved)); err != nil {
t.Fatal(err)
}
disagree := func(m delivery.Module) bool {
return m.FP2 == "" && strings.HasPrefix(m.None2, "builders disagree: go1.27.2 recipe 1 on n2")
}
if m := read(disagree); !disagree(m) {
t.Fatalf("within 5 s of a merge after n2's assignment avahi has fp2 %q, none2 %q: want the builders to disagree", m.FP2, m.None2)
t.Fatalf("within 5 s of n2's assignment avahi has fp2 %q, none2 %q: want the builders to disagree", m.FP2, m.None2)
}
if _, err := assignments.Put(ctx, "n2.mesh-build-go", []byte(`{"node":"n2","module":"mesh-build-go","ended":"2026-10-24T09:00:00Z"}`)); err != nil {
t.Fatal(err)
@@ -240,21 +240,29 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil {
return err
}
candidates, err := process.EssentialBucket(ctx, js, "indexer", delivery.CandidatesBucket, "write a candidate's fingerprints")
bucket := func(name string) (jetstream.KeyValue, error) {
kv, err := js.KeyValue(ctx, name)
if err != nil {
return nil, fmt.Errorf("the bucket %s: %w", name, err)
}
return kv, nil
}
candidates, err := bucket(delivery.CandidatesBucket)
if err != nil {
return err
}
modules, err := process.EssentialBucket(ctx, js, "indexer", delivery.ModulesBucket, "write a module's fingerprint")
modules, err := bucket(delivery.ModulesBucket)
if err != nil {
return err
}
manifests, err := process.EssentialBucket(ctx, js, "indexer", delivery.ManifestsBucket, "keep a module's manifest")
manifests, err := bucket(delivery.ManifestsBucket)
if err != nil {
return err
}
builds, err := bucket(delivery.BuildsBucket)
if err != nil {
return err
}
// The controller's build records only name the directory of a module whose key names none (fillPaths): without them
// such a module waits for its path, every other is indexed (issue 1945).
builds := process.OptionalBucket(ctx, js, delivery.BuildsBucket, "a module whose key names no directory is not given one from the controller's build records until the indexer starts again", logger.Printf)
known := index.NewBuilds()
// The controller's build records name the directory of a module whose key names none (fillPaths); no build of any
// module moves a fingerprint since no fingerprint takes the builder (ADR 0530 §1).
@@ -265,14 +273,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
known.Delete(e.Key())
}
}
errs := make(chan error, 1)
if builds != nil {
if w, err := settle(ctx, builds, each); err != nil {
process.NotWatched(logger.Printf, delivery.BuildsBucket, err, "a module whose key names no directory is not given one until the indexer starts again")
} else {
go func() { errs <- follow(ctx, w, each) }()
}
w, err := settle(ctx, builds, each)
if err != nil {
return err
}
errs := make(chan error, 1)
go func() { errs <- follow(ctx, w, each) }()
// The indexer asks the forge for the objects of a manifest's named paths as its own account (ADR 0499 clause 3).
account, err := process.ReadAccount(path)
@@ -290,17 +296,22 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does
// not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate.
sourceErrs := make(chan error, 1)
if source := process.OptionalBucket(ctx, js, delivery.SourceBucket, "no source revision is read until the indexer starts again", logger.Printf); source != nil {
switch source, err := js.KeyValue(ctx, delivery.SourceBucket); {
case errors.Is(err, jetstream.ErrBucketNotFound):
logger.Printf("the bucket %s does not exist yet: no source revision is read until the indexer starts again", delivery.SourceBucket)
case err != nil:
return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err)
default:
each := func(e jetstream.KeyValueEntry) {
if opOf(e) {
applySource(ctx, ix, e.Key(), e.Revision(), e.Value(), logger)
}
}
if sw, err := settle(ctx, source, each); err != nil {
process.NotWatched(logger.Printf, delivery.SourceBucket, err, "no source revision is read until the indexer starts again")
} else {
go func() { sourceErrs <- follow(ctx, sw, each) }()
sw, err := settle(ctx, source, each)
if err != nil {
return err
}
go func() { sourceErrs <- follow(ctx, sw, each) }()
}
// Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's
// assignments are settled first, so the toolchains settle through the gate already knowing them.
@@ -370,8 +381,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
cons, err := js.Consumer(ctx, forgeStream, cred.Node+"_"+cred.Module)
if err != nil {
// Essential: the forge's merges are what the indexer indexes.
return fmt.Errorf("the consumer %s_%s on %s cannot be read (%v): the indexer hears no merge without it, so it stops; where the bus refused it or did not answer, this module's consumes (made by the controller with its grant) have not reached the bus yet", cred.Node, cred.Module, forgeStream, err)
return fmt.Errorf("the consumer %s_%s on %s: %w", cred.Node, cred.Module, forgeStream, err)
}
cc, err := cons.Consume(func(m jetstream.Msg) { handle(ctx, m, ix, logger) },
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
@@ -3,7 +3,7 @@ package main
import (
"context"
"errors"
"git.novox.be/novox/mesh-sdk/go/process"
"fmt"
"log"
"strings"
"sync"
@@ -55,14 +55,11 @@ func (w *previewWatch) ensure() error {
w.logger.Printf("the bucket %s does not exist yet: it is watched when the checker says it wrote a preview", delivery.PreviewsBucket)
return nil
case err != nil:
// Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the checker's next preview.
w.logger.Printf("the bucket %s cannot be read (%v): it is watched when the checker says it wrote a preview", delivery.PreviewsBucket, err)
return nil
return fmt.Errorf("the bucket %s: %w", delivery.PreviewsBucket, err)
}
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.ix.PutPreview(e.Key(), e.Value(), !opOf(e)) })
if err != nil {
process.NotWatched(w.logger.Printf, delivery.PreviewsBucket, err, "it is watched when the checker says it wrote a preview")
return nil
return err
}
w.watched = true
go func() {
@@ -28,11 +28,8 @@ import (
//
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
// key <node>.<module>) hold an open assignment of its build module on its node, and only once it was written since that
// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report
// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node
// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran
// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot
// key <node>.<module>) hold an open assignment of its build module on its node; it is passed on as deleted otherwise,
// and passed on again when the module is assigned there again. Until the assignments are settled (or where they cannot
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
// refusal is the guard of novox/hq ADR 0465, and it stays.
type toolchainGate struct {
@@ -41,14 +38,14 @@ type toolchainGate struct {
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
logf func(format string, a ...any)
known bool // the assignments are settled
standing map[string]time.Time // <node>.<module> with an open assignment -> since when it stands
standing map[string]bool // <node>.<module> with an open assignment
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
left map[string]string // <module>/<key> -> why it was said left out, so each reason is said once
left map[string]bool // <module>/<key> said as left out, so it is said once
}
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{},
raw: map[string]map[string][]byte{}, left: map[string]string{}}
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]bool{},
raw: map[string]map[string][]byte{}, left: map[string]bool{}}
for _, m := range modules {
g.modules[m] = true
}
@@ -70,31 +67,20 @@ func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool
return g.pass(module, key)
}
// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers
// whether what it passed on changed.
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool {
// Assignment takes one change of the controller's assignments in and answers whether what it passed on changed.
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool) bool {
node, module, ok := strings.Cut(key, ".")
if !ok || !g.modules[module] {
return false
}
assigned, open := openAssignment(value)
open = open && !deleted
open := !deleted && openAssignment(value)
g.mu.Lock()
defer g.mu.Unlock()
was, stood := g.standing[key]
since := assigned
if since.IsZero() {
// A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it.
since = at
if stood {
since = was
}
}
if open == stood && (!open || since.Equal(was)) {
if g.standing[key] == open {
return false
}
if open {
g.standing[key] = since
g.standing[key] = true
} else {
delete(g.standing, key)
}
@@ -125,58 +111,37 @@ func (g *toolchainGate) Settled() bool {
return changed
}
// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not
// known), deleted otherwise. The caller holds mu.
// pass hands one key on as it stands: its value when its node is assigned (or the assignments are not known), deleted
// otherwise. The caller holds mu.
func (g *toolchainGate) pass(module, key string) bool {
value, held := g.raw[module][key]
if !held {
return g.forward(module, key, nil, true)
}
node, _, err := delivery.ParseToolchainKey(key)
why := ""
if err == nil && g.known {
why = g.leftOut(module, node, value)
}
if why == "" {
if err != nil || !g.known || g.standing[node+"."+module] {
delete(g.left, module+"/"+key)
return g.forward(module, key, value, false)
}
if g.left[module+"/"+key] != why {
g.left[module+"/"+key] = why
g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why)
if !g.left[module+"/"+key] {
g.left[module+"/"+key] = true
g.logf("%s.%s is left out of the agreed toolchain: %s has no assignment on %s, so it is no builder (issue 1940)",
delivery.ToolchainsBucket(module), key, module, node)
}
return g.forward(module, key, nil, true)
}
// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module,
// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse.
// The caller holds mu.
func (g *toolchainGate) leftOut(module, node string, value []byte) string {
since, open := g.standing[node+"."+module]
if !open {
return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node)
}
tc, err := delivery.DecodeToolchain(value)
if err == nil && tc.At.Before(since) {
return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again",
tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node)
}
return ""
}
// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not
// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when
// it was assigned, zero where the value does not say.
func openAssignment(value []byte) (time.Time, bool) {
// openAssignment is whether a value of the controller's assignments holds an assignment that stands: one that has not
// ended and has no state (a pending one is not on the machine yet; mesh-controller internal/chain Assignment).
func openAssignment(value []byte) bool {
var a struct {
Assigned time.Time `json:"assigned"`
Ended time.Time `json:"ended"`
State string `json:"state"`
Ended time.Time `json:"ended"`
State string `json:"state"`
}
if json.Unmarshal(value, &a) != nil {
return time.Time{}, false
return false
}
return a.Assigned, a.Ended.IsZero() && a.State == ""
return a.Ended.IsZero() && a.State == ""
}
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
@@ -190,19 +155,17 @@ func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate,
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
return errs, nil
}
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e)) }
w, err := settle(ctx, kv, each)
if err != nil {
// Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read.
logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
return errs, nil
return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err)
}
if gate.Settled() {
changed(ctx)
}
go func() {
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) {
if gate.Assignment(e.Key(), e.Value(), !opOf(e)) {
changed(ctx)
}
})
@@ -3,7 +3,6 @@ package main
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
import (
"fmt"
"strings"
"testing"
"time"
@@ -45,10 +44,10 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
}
for _, node := range []string{"g14", "novox", "shanks"} {
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{})
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false)
}
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{})
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false)
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false) // another module's: no matter
if !gate.Settled() {
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
}
@@ -64,20 +63,20 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
t.Fatal("an unassigned node's report changed what was passed on")
}
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) {
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false) {
t.Fatal("assigning ace again passed nothing on")
}
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
t.Fatalf("builders that run different versions still disagree; got %q", none)
}
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) {
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false) {
t.Fatal("a pending assignment is no assignment on the machine")
}
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
t.Fatalf("got %q, %q", v, none)
}
gate.Assignment("ace.mesh-build-node", nil, true, time.Time{})
gate.Assignment("ace.mesh-build-node", nil, true)
if v, _ := agreedOf(set); v != "v26.12.1" {
t.Fatalf("got %q", v)
}
@@ -91,75 +90,23 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
}
}
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript
// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node
// writes it again. An assignment whose value names no time stands since the revision that opened it.
// A newly assigned build node that has not reported yet changes nothing; its report counts as it comes.
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again
report := func(t *testing.T, node, version string, at time.Time) (string, []byte) {
key, _ := gateReport(t, node, version)
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
if err != nil {
t.Fatal(err)
}
return key, value
set := &buildrun.ToolchainSet{}
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(string, ...any) {})
gate.Settled()
if gate.Assignment("g14.mesh-build-node", []byte(`{}`), false) {
t.Fatal("an assignment with no report passed something on")
}
for _, c := range []struct {
name string
assignment string // ace's assignment value
revision time.Time // when the bus wrote it
at time.Time // when ace wrote its report
counts bool
}{
{"no report yet, then one", `{}`, time.Time{}, time.Now(), true},
{"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false},
{"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false},
{"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true},
} {
t.Run(c.name, func(t *testing.T) {
set := &buildrun.ToolchainSet{}
var said []string
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
for _, node := range []string{"g14", "novox", "shanks"} {
gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{})
key, value := gateReport(t, node, "v26.12.1")
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
}
gate.Settled()
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) {
t.Fatal("an assignment with no report passed something on")
}
key, value := report(t, "ace", "v26.12.0", c.at)
if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts {
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
}
// A rewrite of the assignment that does not move it changes nothing, and nothing is said again.
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) {
t.Fatal("the same assignment written again changed what was passed on")
}
v, none := agreedOf(set)
if c.counts {
if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none)
}
return
}
if v != "v26.12.1" || none != "" {
t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none)
}
if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") {
t.Fatalf("said %q, want once why ace's report is left out", said)
}
// ace writes its report again since it was assigned: it counts.
key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute))
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
t.Fatal("ace's new report was not passed on")
}
if n := len(set.Entries("typescript")); n != 4 {
t.Fatalf("four builders report; got %d", n)
}
})
key, value := gateReport(t, "g14", "v26.12.1")
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
t.Fatal("an assigned node's report was not passed on")
}
key, value = gateReport(t, "ace", "v26.12.0")
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
t.Fatal("a report of a node with no assignment key was passed on after the assignments settled")
}
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
t.Fatalf("got %q, %q", v, none)
}
}
@@ -10,8 +10,6 @@ import (
"strings"
"sync"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
@@ -75,14 +73,11 @@ func (w *toolchainWatch) ensure(module string) error {
w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module)
return nil
case err != nil:
// Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the module's next report.
w.logger.Printf("the bucket %s cannot be read (%v): %s's toolchains are not heard until its next report", name, err, module)
return nil
return fmt.Errorf("the bucket %s: %w", name, err)
}
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.sink.put(module, e.Key(), e.Value(), !opOf(e)) })
if err != nil {
process.NotWatched(w.logger.Printf, name, err, fmt.Sprintf("%s's toolchains are not heard until its next report", module))
return nil
return err
}
w.watched[module] = true
go func() {
@@ -3,7 +3,6 @@ package main
import (
"bytes"
"context"
"errors"
"log"
"strings"
"sync"
@@ -22,7 +21,6 @@ type laterBuckets struct {
mu sync.Mutex
appears map[string]jetstream.KeyValue // buckets that exist now
asked map[string]int
refuse error // answered for a bucket that does not exist, when set: a bus that refuses it (issue 1945)
}
func (l *laterBuckets) KeyValue(_ context.Context, name string) (jetstream.KeyValue, error) {
@@ -32,9 +30,6 @@ func (l *laterBuckets) KeyValue(_ context.Context, name string) (jetstream.KeyVa
if kv, ok := l.appears[name]; ok {
return kv, nil
}
if l.refuse != nil {
return nil, l.refuse
}
return nil, jetstream.ErrBucketNotFound
}
@@ -105,12 +100,6 @@ func TestABuildModulesBucketThatAppearsLaterIsWatchedWhenItsEventArrives(t *test
recompute: func(context.Context) error { mu.Lock(); recomputes++; mu.Unlock(); return nil },
}
var logs bytes.Buffer
// 1945: a bus that refuses both buckets, or does not answer, leaves them unwatched, said, and the indexer running.
refusing := &laterBuckets{appears: map[string]jetstream.KeyValue{}, asked: map[string]int{}, refuse: errors.New("context deadline exceeded")}
if err := newToolchainWatch(ctx, refusing, sink, log.New(&logs, "", 0)).start(); err != nil || !strings.Contains(logs.String(), "cannot be read") {
t.Fatalf("a refused toolchains bucket: start %v, log %q; want it said and the start going on", err, logs.String())
}
logs.Reset()
w := newToolchainWatch(ctx, buckets, sink, log.New(&logs, "", 0))
if err := w.start(); err != nil { // neither bucket exists: nothing is watched, nothing is written, nobody waits
t.Fatal(err)
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232922-99407c37df95
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+6 -2
View File
@@ -1,7 +1,11 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024231716-571ac521f053 h1:8Kbp3aJG3jGndDOfmGdMSDgJVEcc1hyu921hVnnlnlQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024231716-571ac521f053/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232250-9b408db24a58 h1:4Damtdqlf6u6nJhvMA99peDy1jmjixtcwFXJ4bWEkLk=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232250-9b408db24a58/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232922-99407c37df95 h1:I8zbgGzRjmgYydPTWo8hmaZW/P/g+qGDUfu3E32/pkU=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232922-99407c37df95/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -71,7 +71,7 @@ func (ix *Indexer) logf(format string, args ...any) {
}
// Apply takes one message of the bridge's stream FORGE into the group of its branch. A pull message adds or moves the
// pull request, a closed one or a merged message takes it out; a group left with none is deleted. A change is written
// pull request, a closed one takes it out, a merged one marks it merged; a group left with none open is deleted. A change is written
// by compare-and-set on the revision read, and read again on a conflict. A lone pull request is a group of one. A
// merged message also writes the fingerprint of each module of its repository (modules.go).
func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error {
@@ -154,12 +154,13 @@ func (ix *Indexer) applyOpen(ctx context.Context, f delivery.Forge) error {
return nil, false, fmt.Errorf("candidates.%s holds what is not a candidate: %w", key, err)
}
kept := slices.DeleteFunc(slices.Clone(c.Pulls), func(p delivery.CandidatePull) bool {
return !named[delivery.OpenPull{Repository: p.Repository, Number: p.Number}]
// a member that merged is open no more, and stays while another member is (issue 1791)
return !p.Merged && !named[delivery.OpenPull{Repository: p.Repository, Number: p.Number}]
})
switch {
case len(kept) == len(c.Pulls):
return nil, false, nil
case len(kept) == 0:
case !anyOpen(kept):
ix.logf("%s: no pull request of the group is open, so no candidate", key)
return nil, true, nil
}
@@ -232,7 +233,16 @@ func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge, digests map[s
if at < 0 {
return cur
}
pulls = slices.Delete(pulls, at, at+1)
if f.Kind == delivery.ForgeMerged {
// A member that merges stays, marked merged, with the head it merged at: the heads hash the group's Approve
// covers holds, so the checker starts no new check for the members still open (issue 1791; issue 1921 S9).
pulls[at].Merged = true
} else {
pulls = slices.Delete(pulls, at, at+1)
}
if !anyOpen(pulls) {
return nil // every member merged or closed: the group is done
}
} else {
entry := delivery.CandidatePull{Head: delivery.Head{Repository: f.Repository, Number: f.Number, Head: f.Head}, Modules: []string{}}
switch {
@@ -276,6 +286,11 @@ func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge, digests map[s
return &delivery.Candidate{V: delivery.Version, Group: f.Branch, Base: base, HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)}
}
// anyOpen says whether a group's pull requests hold one that has not merged.
func anyOpen(pulls []delivery.CandidatePull) bool {
return slices.ContainsFunc(pulls, func(p delivery.CandidatePull) bool { return !p.Merged })
}
// NoneNoPreview is why a candidate's touched module has no fingerprint of the new form: the merge preview is not
// computed in this release (novox/hq ADR 0465 clause 3; the checker comes with issue 1445).
const NoneNoPreview = "no merge preview is computed"
@@ -206,8 +206,8 @@ func TestDirsNotKnownKeepTheModulesOfTheSameHead(t *testing.T) { // 1238/AC4
}
}
func TestClosedAndMergedTakeAPullOutAndTheLastDeletesTheKey(t *testing.T) { // 1238/AC5
for _, how := range []string{"closed", "merged"} {
func TestClosedTakesAPullOutAndTheLastDeletesTheKey(t *testing.T) { // 1238/AC5 (a merge: TestAMergedMemberStays...)
for _, how := range []string{"closed"} {
ix, st, _ := newIndexer(t)
ctx := context.Background()
_ = ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"))
@@ -387,3 +387,70 @@ func TestAnOpenMessageDropsEveryCandidateItDoesNotName(t *testing.T) { // 1279/A
t.Errorf("after an empty open message the candidates hold %v", st.vals)
}
}
// 1791 (issue 1921, try speed S9): a member of a group that merges while another is open stays in the candidate, marked
// merged, with its head: the heads hash the Approve covers holds, so no new check starts for the members left. An open
// message does not drop it, a close of the last open member deletes the group, and the last merge does too.
func TestAMergedMemberStaysInItsGroupUntilEveryMemberMerged(t *testing.T) {
merged := func(repo string, n int, head string) delivery.Forge {
return delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: repo, Branch: "feat/x", Number: n, Head: head, Merge: h3, Root: "r", Trees: map[string]string{"a": "t"}}
}
ix, st, _ := newIndexer(t)
ctx := context.Background()
for _, f := range []delivery.Forge{pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"), pull("novox/mesh-catalog", 300, "feat/x", h2, "modules/c"),
pull("novox/mesh-sdk", 7, "feat/x", h3)} {
if err := ix.Apply(ctx, f); err != nil {
t.Fatal(err)
}
}
before, _ := read(t, st, "feat/x")
if err := ix.Apply(ctx, merged("novox/mesh-sdk", 7, h3)); err != nil {
t.Fatal(err)
}
if err := ix.Apply(ctx, merged("novox/mesh-delivery", 12, h1)); err != nil {
t.Fatal(err)
}
c, ok := read(t, st, "feat/x")
if !ok || len(c.Pulls) != 3 || c.HeadsHash != before.HeadsHash {
t.Fatalf("after two merges the candidate = %+v (kept %v), want the three members and the same heads hash", c, ok)
}
for _, p := range c.Pulls {
if want := p.Repository != "novox/mesh-catalog"; p.Merged != want {
t.Errorf("%s#%d merged = %v, want %v", p.Repository, p.Number, p.Merged, want)
}
}
// The open message names only the member still open: the merged ones stay.
if err := ix.Apply(ctx, open(delivery.OpenPull{Repository: "novox/mesh-catalog", Number: 300})); err != nil {
t.Fatal(err)
}
if c, _ := read(t, st, "feat/x"); len(c.Pulls) != 3 || c.HeadsHash != before.HeadsHash {
t.Errorf("the open message dropped a merged member: %+v", c)
}
// The last open member merges: the group is done.
if err := ix.Apply(ctx, merged("novox/mesh-catalog", 300, h2)); err != nil {
t.Fatal(err)
}
if _, ok := read(t, st, "feat/x"); ok {
t.Error("every member merged and the group is still a candidate")
}
// A close of the last open member deletes the group too.
ix, st, _ = newIndexer(t)
_ = ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"))
_ = ix.Apply(ctx, pull("novox/mesh-catalog", 300, "feat/x", h2, "modules/c"))
_ = ix.Apply(ctx, merged("novox/mesh-delivery", 12, h1))
closed := pull("novox/mesh-catalog", 300, "feat/x", h2)
closed.Action = delivery.PullClosed
if err := ix.Apply(ctx, closed); err != nil {
t.Fatal(err)
}
if _, ok := read(t, st, "feat/x"); ok {
t.Error("the last open member closed and the group, holding only a merged one, is still a candidate")
}
// A lone pull request that merges leaves no group.
ix, st, _ = newIndexer(t)
_ = ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"))
_ = ix.Apply(ctx, merged("novox/mesh-delivery", 12, h1))
if _, ok := read(t, st, "feat/x"); ok {
t.Error("a lone merged pull request is still a candidate")
}
}
@@ -22,8 +22,7 @@ import (
// form (Module.FP, None) is no longer written: it was computed from those records (ADR 0453 clause 6). A module the
// bucket holds of another repository is left as it is and said: a move between repositories is the operator's (ADR 0420
// clause 7). Two directories naming one module write the one the bucket holds, or the first by name, and say the other.
// A value the module already holds but for its commit is not written again: its commit stays the one its fingerprint first
// appeared at (writeOverlay), so a merge writes only the modules it changes.
// A value the module already holds is not written again.
func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
digests, err := ix.keepManifests(ctx, f)
if err != nil {
@@ -85,51 +85,14 @@ func moduleOf(t *testing.T, st *memStore, key string) (delivery.Module, bool) {
const aInputs = `{"Module":"a","Tree":"oldtree","Bases":["registry.example/base@sha256:aaa"],"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}`
func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value writes nothing); the operator 2026-10-25 (a module's value moves only with the module)
const repo, next = "novox/mesh-delivery", "cccccccccccccccccccccccccccccccccccccccc"
three := map[string]string{"modules/a": "treeA", "modules/b": "treeB", "modules/c": "treeC"}
for _, tc := range []struct {
name string
then func(ix *Indexer) error // after the first merge of a, b and c at merge, stamped by source revision 7
// writes is the values written by then; commit is each module's commit after.
writes int
commit map[string]string
}{
{name: "the same merge again", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge},
then: func(ix *Indexer) error { return ix.Apply(context.Background(), mergedMsg(repo, three)) }},
{name: "a merge touching one of three modules", writes: 1, commit: map[string]string{"a": next, "b": merge, "c": merge},
then: func(ix *Indexer) error {
m := mergedMsg(repo, map[string]string{"modules/a": "treeA2", "modules/b": "treeB", "modules/c": "treeC"})
m.Merge = next
return ix.Apply(context.Background(), m)
}},
{name: "a later source revision of another repository's merge", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge},
then: func(ix *Indexer) error {
return ix.ApplySource(context.Background(), 9, delivery.Source{Group: "g", Heads: map[string]string{repo: merge, "novox/mesh-catalog": otherMerge}})
}},
} {
t.Run(tc.name, func(t *testing.T) {
ix, mods := modulesIndexer(t)
ctx := context.Background()
if err := ix.Apply(ctx, mergedMsg(repo, three)); err != nil {
t.Fatal(err)
}
if err := ix.ApplySource(ctx, 7, sourceOf(repo, merge)); err != nil {
t.Fatal(err)
}
writes := mods.writes
if err := tc.then(ix); err != nil {
t.Fatal(err)
}
if got := mods.writes - writes; got != tc.writes {
t.Errorf("%d values written, want %d", got, tc.writes)
}
for module, want := range tc.commit {
if m, ok := moduleOf(t, mods, module); !ok || m.Commit != want {
t.Errorf("%s = %+v (%v), want commit %.8s", module, m, ok, want)
}
}
})
func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value writes nothing)
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs))
m := mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})
_ = ix.Apply(context.Background(), m)
rev, writes := mods.revs["a"], mods.writes
_ = ix.Apply(context.Background(), m)
if mods.revs["a"] != rev || mods.writes != writes {
t.Errorf("the same merge wrote again (%d writes, was %d)", mods.writes, writes)
}
}
@@ -112,12 +112,12 @@ func (ix *Indexer) RecomputeCandidates(ctx context.Context) error {
return nil
}
// previewOf is the preview of a head merged onto the trunk head the source heads name: found, or stale when only a preview
// previewOf is the preview of a head merged onto the trunk head the source names now: found, or stale when only a preview
// onto another trunk head exists, or neither (found false, stale false) when the source is not known or none exists.
func (ix *Indexer) previewOf(heads map[string]string, repository, head string) (pv delivery.Preview, found, stale bool) {
func (ix *Indexer) previewOf(repository, head string) (pv delivery.Preview, found, stale bool) {
ix.mu.Lock()
defer ix.mu.Unlock()
trunk := heads[repository]
trunk := ix.source.heads[repository]
if trunk == "" {
return delivery.Preview{}, false, false
}
@@ -137,23 +137,14 @@ func (ix *Indexer) previewOf(heads map[string]string, repository, head string) (
// withPreviews gives each touched module of the candidate's pull requests the fingerprint its preview gives it, or the
// reason it has none, and the candidate its fpset. It answers whether the candidate changed. A module of a pull request with
// no preview keeps what it had.
//
// The candidate's Base is the source revision whose trunk heads the previews were read onto (novox/hq ADR 0437 §5): written
// whenever the fingerprints are, so the builder merges a preview onto the trunk head its fingerprint was computed on (the head
// of the pull request's repository in that revision), never onto a module value's commit, which stays at the commit its
// fingerprint first appeared at.
func (ix *Indexer) withPreviews(ctx context.Context, c *delivery.Candidate) bool {
ix.mu.Lock()
rev, heads := ix.source.rev, ix.source.heads
ix.mu.Unlock()
changed, read := false, false
changed := false
for i := range c.Pulls {
p := &c.Pulls[i]
pv, found, stale := ix.previewOf(heads, p.Repository, p.Head.Head)
pv, found, stale := ix.previewOf(p.Repository, p.Head.Head)
if !found && !stale {
continue
}
read = true
// A touched module's directory: the one the modules bucket holds of the repository, else the preview's directory
// whose manifest is the one the pull request gave the module (a module the pull request adds; ADR 0501 clause 6).
dirs := map[string]string{}
@@ -207,9 +198,6 @@ func (ix *Indexer) withPreviews(ctx context.Context, c *delivery.Candidate) bool
if changed {
c.FPSet = delivery.FingerprintSet(c.Pulls)
}
if read && rev != 0 && c.Base != rev && (changed || c.Base == 0) {
c.Base, changed = rev, true
}
return changed
}
@@ -2,8 +2,6 @@ package index
import (
"context"
"encoding/json"
"os"
"strings"
"testing"
@@ -89,16 +87,9 @@ func TestACandidatesModuleGetsTheFingerprintATrunkModuleOfTheSameTreeHas(t *test
if got.FP2 != trunk.FP2 || got.None2 != "" || got.Manifest != delivery.ManifestDigest([]byte(avahiOnTools(t))) {
t.Fatalf("candidate avahi = %+v, want the trunk's fingerprint %s", got, trunk.FP2)
}
c := candidateOf(t, cands)
if c.FPSet != delivery.FingerprintSet(c.Pulls) || c.FPSet == "" {
if c := candidateOf(t, cands); c.FPSet != delivery.FingerprintSet(c.Pulls) || c.FPSet == "" {
t.Errorf("fpset = %q", c.FPSet)
}
// The candidate names the source revision the preview was read onto (5, whose catalogue head is the preview's trunk):
// the builder merges the preview onto that head, never onto the module value's commit.
if c.Base != 5 {
t.Errorf("candidate base = %d, want the source revision 5 its preview was read onto", c.Base)
}
written(t, writtenCandidate{Source: sourceRev{Rev: 5, Heads: map[string]string{catalogueRepo: trunkNow}}, Candidate: c})
// Another tree gives another fingerprint.
pv := avahiPreview(t, trunkNow, h1)
m := pv.Modules["modules/avahi"]
@@ -200,38 +191,3 @@ func TestAPreviewOfConflictsGivesTheCandidatesModuleTheReasonItDoesNotMerge(t *t
t.Errorf("candidate avahi = %+v, want it not merging and naming the files", got)
}
}
// writtenCandidate is what the builder's tests read of the indexer (mesh-delivery-builder internal/build): a candidate as the
// indexer writes it and the gate's source revision it names, kept in testdata/candidate-written.json so the builder is tested
// on what the indexer writes, never on a candidate made by hand. MESH_WRITE_TESTDATA=1 writes it again.
type writtenCandidate struct {
Source sourceRev `json:"source"`
Candidate delivery.Candidate `json:"candidate"`
}
type sourceRev struct {
Rev uint64 `json:"rev"`
Heads map[string]string `json:"heads"`
}
func written(t *testing.T, w writtenCandidate) {
t.Helper()
data, err := json.MarshalIndent(w, "", " ")
if err != nil {
t.Fatal(err)
}
data = append(data, '\n')
const file = "testdata/candidate-written.json"
if os.Getenv("MESH_WRITE_TESTDATA") == "1" {
if err := os.WriteFile(file, data, 0o644); err != nil {
t.Fatal(err)
}
}
kept, err := os.ReadFile(file)
if err != nil {
t.Fatal(err)
}
if string(kept) != string(data) {
t.Errorf("%s is not what the indexer writes now (MESH_WRITE_TESTDATA=1 writes it again):\n%s", file, data)
}
}
@@ -32,8 +32,8 @@ func (ix *Indexer) srcOf(repository, merge string) uint64 {
}
// ApplySource takes a revision of the gate's source.main in. An older revision than the latest seen is left. For each
// repository the source names, each module built from it whose fingerprint was computed at the head the source names, and
// that carries no Src yet, is written again with Src the revision (once: a later revision leaves it); a module computed at another commit is left (a merge heard later is stamped
// repository the source names, each module built from it whose fingerprint was computed at the head the source names is
// written again with Src the revision; a module computed at another commit is left (a merge heard later is stamped
// when it is written, applyMerged), and a repository none of the indexer's modules is built from writes nothing. The
// modules of a repository are those the modules bucket holds of it (ADR 0501 clause 6).
func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s delivery.Source) error {
@@ -71,9 +71,7 @@ func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s delivery.Sourc
ix.logf("modules.%s holds what is not a module, left: %v", key, err)
return nil, false, nil
}
// Stamped once, with the first revision naming the commit as the head: a later revision (a merge of another
// repository) leaves the value as it is, so no revision of the source rewrites a module that did not change.
if cur.Repository != repository || cur.Commit != head || cur.Src != 0 {
if cur.Repository != repository || cur.Commit != head || cur.Src == rev {
return nil, false, nil
}
cur.Src = rev
@@ -38,15 +38,15 @@ func TestASourceRevisionStampsTheModulesOfTheRepositoriesHeadItNames(t *testing.
if c, ok := moduleOf(t, mods, "c"); ok {
t.Errorf("module c = %+v: written for a head no merge message of the indexer carried", c)
}
// The same revision again changes nothing, and so does a later one: Src is the first revision naming the commit.
// The same revision again changes nothing; a later revision moves Src.
if err := ix.ApplySource(ctx, 7, s); err != nil {
t.Fatal(err)
}
if err := ix.ApplySource(ctx, 9, s); err != nil {
t.Fatal(err)
}
if m, _ := moduleOf(t, mods, "a"); m.Src != 7 {
t.Errorf("module a Src = %d, want 7", m.Src)
if m, _ := moduleOf(t, mods, "a"); m.Src != 9 {
t.Errorf("module a Src = %d, want 9", m.Src)
}
}
@@ -1,31 +0,0 @@
{
"source": {
"rev": 5,
"heads": {
"novox/mesh-catalog": "5555555555555555555555555555555555555555"
}
},
"candidate": {
"v": 1,
"group": "feat/x",
"base": 5,
"heads": "dfe9fb8f79e8",
"pulls": [
{
"repository": "novox/mesh-catalog",
"number": 301,
"head": "0123456789abcdef0123456789abcdef01234567",
"modules": [
"avahi"
],
"fps": {
"avahi": {
"fp2": "src2:e8d595aea1b13f1f5578ef1f245dab0d28a351c28079ecdcd2276c63a932d7be",
"manifest": "8a3856b6c0566d76f2556fc4f46c5251dc7935517bb6bd15809ded7e5546a0d8"
}
}
}
],
"fpset": "01c840ec276b"
}
}
@@ -112,9 +112,6 @@ type known struct {
languages map[string]string
bases []string
contexts bool
// held is the fingerprint the bucket holds for the module and the toolchains it was computed with (for a merge, the
// value before it): what stillComputed keeps while the builders agree on no toolchain.
held delivery.Module
}
// Recompute writes again every module whose new-form fingerprint, or reason for none, is no longer what it holds: from
@@ -162,23 +159,19 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
stale := map[string]bool{} // values an SDK that could not read their manifest wrote, judged again (issue 1923)
all := map[string]*known{}
for _, key := range keys {
cur, merged := overlay[key]
var before delivery.Module
value, _, found, err := ix.Modules.Get(ctx, key)
if err != nil {
return fmt.Errorf("modules.%s: %w", key, err)
}
if found {
if before, err = delivery.DecodeModule(value); err != nil && !merged {
ix.logf("modules.%s holds what is not a module, left: %v", key, err)
continue
cur, ok := overlay[key]
if !ok {
value, _, found, err := ix.Modules.Get(ctx, key)
if err != nil {
return fmt.Errorf("modules.%s: %w", key, err)
}
}
if !merged {
if !found {
continue
}
cur = before
if cur, err = delivery.DecodeModule(value); err != nil {
ix.logf("modules.%s holds what is not a module, left: %v", key, err)
continue
}
}
held[key] = cur
if cur.Manifest == "" || cur.Inputs2 == nil {
@@ -193,7 +186,7 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
cur, stale[key] = again, true
held[key] = cur
}
all[key] = &known{key: key, cur: cur, languages: languagesOf(m), bases: basesOf(m), contexts: hasContext(m), held: before}
all[key] = &known{key: key, cur: cur, languages: languagesOf(m), bases: basesOf(m), contexts: hasContext(m)}
}
tools := ix.agreed()
type result struct {
@@ -258,9 +251,6 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
r.none = strings.Join(missing, "; ")
case noToolchain != "":
r.none = noToolchain
if fp, ok := stillComputed(module, kept, k, toolchains); ok {
r = result{fp: fp, tcs: k.held.Toolchains}
}
default:
r.fp, r.none = delivery.SourceFingerprint2(kept.With(module, toolchains))
if r.fp != "" {
@@ -319,12 +309,6 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
// writeOverlay writes the records of a merge, each once and only if it differs from what the bucket holds; finals has
// the ones that were computed, the others are written as the merge gave them.
//
// **A module the merge did not change is not written** (the operator, 2026-10-25: "only if a module has got a new version in
// commit x, should a new module version get created linked to commit x"). A value names the commit its fingerprint first
// appeared at: when the merge gives a module the value it holds but for the commit (and the source revision stamped on it),
// the held value stands, its commit unchanged, so a catalogue merge touching one module writes one key, not one per module of
// the repository, and the controller's take-in hears only that one. The repository's trunk head is the gate's source.main.
func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string]delivery.Module) error {
for _, key := range slices.Sorted(maps.Keys(overlay)) {
next := overlay[key]
@@ -333,7 +317,7 @@ func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string]
}
err := write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
if found {
if cur, err := delivery.DecodeModule(value); err == nil && sameButWhere(cur, next) {
if cur, err := delivery.DecodeModule(value); err == nil && reflect.DeepEqual(cur, withV(next)) {
return nil, false, nil
}
}
@@ -347,13 +331,6 @@ func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string]
return nil
}
// sameButWhere says the held value cur is next but for where it was computed (Commit and Src): the module is unchanged.
func sameButWhere(cur, next delivery.Module) bool {
next = withV(next)
next.Commit, next.Src = cur.Commit, cur.Src
return reflect.DeepEqual(cur, next)
}
// manifestRaw is manifests.<digest>: the manifest as stored and parsed by the running SDK.
func (ix *Indexer) manifestRaw(ctx context.Context, digest string) (string, catalogue.Manifest, error) {
key, err := delivery.ManifestsKey(digest)
@@ -404,32 +381,6 @@ func hasContext(m catalogue.Manifest) bool {
return false
}
// stillComputed is the fingerprint a module already holds, kept while a language it is built with has no agreed toolchain
// (issue 1940, 2026-10-25 20:56: a build node assigned again brought a stale report into the agreement, every module's
// fingerprint was blanked to "builders disagree", and the controller's take-in refused modules whose fingerprints were
// computed and built). Only computing a new fingerprint needs the agreement: the held one is kept when its kept inputs,
// with the toolchains it was computed with in place of the languages agreed on none, give it again. Any other input that
// moved (a merge, a base, an agreed toolchain) gives another fingerprint, and the module has none until they agree.
func stillComputed(module string, kept delivery.KeptInputs2, k *known, agreed map[string]delivery.Toolchain2) (string, bool) {
if k.held.FP2 == "" {
return "", false
}
toolchains := map[string]delivery.Toolchain2{}
for artifact, language := range k.languages {
if tc, ok := agreed[artifact]; ok {
toolchains[artifact] = tc
continue
}
used, ok := k.held.Toolchains[language]
if !ok {
return "", false
}
toolchains[artifact] = delivery.Toolchain2{Language: language, Version: used.Version, Recipe: used.Recipe}
}
fp, none := delivery.SourceFingerprint2(kept.With(module, toolchains))
return fp, none == "" && fp == k.held.FP2
}
// toolchainsUsed are the toolchains a fingerprint was computed with, by language (an artifact's toolchain is its language's).
func toolchainsUsed(by map[string]delivery.Toolchain2) map[string]delivery.ToolchainUsed {
if len(by) == 0 {
@@ -123,16 +123,9 @@ func TestAgreedToolchainChangesAGoModulesReasonWithNoMerge(t *testing.T) { // 14
}
func TestNodesReportingDifferentVersionsLeaveTheReasonNamingBoth(t *testing.T) { // 1444/AC6
// Both reports are read before the module's first fingerprint is computed (a fingerprint computed before they
// disagree is kept: TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith).
ix, mods := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")})
for node, version := range map[string]string{"novox": "go1.27.1", "ace": "go1.26.0"} {
key, _ := delivery.ToolchainKey(node, "go")
ix.PutToolchain("mesh-build-go", key, toolchainValue(t, "go", version, "1"), false)
}
if err := ix.Recompute(context.Background()); err != nil {
t.Fatal(err)
}
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.26.0")
m, _ := moduleOf(t, mods, "avahi")
if !strings.Contains(m.None2, "builders disagree") || !strings.Contains(m.None2, "go1.27.1") || !strings.Contains(m.None2, "go1.26.0") {
t.Errorf("none2 = %q, want 'builders disagree' naming both versions", m.None2)
@@ -267,13 +260,11 @@ func TestAMergeKeepsAFingerprintTheModuleStillHas(t *testing.T) { // 1444 (revie
w := mods.writes
// the same catalogue merged again at a later commit: nothing moved
_, _, f := newFormIndexerUnmerged(t, builderModules(t)...)
f.Merge = otherMerge
if err := ix.Apply(ctx, f); err != nil {
t.Fatal(err)
}
// and the commit stays the one the fingerprint first appeared at (the operator, 2026-10-25)
if after, _ := moduleOf(t, mods, "mesh-tools"); after.FP2 != before.FP2 || after.Commit != before.Commit || mods.writes != w {
t.Errorf("fp2 = %q at %.8s (was %q at %.8s), writes %d more", after.FP2, after.Commit, before.FP2, before.Commit, mods.writes-w)
if after, _ := moduleOf(t, mods, "mesh-tools"); after.FP2 != before.FP2 || mods.writes != w {
t.Errorf("fp2 = %q (was %q), writes %d more", after.FP2, before.FP2, mods.writes-w)
}
}
@@ -297,24 +288,9 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
if n.FP2 == m.FP2 || n.Toolchains["go"].Version != "go1.27.2" {
t.Errorf("after the upgrade: fp2 %q (was %q) toolchains %+v, want the new version with the new fingerprint", n.FP2, m.FP2, n.Toolchains)
}
// Nodes that disagree keep the fingerprint already computed, with its toolchains (issue 1940, 2026-10-25 20:56: a
// stale report blanked every module and the controller's take-in refused modules already built); only a new
// fingerprint needs the agreement: a merge that moves the module's tree gives it none, naming the disagreement.
// Nodes that disagree leave no fingerprint and no toolchains.
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.26.0")
if d, _ := moduleOf(t, mods, "avahi"); d.FP2 != n.FP2 || !reflect.DeepEqual(d.Toolchains, n.Toolchains) || d.None2 != "" {
t.Errorf("disagreeing nodes: fp2 %q toolchains %+v none2 %q, want the fingerprint kept (%q) with %+v", d.FP2, d.Toolchains, d.None2, n.FP2, n.Toolchains)
}
_, _, f := newFormIndexerUnmerged(t, builderModules(t)...)
f.Trees["modules/avahi"] = "tree-avahi-moved"
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
if d, _ := moduleOf(t, mods, "avahi"); d.FP2 != "" || d.Toolchains != nil || !strings.HasPrefix(d.None2, "builders disagree") {
t.Errorf("a new tree while nodes disagree: fp2 %q toolchains %+v none2 %q, want no fingerprint and the disagreement", d.FP2, d.Toolchains, d.None2)
}
// They agree again: the new tree gets its fingerprint.
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.27.2")
if a, _ := moduleOf(t, mods, "avahi"); a.FP2 == "" || a.FP2 == n.FP2 {
t.Errorf("agreed again: fp2 %q (was %q), want a new fingerprint", a.FP2, n.FP2)
if d, _ := moduleOf(t, mods, "avahi"); d.FP2 != "" || d.Toolchains != nil {
t.Errorf("disagreeing nodes: fp2 %q toolchains %+v, want neither", d.FP2, d.Toolchains)
}
}
@@ -1,213 +0,0 @@
package main
import (
"encoding/json"
"fmt"
"slices"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The assigner's asks reach the operator (issue 1906). The assigner writes its asks (the hold of a fingerprint, the restore
// of a data module, the backup before an apply and a move of a module of policy record) in its bucket asks; until now the
// teller only filled them from an answer given in the view, so an ask nobody opened the view for reached nobody: on
// 2026-10-25 seven modules stayed held on every machine and the operator was never asked. Now the teller raises each on the
// operator channel as it raises the gate's merge asks: as the asker mesh-delivery-teller, once per router id, through the same
// records, the same wait for room and the same warrants. The operator's answer, from the phone or from the view, is written
// under the ask's own key in the bucket answers, which the assigner reads (mesh-delivery-assigner pass.go answeredWith); none
// of these asks needs proof of who answered (mesh-sdk go/delivery AskNeedsProof).
// mostAssignerOpen is how many of the assigner's asks the teller keeps open at the router at once. The router keeps at most
// asks.MostOpen open asks of one asker, and the teller asks both the gate's merge asks and the assigner's as one asker, so a burst
// of holds (seven at once on 2026-10-25) would take all the room and a merge Approve could not be asked; one place is always
// left for a merge ask. An assigner ask beyond these waits in the wait for room and is asked when one of them ends.
const mostAssignerOpen = asks.MostOpen - 1
// assignerOpenRecord is the key of the router ids of the assigner's asks the teller asked.
const assignerOpenRecord = "assigner.open"
// assignerOpen is the router ids of the assigner's asks open now: those asked whose ask is neither answered nor ended. An id
// that ended is dropped from the record as it is read.
func (t *Teller) assignerOpen() ([]string, *stdio.StateEntry) {
e, err := t.Store.Get(assignerOpenRecord)
if err != nil || e == nil {
return nil, e
}
var ids []string
if err := json.Unmarshal(e.Value, &ids); err != nil {
t.logf("the record %s cannot be read: %v", assignerOpenRecord, err)
return nil, e
}
return slices.DeleteFunc(ids, func(id string) bool {
k, _, err := t.keptOf(id)
return err == nil && !open(k)
}), e
}
// assignerRoom says whether another of the assigner's asks may be asked now.
func (t *Teller) assignerRoom() bool {
ids, _ := t.assignerOpen()
return len(ids) < mostAssignerOpen
}
// assignerAsked keeps that the assigner's ask with this router id is asked.
func (t *Teller) assignerAsked(id string) {
ids, e := t.assignerOpen()
if !slices.Contains(ids, id) {
ids = append(ids, id)
}
var err error
if e == nil {
_, err = t.Store.Create(assignerOpenRecord, ids)
} else {
_, err = t.Store.Update(assignerOpenRecord, ids, e.Revision)
}
if err != nil {
t.logf("the record %s could not be kept: %v", assignerOpenRecord, err)
}
}
// waitForAssignerRoom puts an assigner ask in the wait for room, once.
func (t *Teller) waitForAssignerRoom(key, id string) {
ws, rec, err := t.waiting()
if err != nil {
t.logf("%v", err)
return
}
if slices.ContainsFunc(ws, func(x waitingAsk) bool { return x.ID == id }) {
return
}
if t.keepWaiting(append(ws, waitingAsk{Key: key, ID: id}), rec) == nil {
t.logf("%s: the assigner's ask %s waits: %d of the assigner's asks are open, the most the teller keeps open so a merge ask always has room", key, id, mostAssignerOpen)
}
}
// isAssignerKey says whether an ask key is one of the assigner's, not the gate's merge ask.
func isAssignerKey(key string) bool {
k, err := delivery.ParseAnyAskKey(key)
return err == nil && k.Kind != delivery.AskKindMerge
}
// assignerAsk reads the assigner's ask at key: found false when the key holds none.
func (t *Teller) assignerAsk(key string) (delivery.AssignerAsk, bool, error) {
if t.Assigner == nil {
return delivery.AssignerAsk{}, false, fmt.Errorf("the teller reads no assigner asks")
}
raw, found, err := t.Assigner.Get(key)
if err != nil || !found {
return delivery.AssignerAsk{}, false, err
}
a, err := delivery.DecodeAssignerAsk(key, raw)
if err != nil {
return delivery.AssignerAsk{}, false, fmt.Errorf("the assigner's key %s holds no ask: %w", key, err)
}
return a, true, nil
}
// raiseAssignerAsk asks the operator the assigner's ask at key, once per router id: an ask answered already (from the view or
// the phone) is not asked; an earlier asking still open is taken back first; an asking already made is only caught up with
// at the router, and one that ended is not asked again, for the router drops a second ask under an id it keeps.
func (t *Teller) raiseAssignerAsk(key string) {
a, found, err := t.assignerAsk(key)
if err != nil {
t.logf("%s: the ask is not raised: %v", key, err)
return
}
if !found {
return
}
id := a.Ask.ID
if raw, _, has, err := t.Answers.Get(delivery.AnswerKey(key)); err != nil {
t.logf("%s: the answers could not be read, so the ask is not raised: %v", key, err)
return
} else if has && answeredAsking(raw) == id {
return
}
prior, grec, err := t.seenAt(gateRecordKey(key))
if err != nil {
t.logf("%s: %v", key, err)
return
}
if grec != nil && prior.ID != id {
t.cancelOpen(key, prior.ID) // a new asking of the same key: the earlier one is taken back
}
k, _, err := t.keptOf(id)
if err != nil {
t.logf("%s: %v", key, err)
return
}
if k != nil {
if grec == nil || prior.ID != id {
_ = t.note(id, key, 0, grec)
}
if open(k) {
t.catchUp(id)
}
return
}
if err := t.note(id, key, 0, grec); err != nil {
t.logf("%s: the record of the ask %s could not be kept, so it is not raised: %v", key, id, err)
return
}
if !t.assignerRoom() {
t.waitForAssignerRoom(key, id)
return
}
if err := t.Asker.Ask(a.Ask); err != nil {
t.logf("%s: the ask %s was not asked: %v", key, id, err)
if kept, _, _ := t.keptOf(id); open(kept) {
_ = t.Store.Delete(keptKey(id)) // kept but not published: forgotten, so the next writing of the key asks it
}
return
}
t.assignerAsked(id)
t.logf("%s: the assigner's ask %s is asked of the operator (%s)", key, id, a.Ask.Headline)
}
// OnAssignerAskGone handles a key the assigner took away from its asks (a hold it lifted by itself): the open router ask of
// its latest asking is taken back, so the operator is not asked about what no longer holds.
func (t *Teller) OnAssignerAskGone(key string) {
t.mu.Lock()
defer t.mu.Unlock()
if !isAssignerKey(key) {
t.logf("the assigner's key %s was taken away and is no ask key of the assigner, and is left", key)
return
}
prior, grec, err := t.seenAt(gateRecordKey(key))
if err != nil || grec == nil {
if err != nil {
t.logf("%s: %v", key, err)
}
return
}
t.cancelOpen(key, prior.ID)
}
// answerAssigner is the act of a warrant for one of the assigner's asks: the ask must still stand at its key under the asking
// and the words the operator answered, and then the answer is written under the ask's key, once. Its options bind no act:
// the assigner reads the option chosen (asker.Client.Take has checked the option, its level and the ask's digest).
func (t *Teller) answerAssigner(key string, o asks.Option, w asks.Warrant) error {
a, found, err := t.assignerAsk(key)
if err != nil {
return err
}
if !found {
t.logf("%s: the ask %s was answered and the assigner has taken it away: the answer changes nothing", key, w.Ask)
return nil
}
if a.Ask.ID != w.Ask {
t.logf("%s: the ask %s was answered, and the assigner has asked again under %s: an answer to an earlier asking changes nothing", key, w.Ask, a.Ask.ID)
return nil
}
if a.Ask.Digest() != w.AskDigest {
return fmt.Errorf("the assigner's ask %s is no longer the ask the operator answered (its digest is %s, the warrant's %s)", key, a.Ask.Digest(), w.AskDigest)
}
body, err := delivery.EncodeAnswer(delivery.Answer{Option: o.ID, Warrant: warrantID(w), Words: w.Says()})
if err != nil {
return err
}
return t.writeAnswer(key, w.Ask, body, w.Says())
}
@@ -1,209 +0,0 @@
package main
import (
"encoding/json"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// 1906/H1: given the assigner's hold ask in its bucket and no answer, when the teller hears the key, then it raises the ask on
// the operator channel as itself, unchanged and once (the key heard again, as a watch replays it, asks nothing more).
func TestAnAssignerAskIsRaisedOnTheOperatorChannelOnce(t *testing.T) {
b := newBed(t)
assigner, _ := withView(b)
key, value, ask := holdAsk(t)
assigner[key] = value
b.t.OnAssignerAsk(key)
b.t.OnAssignerAsk(key)
asked := b.askedOf(asks.AskSubject(delivery.TellerModule))
if len(asked) != 1 {
t.Fatalf("published %d asks, want one; log:\n%s", len(asked), b.logs.String())
}
var got asks.Ask
if err := json.Unmarshal(asked[0].body, &got); err != nil {
t.Fatal(err)
}
if got.ID != ask.ID || got.Digest() != ask.Digest() {
t.Errorf("the router's ask %+v differs from the assigner's", got)
}
}
// 1906/H1: without the view's bucket the assigner's asks are still raised.
func TestAnAssignerAskIsRaisedWithoutTheView(t *testing.T) {
b := newBed(t)
assigner := memGate{}
b.t.Assigner = assigner
key, value, _ := holdAsk(t)
assigner[key] = value
b.t.OnAssignerAsk(key)
if n := len(b.askedOf(asks.AskSubject(delivery.TellerModule))); n != 1 {
t.Fatalf("published %d asks, want one; log:\n%s", n, b.logs.String())
}
}
// 1906/H1: the operator's answer on the phone comes back by the same path as the view's: written under the ask's own key in
// the bucket answers, naming the asking, once; a second warrant writes nothing more and nothing is said as a fault.
func TestTheOperatorsAnswerToAnAssignerAskIsWrittenUnderItsKey(t *testing.T) {
b := newBed(t)
assigner, _ := withView(b)
key, value, ask := holdAsk(t)
assigner[key] = value
b.t.OnAssignerAsk(key)
w := ackFor(ask, "release")
b.deliver(t, w)
raw, ok := b.answers.written[delivery.AnswerKey(key)]
if !ok {
t.Fatalf("no answer under %s; log:\n%s", key, b.logs.String())
}
if a, err := delivery.DecodeAnswer(raw); err != nil || a.Option != "release" || answeredAsking(raw) != ask.ID {
t.Errorf("answer = %+v, %v", a, err)
}
b.deliver(t, w)
if b.answers.writes != 1 || len(b.conds) != 0 {
t.Errorf("%d writes, conditions %+v", b.answers.writes, b.conds)
}
}
// 1906/H1: an answer to an asking the assigner has since taken away or asked again changes nothing.
func TestAnAnswerToAnAssignerAskTakenAwayWritesNothing(t *testing.T) {
b := newBed(t)
assigner, _ := withView(b)
key, value, ask := holdAsk(t)
assigner[key] = value
b.t.OnAssignerAsk(key)
delete(assigner, key)
b.deliver(t, ackFor(ask, "release"))
if b.answers.writes != 0 {
t.Errorf("an answer to a withdrawn ask was written")
}
}
// 1906/H1, H2: an ask answered in the view is taken back from the phone, and an ask the assigner took away (a hold it lifted by
// itself) is taken back too; an ask answered before it is heard is never raised.
func TestAnAssignerAskAnsweredInTheViewOrTakenAwayIsTakenBack(t *testing.T) {
cancelled := func(b *bed, id string) bool {
for _, p := range b.askedOf(asks.CancelSubject(delivery.TellerModule)) {
if string(p.body) == `{"id":"`+id+`"}` {
return true
}
}
return false
}
b := newBed(t)
assigner, view := withView(b)
key, value, ask := holdAsk(t)
assigner[key] = value
b.t.OnAssignerAsk(key)
view[key+".view"] = viewAnswer(t, "release", "")
b.t.OnViewAnswer(key + ".view")
if !cancelled(b, ask.ID) {
t.Errorf("the ask answered in the view was not taken back; log:\n%s", b.logs.String())
}
b = newBed(t)
assigner, _ = withView(b)
assigner[key] = value
b.t.OnAssignerAsk(key)
delete(assigner, key)
b.t.OnAssignerAskGone(key)
if !cancelled(b, ask.ID) {
t.Errorf("the ask taken away was not taken back; log:\n%s", b.logs.String())
}
b = newBed(t)
assigner, view = withView(b)
assigner[key] = value
view[key+".view"] = viewAnswer(t, "keep", "")
b.t.OnAssignerAsk(key)
if n := len(b.askedOf(asks.AskSubject(delivery.TellerModule))); n != 0 {
t.Errorf("an ask answered in the view was raised %d times", n)
}
}
// 1906/H1: an assigner ask refused for want of room waits and is asked again when another of the teller's asks ends, as a merge
// ask does; a refusal for its words is said as a condition about a delivery ask, not a merge.
func TestAnAssignerAskWaitsForRoomAndARefusalIsSaidAsADeliveryAsk(t *testing.T) {
b := newBed(t)
assigner, _ := withView(b)
_, first := b.askOf(t, 1, 1)
b.askOf(t, 2, 2)
b.askOf(t, 3, 3)
key, value, ask := holdAsk(t)
assigner[key] = value
b.t.OnAssignerAsk(key)
b.refuse(t, ask.ID, roomRefusal)
b.deliver(t, warrantFor(first.Ask, "approve"))
got := b.askedIDs()
if len(got) != 5 || got[4] != ask.ID {
t.Fatalf("asks = %v, want the assigner's %s asked again", got, ask.ID)
}
b = newBed(t)
assigner, _ = withView(b)
assigner[key] = value
b.t.OnAssignerAsk(key)
b.refuse(t, ask.ID, "the ask has no headline")
if len(b.conds) != 1 || b.conds[0].Subject.Scope != "delivery" || strings.Contains(b.conds[0].Explanation, "merge") {
t.Errorf("conditions = %+v", b.conds)
}
}
// ackFor is the router's warrant for an option of level acknowledge, as the assigner's asks offer.
func ackFor(a asks.Ask, option string) asks.Warrant {
w := warrantFor(a, option)
w.Level, w.Label = asks.Acknowledge, option
return w
}
// holdAskOf is the assigner's hold ask of module plex at the n-th fingerprint, shaped as holdAsk.
func holdAskOf(t *testing.T, n int) (string, []byte, asks.Ask) {
t.Helper()
key, err := delivery.HoldAskKey("plex", "src2:"+strings.Repeat(string(rune('0'+n)), 64))
if err != nil {
t.Fatal(err)
}
_, _, ask := holdAsk(t)
ask.ID = delivery.AskID(key)
value, err := delivery.EncodeAssignerAsk(key, delivery.AssignerAsk{Ask: ask})
if err != nil {
t.Fatal(err)
}
return key, value, ask
}
// 1906/H5: given seven holds at once (as on 2026-10-25), when they are raised, then at most two are open at the router, a merge
// ask is still asked, and each hold answered lets the next waiting one be asked.
func TestABurstOfHoldsLeavesRoomForAMergeAsk(t *testing.T) {
b := newBed(t)
assigner, _ := withView(b)
var held []asks.Ask
for n := 0; n < 7; n++ {
key, value, ask := holdAskOf(t, n)
assigner[key] = value
b.t.OnAssignerAsk(key)
held = append(held, ask)
}
if got := b.askedIDs(); len(got) != mostAssignerOpen || got[0] != held[0].ID || got[1] != held[1].ID {
t.Fatalf("asked %v, want the first %d holds only; log:\n%s", got, mostAssignerOpen, b.logs.String())
}
_, merge := b.askOf(t, 1, 1)
if got := b.askedIDs(); len(got) != mostAssignerOpen+1 || got[mostAssignerOpen] != merge.Ask.ID {
t.Fatalf("asked %v, want the merge ask asked beside the holds", got)
}
if len(b.dropped) != 0 || len(b.conds) != 0 {
t.Errorf("router dropped %v, conditions %+v", b.dropped, b.conds)
}
b.deliver(t, ackFor(held[0], "release"))
got := b.askedIDs()
if len(got) != mostAssignerOpen+2 || got[len(got)-1] != held[2].ID {
t.Fatalf("asked %v after a hold was answered, want the third hold asked next; log:\n%s", got, b.logs.String())
}
// The merge ask answered frees its own place, and no hold takes it.
b.deliver(t, warrantFor(merge.Ask, "approve"))
if got := b.askedIDs(); len(got) != mostAssignerOpen+2 {
t.Errorf("asked %v after the merge ask was answered, want no further hold", got)
}
}

Some files were not shown because too many files have changed in this diff Show More