novox/mesh-delivery
19
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Compare commits

merge into: novox/mesh-delivery:feat/put-version
Branches Tags
novox/mesh-delivery:main novox/mesh-delivery:fix/1945-vault-hashing novox/mesh-delivery:fix/1945-user-list-follows novox/mesh-delivery:feat/put-version novox/mesh-delivery:fix/1945-bus-user-list-from-old-form novox/mesh-delivery:feat/try-speed novox/mesh-delivery:fix/1921-every-failure novox/mesh-delivery:feat/r4g-finish-composer novox/mesh-delivery:fix/1901 novox/mesh-delivery:fix/r4e-r2 novox/mesh-delivery:fix/r4e-k1 novox/mesh-delivery:feat/r4e-builder novox/mesh-delivery:feat/r4e-checker novox/mesh-delivery:feat/r4e-view novox/mesh-delivery:feat/r4c5-checker
..
pull from: novox/mesh-delivery:fix/1901
Branches Tags
novox/mesh-delivery:fix/1945-vault-hashing novox/mesh-delivery:main novox/mesh-delivery:fix/1945-user-list-follows novox/mesh-delivery:feat/put-version novox/mesh-delivery:fix/1945-bus-user-list-from-old-form novox/mesh-delivery:feat/try-speed novox/mesh-delivery:fix/1921-every-failure novox/mesh-delivery:feat/r4g-finish-composer novox/mesh-delivery:fix/1901 novox/mesh-delivery:fix/r4e-r2 novox/mesh-delivery:fix/r4e-k1 novox/mesh-delivery:feat/r4e-builder novox/mesh-delivery:feat/r4e-checker novox/mesh-delivery:feat/r4e-view novox/mesh-delivery:feat/r4c5-checker
1 Commits
feat/put-version .. fix/1901
Author SHA1 Message Date
jschoubben 970652f1b9 mesh-delivery-checker: a failed test's verdict keeps its last lines and the status line names its error (issue 1901) 2026-10-23 18:07:47 +02:00
133 changed files with 716 additions and 12285 deletions
Expand all files Collapse all files
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/asks.go
-161
View File
@@ -1,161 +0,0 @@
package main
import (
"context"
"sort"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"mesh-delivery-assigner/internal/assign"
)
// AssignerModule is the asker the assigner's compose asks name.
const AssignerModule = "mesh-delivery-assigner"
// composeAsks are the compose asks the assigner put on the Go build seat's check stream (novox/hq ADR 0482 clause 4,
// issues 1622 and 1623), by what each composes: one ask per key, put once and followed by a watch of its step key until
// it ends, which wakes a pass. No timer: an ask that waits in the stream waits (its backlog is the view's and the
// watchman's to show). A restart forgets them and asks again; the build module keeps a composer built for a commit, so a
// second ask builds nothing.
type composeAsks struct {
ctx context.Context
asker buildrun.Asker
wake func()
logf func(string, ...any)
now func() time.Time
mu sync.Mutex
asks map[string]*composeAsk
}
type composeAsk struct {
id string
cancel context.CancelFunc
ran *assign.Ran
}
func newComposeAsks(ctx context.Context, asker buildrun.Asker, wake func(), logf func(string, ...any)) *composeAsks {
return &composeAsks{ctx: ctx, asker: asker, wake: wake, logf: logf, now: time.Now, asks: map[string]*composeAsk{}}
}
// get is the end of the ask that composes key, or nil and its id while it runs. The first get of a key puts the ask.
// A job its layout refuses ends at once, failed with why: never silent.
func (a *composeAsks) get(key string, job delivery.ComposeJob) (*assign.Ran, string) {
a.mu.Lock()
defer a.mu.Unlock()
if k, ok := a.asks[key]; ok {
return k.ran, k.id
}
ask := delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: AssignerModule,
At: a.now(), Compose: &job}
ctx, cancel := context.WithCancel(a.ctx)
k := &composeAsk{id: ask.ID, cancel: cancel}
a.asks[key] = k
if err := a.asker.Put(ctx, ask); err != nil {
a.logf("the compose ask %s for %s could not be put: %v", ask.ID, key, err)
k.ran = &assign.Ran{State: delivery.StepFailed, Reason: "the compose ask could not be put on the Go build seat: " + err.Error()}
cancel()
return k.ran, k.id
}
go func() {
step, err := a.asker.Wait(ctx, ask)
if ctx.Err() != nil {
return
}
ran := assign.RanOf(step)
if err != nil {
ran = assign.Ran{State: delivery.StepFailed, Reason: "the compose ask's step could not be followed: " + err.Error()}
}
a.mu.Lock()
k.ran = &ran
a.mu.Unlock()
a.wake()
}()
return nil, k.id
}
// sweep forgets, and stops following, every ask whose key no pass asked for this time: a candidate gone or moved on.
func (a *composeAsks) sweep(seen map[string]bool) {
a.mu.Lock()
defer a.mu.Unlock()
for key, k := range a.asks {
if !seen[key] {
k.cancel()
delete(a.asks, key)
}
}
}
// composeAtHead is the composition of a group with a mesh-sdk member: the composer at its head and at each reader's
// mesh-sdk, asked of the Go build seat, and the verdict once every ask ended; until then the verdict waits, naming the
// asks. The keys it used are marked in seen.
func (p *passer) composeAtHead(ctx context.Context, m assign.Mesh, ch assign.Change, seen map[string]bool, began time.Time) delivery.CompositionVerdict {
waiting := func(words string) delivery.CompositionVerdict {
return delivery.CompositionVerdict{Group: ch.Group, Heads: ch.Heads, State: delivery.CompositionWaits, Words: words}
}
head, err := assign.HeadJob(m, ch)
if err != nil {
return delivery.CompositionVerdict{Group: ch.Group, Heads: ch.Heads, State: delivery.CompositionFail, SDK: "mesh-sdk " + ch.SDKHead,
Words: "the compose ask at the group's mesh-sdk head cannot be made: " + err.Error()}
}
var pending []string
key := "head." + ch.Group + "." + ch.Heads + "." + ch.SDKHead
seen[key] = true
headRan, id := p.asks.get(key, head)
if headRan == nil {
pending = append(pending, "mesh-sdk "+ch.SDKHead+" (ask "+id+")")
}
var readers []assign.Reader
var notJudged []string
for _, module := range sortedNames(assign.ReaderModules) {
repository := assign.ReaderModules[module]
var commits []string
member := false
if h := ch.Members[repository]; h != "" {
commits, member = []string{h}, true
} else {
commits = p.w.runningCommits(ctx, kvGetter(p.kvs[delivery.BuildsBucket]), module)
}
if len(commits) == 0 {
notJudged = append(notJudged, "not judged whether "+module+" reads the group's manifests: no build of it is assigned with a "+
"commit on record, and the group holds no member of "+repository)
continue
}
for _, commit := range commits {
job, err := assign.ReaderJob(ch, module, repository, commit)
if err != nil {
readers = append(readers, assign.Reader{Module: module, Commit: commit, Member: member,
Ran: assign.Ran{State: delivery.StepFailed, Reason: err.Error()}})
continue
}
rkey := "reader." + ch.Group + "." + ch.Heads + "." + module + "." + commit
seen[rkey] = true
ran, rid := p.asks.get(rkey, job)
if ran == nil {
pending = append(pending, module+" at "+commit+" (ask "+rid+")")
continue
}
readers = append(readers, assign.Reader{Module: module, Commit: commit, Member: member, Ran: *ran})
}
}
if len(pending) > 0 {
return waiting("composing on the Go build seat with the mesh-sdk of " + strings.Join(pending, ", "))
}
v := assign.FromComposer(m, ch, *headRan, readers, time.Since(began))
v.Warnings = append(v.Warnings, notJudged...)
return v
}
func sortedNames(m map[string]string) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compose_integration_test.go
-180
View File
@@ -1,180 +0,0 @@
//go:build integration
package main
import (
"encoding/json"
"errors"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/composer"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provider"
)
// holder is a test holder of the Go build seat: it takes every ask from the seat's check worker, as mesh-build-go does,
// and answers a compose ask by running mesh-sdk's composer over the request filled from the indexer's manifests (the
// linked composer standing for the one the build module builds at the asked commit), writing its step on its node's key.
type holder struct {
mu sync.Mutex
asks []delivery.StepAsk
}
func (h *holder) taken() []delivery.StepAsk {
h.mu.Lock()
defer h.mu.Unlock()
return append([]delivery.StepAsk(nil), h.asks...)
}
func (b *bed) hold(t *testing.T) *holder {
t.Helper()
seat, _ := delivery.AskStream(delivery.AskGo)
worker, _ := delivery.AskWorker(delivery.AskGo, delivery.AskCheck)
subject, _ := delivery.AskSubject(delivery.AskGo, delivery.AskCheck)
cons, err := b.js.CreateOrUpdateConsumer(b.ctx, seat, jetstream.ConsumerConfig{Durable: worker, FilterSubject: subject, AckPolicy: jetstream.AckExplicitPolicy})
if err != nil {
t.Fatal(err)
}
steps := b.buckets[delivery.StepsBucket(goBuildModule)]
h := &holder{}
cc, err := cons.Consume(func(m jetstream.Msg) {
a, err := delivery.DecodeStepAsk(m.Data())
if err != nil || a.Compose == nil {
t.Errorf("the seat was put an ask that is not a compose: %v %s", err, m.Data())
m.Ack()
return
}
h.mu.Lock()
h.asks = append(h.asks, a)
h.mu.Unlock()
req, err := composer.Decode(a.Compose.Request)
if err == nil {
err = composer.Fill(&req, func(digest string) ([]byte, error) {
key, _ := delivery.ManifestsKey(digest)
e, err := b.buckets[delivery.ManifestsBucket].Get(b.ctx, key)
if err != nil {
return nil, err
}
man, err := delivery.DecodeManifest(e.Value())
return []byte(man.JSON), err
})
}
step := delivery.Step{ID: a.ID, Tool: "compose", Node: "ace", State: delivery.StepPassed, Started: time.Now().UTC(), Ended: time.Now().UTC()}
if err != nil {
step.State, step.Reason = delivery.StepFailed, err.Error()
} else if v, err := composer.Run(req); err != nil {
step.State, step.Reason = delivery.StepFailed, err.Error()
} else {
vr, _ := json.Marshal(v)
step.Result, _ = json.Marshal(delivery.ComposeAnswer{Built: true, Key: "composer-0123456789abcdef", Verdict: vr})
}
raw, err := delivery.EncodeStep(step)
if err != nil {
t.Errorf("the step cannot be written: %v", err)
}
key, _ := delivery.AskStepKey("ace", a.ID)
if _, err := steps.Put(b.ctx, key, raw); err != nil {
t.Errorf("the step %s: %v", key, err)
}
m.Ack()
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(cc.Stop)
return h
}
// 1622/AC1-AC3 and 1623 on the bus: started whole, the assigner composes a group holding a mesh-sdk pull request by
// asking the Go build seat for the composer at the member's head, and for the composer at the mesh-sdk the running
// controller pins; it waits naming the asks while they run, writes the verdict from their answers naming the mesh-sdk the
// head's composer was built with, and asks nothing more on a later pass of the same heads.
func TestAGroupWithAnSDKMemberIsComposedByTheComposerAtItsHead(t *testing.T) {
b := startBed(t, "")
trunk := b.manifest(t, "web", `{"module":"web","version":"1"}`)
changed := b.manifest(t, "web", `{"module":"web","version":"2"}`)
mod, err := delivery.EncodeModule(delivery.Module{FP2: delivery.SourcePrefix2 + strings.Repeat("a", 64), Manifest: trunk, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
if err != nil {
t.Fatal(err)
}
b.putJSON(t, delivery.ModulesBucket, "web", mod)
b.putJSON(t, provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"build-1"}`))
running := strings.Repeat("3", 40)
b.putJSON(t, provider.AssignmentsBucket, "novox.mesh-controller", []byte(`{"node":"novox","module":"mesh-controller","build":"c-1"}`))
b.putJSON(t, delivery.BuildsBucket, delivery.BuildKey("mesh-controller", "c-1"), []byte(`{"id":"c-1","repository":"novox/mesh-controller","commit":"`+running+`"}`))
sdkHead := strings.Repeat("4", 40)
heads := []delivery.Head{{Repository: "novox/mesh-sdk", Number: 80, Head: sdkHead}, {Repository: "novox/web", Number: 7, Head: strings.Repeat("2", 40)}}
hash, _ := delivery.HeadsHash(heads)
pulls := []delivery.CandidatePull{{Head: heads[0]}, {Head: heads[1],
Modules: []string{"web"}, Fps: map[string]delivery.CandidateModule{"web": {FP2: delivery.SourcePrefix2 + strings.Repeat("b", 64), Manifest: changed}}}}
cand, err := delivery.EncodeCandidate(delivery.Candidate{Group: "feat/x", HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
if err != nil {
t.Fatal(err)
}
ckey, _ := delivery.CandidatesKey("feat/x")
b.putJSON(t, delivery.CandidatesBucket, ckey, cand)
b.start(t)
key, _ := delivery.CompositionKey("feat/x", hash)
// No holder takes the asks yet: the composition waits, naming them.
waits := b.verdict(t, key, func(v delivery.CompositionVerdict) bool { return v.State == delivery.CompositionWaits })
if !strings.Contains(waits.Words, "mesh-sdk "+sdkHead) || !strings.Contains(waits.Words, "mesh-controller at "+running) {
t.Fatalf("the waiting composition says %q, want the head's and the running controller's asks", waits.Words)
}
h := b.hold(t)
v := b.verdict(t, key, func(v delivery.CompositionVerdict) bool { return v.State != delivery.CompositionWaits })
if v.State != delivery.CompositionPass || len(v.Machines) != 1 || v.Machines[0] != "g14" || v.SDK == "" {
t.Fatalf("composition = %+v, want a pass on g14", v)
}
asks := h.taken()
if len(asks) != 2 {
t.Fatalf("the seat was asked %d composes, want two (the head, the running controller)", len(asks))
}
byRepo := map[string]delivery.StepAsk{}
for _, a := range asks {
byRepo[a.Compose.Fetch["repository"].(string)] = a
if a.Asker != AssignerModule || a.Kind != delivery.AskCheck || a.Language != delivery.AskGo {
t.Errorf("ask = %+v", a)
}
}
if a := byRepo["novox/mesh-sdk"]; a.Compose == nil || a.Compose.Fetch["commit"] != sdkHead || a.Compose.Dir != "go" {
t.Errorf("the head's ask = %+v", a.Compose)
}
if a := byRepo["novox/mesh-controller"]; a.Compose == nil || a.Compose.Fetch["commit"] != running || a.Compose.Dir != delivery.RootPath {
t.Errorf("the running controller's ask = %+v", a.Compose)
}
// A later pass of the same heads (woken by any change) asks nothing more.
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
b.putJSON(t, delivery.PresenceBucket, "ace", on)
time.Sleep(500 * time.Millisecond)
if n := len(h.taken()); n != 2 {
t.Errorf("a later pass of the same heads asked again: %d asks", n)
}
}
// verdict waits for the composition under key to satisfy ok, and answers it.
func (b *bed) verdict(t *testing.T, key string, ok func(delivery.CompositionVerdict) bool) delivery.CompositionVerdict {
t.Helper()
var last delivery.CompositionVerdict
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
e, err := b.buckets[delivery.AssignerCompositionsBucket].Get(b.ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
continue
}
if err != nil {
t.Fatal(err)
}
if last, err = delivery.DecodeCompositionVerdict(e.Value()); err != nil {
t.Fatal(err)
}
if ok(last) {
return last
}
}
t.Fatalf("no composition under %s as wanted after 10 s; last %+v", key, last)
return last
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go
+35 -642
View File
@@ -5,7 +5,6 @@ package main
import (
"context"
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
@@ -18,13 +17,9 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/announce"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/provider"
"mesh-delivery-assigner/internal/assign"
)
// bed is an embedded bus with JetStream, every bucket the assigner reads and keeps, and the stream the mesh's events
@@ -78,30 +73,12 @@ func startBed(t *testing.T, leaveOut string) *bed {
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>"}}); err != nil {
t.Fatal(err)
}
// The operator-channel seat's stream of conditions, which the assigner says its conditions on.
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "CONDITIONS", Subjects: []string{"mesh.seat.operator-channel.accept.condition.>"}}); err != nil {
t.Fatal(err)
}
// The Go build seat's work queue, as the broker derives it from the seat's accepted kinds (delivery.AskStream).
seat, _ := delivery.AskStream(delivery.AskGo)
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: seat, Subjects: []string{"mesh.seat.node-build-go.accept.>"},
Retention: jetstream.WorkQueuePolicy}); err != nil {
t.Fatal(err)
}
return b
}
// start runs the program as the node does, through run with its environment and a broker file; it answers the
// channel run's error arrives on.
func (b *bed) start(t *testing.T) <-chan error {
t.Helper()
done, _ := b.startStoppable(t)
return done
}
// startStoppable is start, and answers the function that stops this one program (a replaced assigner is stopped, then
// another started).
func (b *bed) startStoppable(t *testing.T) (<-chan error, func()) {
t.Helper()
broker := filepath.Join(b.dir, "broker")
cred, _ := json.Marshal(process.Account{URL: b.srv.ClientURL(), Node: "n1", Module: "mesh-delivery-assigner", User: "n1.mesh-delivery-assigner", Password: "pw"})
@@ -111,23 +88,19 @@ func (b *bed) startStoppable(t *testing.T) (<-chan error, func()) {
env := map[string]string{"MESH_BROKER_FILE": broker}
ctx, stop := context.WithCancel(b.ctx)
done := make(chan error, 1)
go func() {
done <- run(ctx, func(k string) string { return env[k] }, io.Discard)
close(done) // a program that ended on its own (a refused start) is not waited for again at the cleanup
}()
stopped := func() {
go func() { done <- run(ctx, func(k string) string { return env[k] }, io.Discard) }()
t.Cleanup(func() {
stop()
select {
case <-done:
case <-time.After(10 * time.Second):
}
}
t.Cleanup(stopped)
return done, stopped
})
return done
}
// 1297/B2b: started whole on its credential, the assigner says on the bus every state its manifest declares, each readable,
// having opened every bucket it names. The states are read from module.json, so a state added there is expected here too.
// 1297/B2b: started whole on its credential, the assigner says on the bus the four states it keeps, each readable,
// having opened every bucket it names.
func TestTheAssignerStartsAndSaysWhatItKeeps(t *testing.T) {
b := startBed(t, "")
done := b.start(t)
@@ -151,21 +124,9 @@ func TestTheAssignerStartsAndSaysWhatItKeeps(t *testing.T) {
}
break
}
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
if err != nil {
t.Fatal(err)
}
m, err := catalogue.ParseManifest(raw)
if err != nil {
t.Fatal(err)
}
if said.Module != "mesh-delivery-assigner" || said.Node != "n1" || len(said.States) != len(m.State) {
t.Fatalf("the assigner said %+v, want the %d states its manifest declares", said, len(m.State))
}
for _, s := range m.State {
if !said.Readable(s.Name) {
t.Errorf("the assigner said %+v: its state %s is not readable", said, s.Name)
}
if said.Module != "mesh-delivery-assigner" || said.Node != "n1" || len(said.States) != 4 || !said.Readable("asks") ||
!said.Readable("wants") || !said.Readable("differences") || !said.Readable("compositions") {
t.Fatalf("the assigner said %+v, want its four states, each readable", said)
}
}
@@ -192,27 +153,6 @@ func (b *bed) putJSON(t *testing.T, bucket, key string, value []byte) {
}
}
// compose says the controller composed module for node at fp, with the digest d (a letter repeated) and the judgement of the
// move to it, in mesh-controller_composed.
func (b *bed) compose(t *testing.T, node, module, fp string, d byte, judgement delivery.Judgement) string {
t.Helper()
key, err := delivery.ComposedKey(node, module, fp)
if err != nil {
t.Fatal(err)
}
digest := "sha256:" + strings.Repeat(string(d), 64)
cd := delivery.ComposedDigest{Digest: digest, Judgement: judgement}
if judgement != delivery.JudgedNeither {
cd.Against, cd.Words = "sha256:"+strings.Repeat("0", 64), "cannot be told"
}
v, err := delivery.EncodeComposedDigest(cd)
if err != nil {
t.Fatal(err)
}
b.putJSON(t, delivery.ControllerComposedBucket, key, v)
return digest
}
// manifest keeps a manifest in the indexer's bucket by its digest and answers the digest.
func (b *bed) manifest(t *testing.T, module, body string) string {
t.Helper()
@@ -241,10 +181,8 @@ func TestAChangeIsComposedOnTheBus(t *testing.T) {
b.putJSON(t, provider.AssignmentsBucket, "ace.other", []byte(`{"node":"ace","module":"other","build":"build-2"}`))
heads := []delivery.Head{{Repository: "novox/web", Number: 7, Head: strings.Repeat("2", 40)}}
hash, _ := delivery.HeadsHash(heads)
// 1927/AC1: the candidate carries the fingerprint set the SDK computes, as the indexer writes it.
pulls := []delivery.CandidatePull{{Head: heads[0],
Modules: []string{"web"}, Fps: map[string]delivery.CandidateModule{"web": {FP2: delivery.SourcePrefix2 + strings.Repeat("b", 64), Manifest: changed}}}}
cand, err := delivery.EncodeCandidate(delivery.Candidate{Group: "feat/x", HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
cand, err := delivery.EncodeCandidate(delivery.Candidate{Group: "feat/x", HeadsHash: hash, Pulls: []delivery.CandidatePull{{Head: heads[0],
Modules: []string{"web"}, Fps: map[string]delivery.CandidateModule{"web": {FP2: delivery.SourcePrefix2 + strings.Repeat("b", 64), Manifest: changed}}}}})
if err != nil {
t.Fatal(err)
}
@@ -273,13 +211,6 @@ func TestAChangeIsComposedOnTheBus(t *testing.T) {
if took := time.Since(began); took > 5*time.Second {
t.Errorf("the composition took %s", took)
}
// 1622/AC3: a group without a mesh-sdk member builds no composer: no ask is put on the Go build seat.
seat, _ := delivery.AskStream(delivery.AskGo)
if st, err := b.js.Stream(b.ctx, seat); err != nil {
t.Fatal(err)
} else if info, err := st.Info(b.ctx); err != nil || info.State.Msgs != 0 {
t.Errorf("a group without a mesh-sdk member put %d asks on the Go build seat (%v)", info.State.Msgs, err)
}
}
// get waits for a key of one of the bed's buckets and answers its value.
@@ -294,13 +225,10 @@ func (b *bed) get(t *testing.T, bucket, key string) []byte {
return nil
}
// 1297/B4 on the bus, 1906/D5: the controller sends ace, g14 and shanks builds while the canary ace does not run F healthy yet.
// Started whole, the assigner wants F on the canary and writes no difference at all: there is no send to compare with (ADR 0539
// §8). 1945/AC19, AC21: shanks, which runs the older fingerprint (composed and granted) with no want, gets its first want at that
// fingerprint, never at F; g14, which runs a build of the old form, gets none, so web is on neither list there and the assigner
// raises the condition naming g14, web and the canary rule; once ace runs F healthy, g14's first want names F and the condition is
// cleared.
func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
// 1297/B4, B5 on the bus: the controller sent F to g14 while the canary ace does not run F healthy yet. Started whole,
// the assigner wants F on the canary, writes the difference on g14 explained by rule 3, and on ace one unexplained (the
// controller sends ace the older fingerprint while ace wants F).
func TestWantsAndTheirDifferencesOnTheBus(t *testing.T) {
b := startBed(t, "")
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
digest := b.manifest(t, "web", `{"module":"web","version":"1"}`)
@@ -312,21 +240,19 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
}
bkey, _ := delivery.BuilderBuildKey("web", fNew)
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
for _, fp := range []string{fNew, fOld} {
gkey, _ := delivery.ControllerGrantKey("web", fp)
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant)
}
runs := map[string]string{"ace": fOld, "g14": "build-9", "shanks": fOld}
for _, node := range []string{"ace", "g14", "shanks"} {
b.compose(t, node, "web", fNew, 'c', delivery.JudgedNeither)
gkey, _ := delivery.ControllerGrantKey("web", fNew)
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant)
for _, node := range []string{"ace", "g14"} {
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
b.putJSON(t, delivery.PresenceBucket, node, on)
have, _ := delivery.EncodeHave(delivery.Have{FP: runs[node], Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
have, _ := delivery.EncodeHave(delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, node+".web", have)
b.putJSON(t, provider.AssignmentsBucket, node+".web", []byte(`{"node":"`+node+`","module":"web","build":"b-`+node+`"}`))
}
b.compose(t, "shanks", "web", fOld, 'a', delivery.JudgedNeither)
b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-8"}`))
b.putJSON(t, provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-9"}`))
b.putJSON(t, delivery.BuildsBucket, delivery.BuildKey("web", "b-8"), []byte(`{"id":"b-8","repository":"novox/web","source-fingerprint":"`+fOld+`"}`))
b.putJSON(t, delivery.BuildsBucket, delivery.BuildKey("web", "b-9"), []byte(`{"id":"b-9","repository":"novox/web","source-fingerprint":"`+fNew+`"}`))
b.start(t)
wkey, _ := delivery.WantKey("ace", "web")
@@ -334,41 +260,18 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
if err != nil || want.FP != fNew {
t.Fatalf("the canary ace's want = %+v, %v", want, err)
}
if w := b.wantNow(t, "shanks", "web", "shanks' first want at what it runs", func(w delivery.Want) bool { return w.FP != "" }); w.FP != fOld || w.Composed != "sha256:"+strings.Repeat("a", 64) || w.Newest != fNew {
t.Errorf("shanks' want = %+v, want the fingerprint it runs naming its composition, held back from F", w)
dkey, _ := delivery.DifferenceKey("g14", "web")
d, err := delivery.DecodeDifference(b.get(t, delivery.AssignerDifferencesBucket, dkey))
if err != nil || d.Rule != delivery.RuleCanary || d.Sent != fNew {
t.Fatalf("g14's difference = %+v, %v; want rule 3", d, err)
}
var neither asks.Condition
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && neither.Key == ""; time.Sleep(100 * time.Millisecond) {
for k, c := range b.conditionsSaid(t) {
if strings.Contains(k, ".neither.") && c.State == asks.StateRaised {
neither = c
}
}
}
if neither.Subject.Machine != "g14" || neither.Subject.ID != "web" || neither.Severity != asks.SeverityUrgent || !strings.Contains(neither.Explanation, "canary") {
t.Errorf("the condition of web on neither list on g14 = %+v", neither)
}
if keys, err := b.buckets[delivery.AssignerDifferencesBucket].Keys(b.ctx); err == nil && len(keys) != 0 {
t.Errorf("differences were written: %v", keys)
akey, _ := delivery.DifferenceKey("ace", "web")
a, err := delivery.DecodeDifference(b.get(t, delivery.AssignerDifferencesBucket, akey))
if err != nil || !a.Unexplained || a.Want != fNew || a.Sent != fOld {
t.Fatalf("ace's difference = %+v, %v; want it unexplained", a, err)
}
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, "want.g14.web"); err == nil {
t.Errorf("g14, which runs an old-form build and is held back by the canary, was given a want: %s", e.Value())
}
// 1945/AC21: once the canary ace runs F healthy, g14's first want names F by the rules as usual, and web leaves neither list
// there by itself: the condition is cleared.
healthyF, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, "ace.web", healthyF)
if w := b.wantNow(t, "g14", "web", "g14's first want at F once the canary runs it healthy", func(w delivery.Want) bool { return w.FP != "" }); w.FP != fNew || w.Composed != "sha256:"+strings.Repeat("c", 64) || w.Newest != "" {
t.Errorf("g14's want = %+v, want F with its composition", w)
}
cleared := false
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && !cleared; time.Sleep(100 * time.Millisecond) {
c, ok := b.conditionsSaid(t)[neither.Key]
cleared = ok && c.State == asks.StateCleared
}
if !cleared {
t.Errorf("the condition of web on neither list on g14 was not cleared once g14 got its want")
t.Errorf("g14, held back by the canary with no want before, was given one: %s", e.Value())
}
}
@@ -388,522 +291,12 @@ func TestAHeldFingerprintIsAskedAbout(t *testing.T) {
b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-1"}`))
b.start(t)
hkey, _ := delivery.HeldKey("web", fNew)
// 1906/H3: the held record names the machine and keeps the health, when and the node's words.
if h, err := delivery.DecodeHeld(b.get(t, delivery.AssignerWantsBucket, hkey)); err != nil || h.Node != "ace" ||
!strings.HasPrefix(h.Words, "health failed, phase running, seen at ") || !strings.HasSuffix(h.Words, ": connection refused") {
if h, err := delivery.DecodeHeld(b.get(t, delivery.AssignerWantsBucket, hkey)); err != nil || h.Node != "ace" {
t.Fatalf("held = %+v, %v", h, err)
}
akey, _ := delivery.HoldAskKey("web", fNew)
a, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
if err != nil || a.Asking <= 0 || a.Ask.ID != delivery.AskingID(akey, a.Asking) || len(a.Ask.Options) != 2 {
if err != nil || a.Ask.ID != delivery.AskingID(akey, 0) || len(a.Ask.Options) != 2 {
t.Fatalf("the ask = %+v, %v", a, err)
}
}
// conditionsSaid reads what the assigner said on the operator channel's subject for conditions, by condition key.
func (b *bed) conditionsSaid(t *testing.T) map[string]asks.Condition {
t.Helper()
stream, err := b.js.Stream(b.ctx, "CONDITIONS")
if err != nil {
t.Fatal(err)
}
out := map[string]asks.Condition{}
cons, err := stream.CreateOrUpdateConsumer(b.ctx, jetstream.ConsumerConfig{AckPolicy: jetstream.AckNonePolicy})
if err != nil {
t.Fatal(err)
}
for {
batch, err := cons.FetchNoWait(100)
if err != nil {
t.Fatal(err)
}
n := 0
for m := range batch.Messages() {
n++
c, err := asks.DecodeCondition(m.Data())
if err != nil {
t.Fatal(err)
}
out[c.Key] = c
}
if n == 0 {
return out
}
}
}
// 1906/D3 (ADR 0539 §19), whole on the bus: a module holding data whose new fingerprint failed its health check on the
// canary is held, not set back, and asked about (restore it, or leave it stopped); the condition names the module, the node
// and the check's words.
func TestAFailedDataModuleIsAskedToRestore(t *testing.T) {
b := startBed(t, "")
fNew := delivery.SourcePrefix2 + strings.Repeat("d", 64)
digest := b.manifest(t, "mail", `{"module":"mail","version":"1","resources":[{"id":"data","type":"directory","path":"/var/lib/mail"}],"data":{"own":[{"id":"mail","path":"${dir:data}","class":"irreplaceable","backup":"copy"}]}}`)
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fNew, Manifest: digest, Repository: "novox/mail", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
b.putJSON(t, delivery.ModulesBucket, "mail", mod)
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
b.putJSON(t, delivery.PresenceBucket, "ace", on)
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
b.putJSON(t, delivery.HaveBucket, "ace.mail", failed)
wantKey, _ := delivery.WantKey("ace", "mail")
want, _ := delivery.EncodeWant(delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("c", 64)})
b.putJSON(t, delivery.AssignerWantsBucket, wantKey, want)
b.putJSON(t, provider.AssignmentsBucket, "ace.mail", []byte(`{"node":"ace","module":"mail","build":"b-1"}`))
b.start(t)
akey, _ := delivery.RestoreAskKey("ace", "mail", fNew)
a, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
if err != nil || len(a.Ask.Options) != 2 || a.Ask.Options[0].ID != "restore" || a.Ask.Options[1].ID != "leave-stopped" {
t.Fatalf("the restore ask = %+v, %v", a, err)
}
if w, err := delivery.DecodeWant(b.get(t, delivery.AssignerWantsBucket, wantKey)); err != nil || w.FP != fNew {
t.Errorf("the data module was set back: want = %+v, %v", w, err)
}
key := ""
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && key == ""; time.Sleep(100 * time.Millisecond) {
for k, c := range b.conditionsSaid(t) {
if c.State == asks.StateRaised && c.Subject.ID == "mail" && strings.Contains(k, ".held.") {
if c.Subject.Machine != "ace" || !strings.Contains(c.Explanation, "connection refused") {
t.Fatalf("the condition = %+v, want it to name ace and the check's words", c)
}
key = k
}
}
}
if key == "" {
t.Fatal("no condition for the held fingerprint was said")
}
}
// 1906/D8 (scenario D of acceptance test 18, ADR 0539 §19), whole on the bus: the assigner is stopped while it writes the wants
// of forty modules and another is started; every module's want is written, and each exactly once, by revision: nothing lost
// and nothing written again.
func TestTheAssignerReplacedResumesWithNothingLostOrRepeated(t *testing.T) {
b := startBed(t, "")
const modules = 40
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
b.putJSON(t, delivery.PresenceBucket, "ace", on)
names := make([]string, modules)
for i := range names {
name := fmt.Sprintf("mod%02d", i)
names[i] = name
fp := delivery.SourcePrefix2 + strings.Repeat(string(rune('a'+i%6)), 63) + fmt.Sprintf("%x", i%16)
digest := b.manifest(t, name, fmt.Sprintf(`{"module":%q,"version":"1"}`, name))
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/" + name, Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
b.putJSON(t, delivery.ModulesBucket, name, mod)
built, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
if err != nil {
t.Fatal(err)
}
bkey, _ := delivery.BuilderBuildKey(name, fp)
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
gkey, _ := delivery.ControllerGrantKey(name, fp)
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant)
b.compose(t, "ace", name, fp, 'c', delivery.JudgedNeither)
b.putJSON(t, provider.AssignmentsBucket, "ace."+name, []byte(fmt.Sprintf(`{"node":"ace","module":%q,"build":"b-%d"}`, name, i)))
}
count := func() int {
keys, err := b.buckets[delivery.AssignerWantsBucket].Keys(b.ctx)
if err != nil {
return 0
}
return len(keys)
}
_, stop := b.startStoppable(t)
for deadline := time.Now().Add(20 * time.Second); count() == 0 && time.Now().Before(deadline); time.Sleep(time.Millisecond) {
}
stop() // replaced, wherever its pass had got to
b.start(t)
for deadline := time.Now().Add(30 * time.Second); count() < modules && time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
}
time.Sleep(time.Second) // and no pass of the second writes any again
for _, name := range names {
key, _ := delivery.WantKey("ace", name)
history, err := b.buckets[delivery.AssignerWantsBucket].History(b.ctx, key)
if err != nil {
t.Errorf("the want of %s was never written: %v", name, err)
continue
}
if len(history) != 1 {
t.Errorf("the want of %s was written %d times, want once", name, len(history))
}
}
}
// 1906/D2 (ADR 0539 §6), whole on the bus: a want waits for the composition of its module for the node, names its digest once it
// appears, and a new revision of the composition at the same fingerprint is written as a new want; the node's own want
// `want.<node>.node` is written once the node's composition exists, and again when its digest changes.
func TestAWantWaitsForItsComposition(t *testing.T) {
b := startBed(t, "")
fNew := delivery.SourcePrefix2 + strings.Repeat("e", 64)
digest := b.manifest(t, "web", `{"module":"web","version":"1"}`)
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fNew, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
b.putJSON(t, delivery.ModulesBucket, "web", mod)
built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
bkey, _ := delivery.BuilderBuildKey("web", fNew)
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
gkey, _ := delivery.ControllerGrantKey("web", fNew)
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant)
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
b.putJSON(t, delivery.PresenceBucket, "ace", on)
b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-1"}`))
b.start(t)
wkey, _ := delivery.WantKey("ace", "web")
time.Sleep(time.Second)
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, wkey); err == nil {
t.Fatalf("a want was written before the composition existed: %s", e.Value())
}
first := b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedNeither)
if w, err := delivery.DecodeWant(b.get(t, delivery.AssignerWantsBucket, wkey)); err != nil || w.FP != fNew || w.Composed != first {
t.Fatalf("the want = %+v, %v; want F naming %s", w, err, first)
}
second := b.compose(t, "ace", "web", fNew, 'd', delivery.JudgedNeither)
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, wkey); err == nil {
if w, err := delivery.DecodeWant(e.Value()); err == nil && w.Composed == second {
break
}
}
}
if w, err := delivery.DecodeWant(b.get(t, delivery.AssignerWantsBucket, wkey)); err != nil || w.Composed != second {
t.Fatalf("after a new revision at the same fingerprint the want = %+v, %v; want it naming %s", w, err, second)
}
// the node's own want
nkey, _ := delivery.WantNodeKey("ace")
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, nkey); err == nil {
t.Fatalf("the node's own want was written before its composition: %s", e.Value())
}
ownKey, _ := delivery.ComposedNodeKey("ace")
own, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("a", 64), Judgement: delivery.JudgedNeither})
b.putJSON(t, delivery.ControllerComposedBucket, ownKey, own)
if w, err := delivery.DecodeNodeWant(b.get(t, delivery.AssignerWantsBucket, nkey)); err != nil || w.Composed != "sha256:"+strings.Repeat("a", 64) {
t.Fatalf("the node's own want = %+v, %v", w, err)
}
}
// fixture is a module web (or the manifest given) at fp on the node ace, which is online: built, granted and assigned.
func (b *bed) fixture(t *testing.T, name, fp, manifest string) {
t.Helper()
digest := b.manifest(t, name, manifest)
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/" + name, Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
b.putJSON(t, delivery.ModulesBucket, name, mod)
built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
bkey, _ := delivery.BuilderBuildKey(name, fp)
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
gkey, _ := delivery.ControllerGrantKey(name, fp)
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant)
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
b.putJSON(t, delivery.PresenceBucket, "ace", on)
b.putJSON(t, provider.AssignmentsBucket, "ace."+name, []byte(`{"node":"ace","module":"`+name+`","build":"b-1"}`))
}
func (b *bed) putWant(t *testing.T, node, module string, w delivery.Want) {
t.Helper()
key, _ := delivery.WantKey(node, module)
v, err := delivery.EncodeWant(w)
if err != nil {
t.Fatal(err)
}
b.putJSON(t, delivery.AssignerWantsBucket, key, v)
}
// wantNow waits until the want of module on node satisfies ok and answers it.
func (b *bed) wantNow(t *testing.T, node, module, what string, ok func(delivery.Want) bool) delivery.Want {
t.Helper()
key, _ := delivery.WantKey(node, module)
var last delivery.Want
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, key); err == nil {
if w, err := delivery.DecodeWant(e.Value()); err == nil {
last = w
if ok(w) {
return w
}
}
}
}
t.Fatalf("never saw: %s; the want is %+v", what, last)
return last
}
// answer says the teller wrote the operator's answer to the ask that stands at key (its warrant names that asking's id).
func (b *bed) answer(t *testing.T, key, option string) {
t.Helper()
a, err := delivery.DecodeAssignerAsk(key, b.get(t, delivery.AssignerAsksBucket, key))
if err != nil {
t.Fatal(err)
}
b.answerTo(t, key, option, a.Ask.ID)
}
// answerTo is answer to the ask with this id (the router's warrant names it).
func (b *bed) answerTo(t *testing.T, key, option, askID string) {
t.Helper()
v, err := delivery.EncodeAnswer(delivery.Answer{Option: option, Warrant: askID + "@2026-10-24T12:00:00Z", Words: "the operator"})
if err != nil {
t.Fatal(err)
}
b.putJSON(t, delivery.AnswersBucket, key, v)
}
func (b *bed) hasKey(bucket, key string) bool {
_, err := b.buckets[bucket].Get(b.ctx, key)
return err == nil
}
// 1906/D4 (ADR 0539 §19), whole on the bus: the operator's release of a held fingerprint writes its want again, by the rules,
// and clears the condition.
func TestAReleaseAnswerWritesTheWantAgain(t *testing.T) {
b := startBed(t, "")
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
b.compose(t, "ace", "web", fOld, 'a', delivery.JudgedNeither)
healthy, _ := delivery.EncodeHave(delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy) // it ran the older fingerprint healthy before
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
b.start(t)
w := b.wantNow(t, "ace", "web", "set back to the older fingerprint", func(w delivery.Want) bool { return w.FP == fOld })
if w.Newest != fNew {
t.Fatalf("the set-back want = %+v", w)
}
hold, _ := delivery.HoldAskKey("web", fNew)
b.get(t, delivery.AssignerAsksBucket, hold)
b.answer(t, hold, "release")
w = b.wantNow(t, "ace", "web", "the held fingerprint's want written again", func(w delivery.Want) bool { return w.FP == fNew })
if w.Composed != "sha256:"+strings.Repeat("b", 64) || w.Rule != 0 {
t.Errorf("the want = %+v", w)
}
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
cleared := false
for k, c := range b.conditionsSaid(t) {
cleared = cleared || (c.Subject.ID == "web" && c.State == asks.StateCleared && strings.Contains(k, ".held."))
}
if cleared {
return
}
}
t.Error("the condition of the held fingerprint was not cleared")
}
// askNow waits for the backup ask at key to be the one numbered asking and answers it.
func (b *bed) askNow(t *testing.T, key string, asking int) delivery.AssignerAsk {
t.Helper()
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err := b.buckets[delivery.AssignerAsksBucket].Get(b.ctx, key); err == nil {
if a, err := delivery.DecodeAssignerAsk(key, e.Value()); err == nil && a.Asking == asking {
return a
}
}
}
t.Fatalf("the ask at %s never became asking %d", key, asking)
return delivery.AssignerAsk{}
}
// 1906/D4 (ADR 0539 §16), whole on the bus: a module held on a node because its backup failed is asked about, and the answer
// is written into the node's want; an answer "again" followed by another failure is asked again, and its answer is another want
// value, so the node retries; an old answer is never taken for a new revision at the same fingerprint.
func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) {
b := startBed(t, "")
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
first := b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: first})
held, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseHeld, Health: delivery.HealthWaits, Detail: "no space left"})
b.putJSON(t, delivery.HaveBucket, "ace.web", held)
b.start(t)
akey, _ := delivery.BackupAskKey("ace", "web", fNew)
a := b.askNow(t, akey, 0)
if !strings.Contains(a.Ask.Explanation, "no space left") || !strings.Contains(a.Ask.Explanation, first) {
t.Fatalf("the backup ask = %+v", a)
}
// an answer to another ask is no answer to this one
b.answerTo(t, akey, delivery.BackupSkip, "someone-else")
time.Sleep(time.Second)
if w := b.wantNow(t, "ace", "web", "the want", func(delivery.Want) bool { return true }); w.Backup != "" {
t.Fatalf("an answer to another ask was taken: %+v", w)
}
b.answerTo(t, akey, delivery.BackupAgain, a.Ask.ID)
w1 := b.wantNow(t, "ace", "web", "the answer in the want", func(w delivery.Want) bool { return w.Backup == delivery.BackupAgain })
if w1.FP != fNew || w1.Composed != first {
t.Errorf("the want = %+v", w1)
}
// the node retries and the backup fails again: a new have value, asked again under the next number
again, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseHeld, Health: delivery.HealthWaits, Detail: "no space left, again"})
b.putJSON(t, delivery.HaveBucket, "ace.web", again)
a2 := b.askNow(t, akey, 2)
if a2.Ask.ID == a.Ask.ID {
t.Fatalf("asked again under the same id")
}
b.answerTo(t, akey, delivery.BackupAgain, a2.Ask.ID)
w2 := b.wantNow(t, "ace", "web", "the second answer another want value", func(w delivery.Want) bool { return w.Backup == delivery.BackupAgain && w.Words != w1.Words })
if w2.Words == w1.Words {
t.Errorf("the second answer left the want as it was: %+v", w2)
}
// a new revision at the same fingerprint, whose backup fails as well, is asked, and the old answer is not taken for it
second := b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedNeither)
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: second})
a3 := b.askNow(t, akey, 3)
if !strings.Contains(a3.Ask.Explanation, second) {
t.Errorf("the ask for the new revision = %+v", a3)
}
time.Sleep(time.Second)
if w := b.wantNow(t, "ace", "web", "the want", func(delivery.Want) bool { return true }); w.Backup != "" {
t.Errorf("the old answer was taken for the new revision: %+v", w)
}
}
const recorded = `{"module":"web","version":"1","upgrade":{"policy":"record","why":"a person takes each build"}}`
// 1906/D4 (ADR 0539 §14), whole on the bus: a move of a module of policy record waits for its ask's yes when judged to recreate
// a container, to move data or not to be judgeable, at a new fingerprint and at a new revision of the same alike; one judged
// neither is written at once.
func TestARecordMoveWaitsForTheYesAndANeitherMoveDoesNot(t *testing.T) {
b := startBed(t, "")
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
b.fixture(t, "web", fNew, recorded)
old := "sha256:" + strings.Repeat("a", 64)
b.putWant(t, "ace", "web", delivery.Want{FP: fOld, Composed: old})
nw := b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedRecreates)
b.start(t)
akey, _ := delivery.RecordAskKey("ace", "web", nw)
a, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
if err != nil || !strings.Contains(a.Ask.Explanation, "recreates a container") {
t.Fatalf("the record ask = %+v, %v", a, err)
}
time.Sleep(time.Second)
if w := b.wantNow(t, "ace", "web", "the old want", func(delivery.Want) bool { return true }); w.FP != fOld {
t.Fatalf("the move was written before the yes: %+v", w)
}
b.answer(t, akey, assign.OptionYes)
b.wantNow(t, "ace", "web", "the move after the yes", func(w delivery.Want) bool { return w.FP == fNew && w.Composed == nw })
// a new revision at the same fingerprint, judged to move data, waits as well
rev := b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedMovesData)
rkey, _ := delivery.RecordAskKey("ace", "web", rev)
b.get(t, delivery.AssignerAsksBucket, rkey)
time.Sleep(time.Second)
if w := b.wantNow(t, "ace", "web", "the want", func(delivery.Want) bool { return true }); w.Composed != nw {
t.Fatalf("the revision was written before the yes: %+v", w)
}
// and a revision judged neither is written at once
free := b.compose(t, "ace", "web", fNew, 'd', delivery.JudgedNeither)
b.wantNow(t, "ace", "web", "a move judged neither written at once", func(w delivery.Want) bool { return w.Composed == free })
}
// 1906/D4 (ADR 0539 §7, §15), whole on the bus: the bus's new fingerprint is not written, and a revision of its composition at the
// fingerprint it runs that differs only in its user list is written at once, with the shape the controller gives it: judged
// recreates (the file is mounted), policy record from the bus, user-list-only; no record ask is made.
func TestAUserListRevisionOfTheBusIsWrittenAtOnce(t *testing.T) {
b := startBed(t, "")
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
b.fixture(t, "bus", fNew, `{"module":"bus","version":"1","provides":[{"name":"mesh-bus"}]}`)
running, _ := delivery.EncodeHave(delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, "ace.bus", running)
b.putWant(t, "ace", "bus", delivery.Want{FP: fOld, Composed: "sha256:" + strings.Repeat("a", 64)})
b.compose(t, "ace", "bus", fNew, 'b', delivery.JudgedNeither)
b.start(t)
time.Sleep(time.Second)
if w := b.wantNow(t, "ace", "bus", "the want", func(delivery.Want) bool { return true }); w.FP != fOld {
t.Fatalf("the assigner moved the bus to a new fingerprint: %+v", w)
}
// the controller composes the bus at the fingerprint it runs with a new user list
key, _ := delivery.ComposedKey("ace", "bus", fOld)
free := "sha256:" + strings.Repeat("e", 64)
v, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: free, Judgement: delivery.JudgedRecreates, Against: "sha256:" + strings.Repeat("a", 64),
Words: "the user list file is mounted", UserListOnly: true, Policy: delivery.PolicyRecord, PolicyFrom: "the bus"})
if err != nil {
t.Fatal(err)
}
b.putJSON(t, delivery.ControllerComposedBucket, key, v)
w := b.wantNow(t, "ace", "bus", "the user-list revision written at once", func(w delivery.Want) bool { return w.Composed == free })
if w.FP != fOld {
t.Errorf("the want = %+v, want it at the fingerprint the bus runs", w)
}
rkey, _ := delivery.RecordAskKey("ace", "bus", free)
if b.hasKey(delivery.AssignerAsksBucket, rkey) {
t.Errorf("the user-list revision was asked")
}
}
// 1906/H2, whole on the bus: given a fingerprint held after one failed check, when every node that runs the module then runs
// it healthy, the assigner lifts the hold by itself: the held record and its ask are taken away, the held condition is
// cleared saying why, and the want stays at the fingerprint.
func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
b := startBed(t, "")
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
b.start(t)
hkey, _ := delivery.HeldKey("web", fNew)
akey, _ := delivery.HoldAskKey("web", fNew)
b.get(t, delivery.AssignerWantsBucket, hkey)
b.get(t, delivery.AssignerAsksBucket, akey)
healthy, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy)
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
if b.hasKey(delivery.AssignerWantsBucket, hkey) || b.hasKey(delivery.AssignerAsksBucket, akey) {
continue
}
for k, c := range b.conditionsSaid(t) {
if c.Subject.ID == "web" && c.State == asks.StateCleared && strings.Contains(k, ".held.") && strings.Contains(c.Resolved, "lifted its hold by itself") {
if w := b.wantNow(t, "ace", "web", "the want at the lifted fingerprint", func(w delivery.Want) bool { return w.FP == fNew }); w.Rule != 0 {
t.Errorf("the want = %+v", w)
}
return
}
}
}
t.Fatalf("the hold was not lifted: held %v, ask %v", b.hasKey(delivery.AssignerWantsBucket, hkey), b.hasKey(delivery.AssignerAsksBucket, akey))
}
// 1906/H2, whole on the bus: a hold lifted by itself and written again after the assigner was started again is asked under a
// new id, so the teller raises it; the answer to the first hold does not carry over (1906/H4).
func TestAHoldWrittenAgainAfterALiftAndARestartIsAskedAgain(t *testing.T) {
b := startBed(t, "")
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
healthy, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
_, stop := b.startStoppable(t)
hkey, _ := delivery.HeldKey("web", fNew)
akey, _ := delivery.HoldAskKey("web", fNew)
first, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
if err != nil {
t.Fatal(err)
}
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy)
for deadline := time.Now().Add(15 * time.Second); b.hasKey(delivery.AssignerWantsBucket, hkey); time.Sleep(100 * time.Millisecond) {
if time.Now().After(deadline) {
t.Fatal("the hold was not lifted")
}
}
stop()
time.Sleep(1100 * time.Millisecond) // the next hold is written in another second
b.answerTo(t, akey, "keep", first.Ask.ID) // an answer to the first hold, which the later hold must not take
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
b.start(t)
again, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
if err != nil {
t.Fatal(err)
}
if again.Ask.ID == first.Ask.ID {
t.Fatalf("the hold written again is asked under the first id %s", first.Ask.ID)
}
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy)
for deadline := time.Now().Add(15 * time.Second); b.hasKey(delivery.AssignerWantsBucket, hkey); time.Sleep(100 * time.Millisecond) {
if time.Now().After(deadline) {
t.Fatal("the stale \"keep\" of the first hold kept the second one held")
}
}
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go
+11 -80
View File
@@ -21,23 +21,12 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/announce"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/provider"
)
func main() {
if os.Getenv("MESH_BROKER_FILE") == "" {
// Started by the tool runner, with no credential of its own: the put's tools (ADR 0546), served through the runtime,
// since a process never serves (ADR 0436). The process the manifest runs names its credential.
if err := serveTools(); err != nil {
fmt.Fprintln(os.Stderr, "mesh-delivery-assigner:", err)
os.Exit(1)
}
return
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
if err := run(ctx, os.Getenv, os.Stderr); err != nil {
@@ -52,33 +41,24 @@ var readBuckets = []string{
provider.AssignmentsBucket,
delivery.BuildsBucket,
delivery.ControllerGrantsBucket,
delivery.ControllerComposedBucket,
delivery.AnswersBucket,
delivery.BuilderBuildsBucket,
delivery.CandidatesBucket,
delivery.ManifestsBucket,
delivery.ModulesBucket,
delivery.PresenceBucket,
delivery.HaveBucket,
delivery.StepsBucket(goBuildModule),
}
// goBuildModule is the Go build module, which takes the assigner's compose asks on its seat's check stream and writes
// their steps in its steps state (novox/hq ADR 0482 clause 4).
const goBuildModule = "mesh-build-go"
// ownBuckets are the assigner's own states, as its manifest declares them.
var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerCompositionsBucket, delivery.AssignerDifferencesBucket,
delivery.AssignerPutsBucket, delivery.AssignerWantsBucket}
var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerCompositionsBucket, delivery.AssignerDifferencesBucket, delivery.AssignerWantsBucket}
// keptStates is what the assigner keeps and who may read it (ADR 0495 §2-§3): its wants by the watchman and the view
// (and the node agent from release 6), its differences by the view, its compositions by the checker, its asks by the view
// and the teller, its puts by whoever follows one (ADR 0546).
// and the teller.
var keptStates = []announce.State{
{Name: "asks", Readable: true},
{Name: "compositions", Readable: true},
{Name: "differences", Readable: true},
{Name: "puts", Readable: true},
{Name: "wants", Readable: true},
}
@@ -86,17 +66,11 @@ var keptStates = []announce.State{
type buckets map[string]jetstream.KeyValue
// open opens every bucket the assigner reads and writes; a bucket it cannot open stops the start, named, since a pass
// without it would write wants from half the mesh. The puts' bucket is the exception: whatever keeps it from opening (it
// does not exist yet, the credential is not granted it yet, the bus does not answer in time), the assigner writes wants
// without it and takes no put further until it starts again (ADR 0546).
func open(ctx context.Context, js jetstream.JetStream, logf func(string, ...any)) (buckets, error) {
// without it would write wants from half the mesh.
func open(ctx context.Context, js jetstream.JetStream) (buckets, error) {
out := buckets{}
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
kv, err := js.KeyValue(ctx, name)
if name == delivery.AssignerPutsBucket && err != nil {
logf("the bucket %s cannot be opened (%v): no put is taken further until the assigner starts again", name, err)
continue
}
if err != nil {
return nil, fmt.Errorf("the bucket %s cannot be opened: %w", name, err)
}
@@ -135,7 +109,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil {
return err
}
kvs, err := open(ctx, js, logger.Printf)
kvs, err := open(ctx, js)
if err != nil {
return err
}
@@ -150,21 +124,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
logger.Printf("what the assigner keeps could not be said on the bus, so its readers are not granted until it is: %v", err)
}
logger.Printf("started on the account %s, reading %d buckets", account.User, len(kvs))
return follow(ctx, js, kvs, sayVia(ctx, js), emitting{said}, logger)
return follow(ctx, kvs, logger)
}
// emitting is the process connection as the puts' caller: its calls, and its events with no options.
type emitting struct{ *process.Conn }
func (e emitting) Emit(eventType string, body any) error { return e.Conn.Emit(eventType, body) }
// watched are the buckets whose every value a pass reads; the manifests are read by digest when a pass needs one, and
// watched only for keys, so a manifest that arrives wakes a composition that waits for it.
var watched = []string{
provider.AssignmentsBucket,
delivery.ControllerGrantsBucket,
delivery.ControllerComposedBucket,
delivery.AnswersBucket,
delivery.BuilderBuildsBucket,
delivery.CandidatesBucket,
delivery.ModulesBucket,
@@ -178,7 +145,7 @@ var watched = []string{
// follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after
// each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends.
func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(asks.Condition) error, bus caller, logger *log.Logger) error {
func follow(ctx context.Context, kvs buckets, logger *log.Logger) error {
w := newWorld()
dirty := make(chan struct{}, 1)
wake := func() {
@@ -187,15 +154,13 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
default:
}
}
errs := make(chan error, len(watched)+2)
settled := make(chan struct{}, len(watched)+2)
errs := make(chan error, len(watched)+1)
settled := make(chan struct{}, len(watched)+1)
watch := func(name string, opts ...jetstream.WatchOpt) error {
kw, err := kvs[name].WatchAll(ctx, opts...)
if err != nil {
return fmt.Errorf("the bucket %s cannot be watched: %w", name, err)
}
// The puts' watch never stops the wants (ADR 0546): when it ends, the puts are no longer heard, said in the journal.
optional := name == delivery.AssignerPutsBucket
go func() {
defer kw.Stop()
for {
@@ -205,10 +170,6 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
return
case e, ok := <-kw.Updates():
if !ok {
if optional {
logger.Printf("the watch of %s ended: no put is taken further until the assigner starts again", name)
return
}
errs <- fmt.Errorf("the watch of %s ended", name)
return
}
@@ -227,15 +188,6 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
}()
return nil
}
followed := watched
if kvs[delivery.AssignerPutsBucket] != nil {
if err := watch(delivery.AssignerPutsBucket); err != nil {
logger.Printf("%v: no put is taken further until the assigner starts again", err)
delete(kvs, delivery.AssignerPutsBucket)
} else {
followed = append(append([]string{}, watched...), delivery.AssignerPutsBucket)
}
}
for _, name := range watched {
var opts []jetstream.WatchOpt
if name == delivery.HaveBucket {
@@ -248,7 +200,7 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
if err := watch(delivery.ManifestsBucket, jetstream.MetaOnly()); err != nil {
return err
}
for range len(followed) + 1 {
for range len(watched) + 1 {
select {
case <-ctx.Done():
return nil
@@ -257,11 +209,7 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
case <-settled:
}
}
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, said: map[string]bool{}, answeredAt: map[string]uint64{}}
p.asks = newComposeAsks(ctx, buildrun.Asker{Publisher: buildrun.JetStreamAsks{JS: js}, Waiter: buildrun.JetStreamAsks{JS: js}}, wake, logger.Printf)
if bus != nil && kvs[delivery.AssignerPutsBucket] != nil {
p.acts = newPutActs(bus, wake, logger.Printf)
}
p := &passer{kvs: kvs, w: w, logf: logger.Printf}
wake()
for {
select {
@@ -275,20 +223,3 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
}
}
}
// sayVia publishes a condition on the operator-channel seat's one subject for conditions, under the assigner's own name
// (ADR 0437 clause 9).
func sayVia(ctx context.Context, js jetstream.JetStream) func(asks.Condition) error {
return func(c asks.Condition) error {
subject, err := asks.ConditionSubject(delivery.AssignerModule)
if err != nil {
return err
}
data, err := asks.EncodeCondition(c)
if err != nil {
return err
}
_, err = js.Publish(ctx, subject, data)
return err
}
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go
-40
View File
@@ -2,8 +2,6 @@ package main
import (
"context"
"errors"
"fmt"
"io"
"os"
"path/filepath"
@@ -12,10 +10,7 @@ import (
"strings"
"testing"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// 1297/B2a: the buckets the program opens are exactly the manifest's reads and states, as the broker names them
@@ -53,27 +48,6 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
if got, want := slices.Sorted(slices.Values(kept)), slices.Sorted(slices.Values(ownBuckets)); !reflect.DeepEqual(got, want) {
t.Errorf("the program announces %v, keeps %v", got, want)
}
// ADR 0546: whatever keeps the puts' bucket from opening (not there, not granted, no answer in time), the assigner opens
// the rest and starts, writing wants; any other bucket it cannot open stops the start, named.
for _, c := range []struct {
missing string
err error
starts bool
}{
{delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true},
{delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true},
{delivery.AssignerPutsBucket, context.DeadlineExceeded, true},
{delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false},
} {
var logged string
kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err}, func(f string, a ...any) { logged = fmt.Sprintf(f, a...) })
switch {
case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, "no put is taken further")):
t.Errorf("%s refused with %v: the start answered %v, %d buckets, logged %q", c.missing, c.err, err, len(kvs), logged)
case !c.starts && (err == nil || !strings.Contains(err.Error(), c.missing)):
t.Errorf("%s refused with %v: the start answered %v", c.missing, c.err, err)
}
}
}
// 1297/B2a: without a credential the program does not start, and says why.
@@ -83,17 +57,3 @@ func TestNoCredential(t *testing.T) {
t.Errorf("run without a credential = %v", err)
}
}
// refusingJS opens every bucket but one, which it refuses with err.
type refusingJS struct {
jetstream.JetStream
missing string
err error
}
func (r refusingJS) KeyValue(_ context.Context, name string) (jetstream.KeyValue, error) {
if name == r.missing {
return nil, r.err
}
return &memKV{values: map[string][]byte{}}, nil
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/mesh-delivery-assigner
BIN
View File
Binary file not shown.
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go
+64 -382
View File
@@ -3,10 +3,8 @@ package main
import (
"context"
"errors"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/sdkversion"
@@ -21,58 +19,75 @@ type passer struct {
w *world
logf func(string, ...any)
sdk string
// say publishes a condition to the operator; said is, by condition key and state, what this process has said, so a
// pass says a condition once and a restart says it again (the router keeps one per key).
say func(asks.Condition) error
said map[string]bool
// answeredAt is, by the id of a backup ask, how many values the have had when its answer was written into the want.
answeredAt map[string]uint64
asks *composeAsks
// now is the clock; nil is time.Now.
now func() time.Time
// neither is, by node and module, each module on neither list whose condition this process raised (issue 1945). Nil until the
// first pass after a start, which clears what a previous process may have raised (startClears).
neither map[string]neitherOne
// acts are the puts' acts (ADR 0546); nil: the puts are not taken further (no bus to call, or no puts bucket).
acts *putActs
}
func (p *passer) clock() time.Time {
if p.now != nil {
return p.now()
}
return time.Now()
}
func (p *passer) pass(ctx context.Context) {
p.compositions(ctx)
p.wants(ctx)
p.puts(ctx)
p.nodeWants(ctx)
p.backups(ctx)
rules := p.wants(ctx)
p.differences(ctx, rules)
}
// nodeWants writes each node's own want, `want.<node>.node`, naming the digest of the node's own composition, once the
// controller has composed it and again whenever that digest changes (ADR 0539 §6).
func (p *passer) nodeWants(ctx context.Context) {
written := p.w.read(delivery.AssignerWantsBucket)
for node, d := range p.w.nodeComposed() {
key, err := delivery.WantNodeKey(node)
// differences writes, for each node and module the assigner wants or the controller sends, the difference between the
// two with the rule that explains it or unexplained (ADR 0528 clause 4), and deletes the key of a difference that
// ended; its history keeps what it was.
func (p *passer) differences(ctx context.Context, rules map[string]map[string]assign.Holding) {
_, assignments := p.w.assigned()
wanted := p.w.wantsOf()
trunk := p.w.trunk()
pairs := map[[2]string]bool{}
for _, a := range assignments {
pairs[[2]string{a.Node, a.Module}] = true
}
for module, byNode := range wanted {
for node := range byNode {
pairs[[2]string{node, module}] = true
}
}
written := p.w.read(delivery.AssignerDifferencesBucket)
keep := map[string]bool{}
for pair := range pairs {
node, module := pair[0], pair[1]
key, err := delivery.DifferenceKey(node, module)
if err != nil {
continue
}
if before, err := delivery.DecodeNodeWant(written[key]); err == nil && before.Composed == d.Digest {
var want *delivery.Want
if w, ok := wanted[module][node]; ok {
want = &w
}
var sent assign.Sent
if a, ok := assignments[node+"."+module]; ok {
sent = assign.Sent{Sends: true, Build: a.Build, FP: p.w.sentFP(ctx, kvGetter(p.kvs[delivery.BuildsBucket]), module, a.Build)}
}
d, ok := assign.Compare(want, sent, trunk[module].FP2, rules[module][node])
if !ok {
continue
}
want := delivery.NodeWant{Composed: d.Digest, Words: "the node's own composition"}
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeNodeWant(want) })
keep[key] = true
if before, err := delivery.DecodeDifference(written[key]); err == nil {
before.V = 0
if delivery.SameDifference(before, d) {
continue
}
}
p.put(ctx, delivery.AssignerDifferencesBucket, key, func() ([]byte, error) { return delivery.EncodeDifference(d) })
}
for key := range written {
if keep[key] {
continue
}
if err := p.kvs[delivery.AssignerDifferencesBucket].Delete(ctx, key); err != nil {
p.logf("the difference %s that ended could not be deleted: %v", key, err)
continue
}
p.w.put(delivery.AssignerDifferencesBucket, key, nil)
}
}
// wants writes, for every module the trunk holds a fingerprint of, each node's want by the six rules (ADR 0429 §6), and
// the hold of a fingerprint whose health check failed. The assigner compares its wants with nothing the controller
// sends: the differences bucket is kept read-only with its history (ADR 0539 §8).
func (p *passer) wants(ctx context.Context) {
// the hold of a fingerprint whose health check failed. It returns, by module then node, the rule that holds a node
// back, for the comparison.
func (p *passer) wants(ctx context.Context) map[string]map[string]assign.Holding {
get := kvGetter(p.kvs[delivery.ManifestsBucket])
assigned, _ := p.w.assigned()
trunk := p.w.trunk()
@@ -93,7 +108,7 @@ func (p *passer) wants(ctx context.Context) {
}
}
wanted := p.w.wantsOf()
puts := p.putsByModule()
rules := map[string]map[string]assign.Holding{}
names := map[string]bool{}
for name := range trunk {
names[name] = true
@@ -101,59 +116,20 @@ func (p *passer) wants(ctx context.Context) {
for name := range wanted {
names[name] = true
}
neither := map[string]neitherOne{}
// The first pass after a start remembers nothing a previous process said: a neither or held condition it raised would stay
// raised for ever once the module has its want or is no longer held. That pass clears, once, the neither condition of every
// assigned module not on neither list and the held condition of every module not held; a clear of a condition that is not open is
// quiet at the router.
first := p.neither == nil
stale := map[string]neitherOne{}
defer func() { p.sayNeither(neither, stale) }()
for name := range names {
fp := trunk[name].FP2
name := name
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name],
Composed: func(node, fp string) (delivery.ComposedDigest, bool) { return p.w.composed(node, name, fp) },
GrantedAt: func(fp string) bool {
key, err := delivery.ControllerGrantKey(name, fp)
return err == nil && p.w.has(delivery.ControllerGrantsBucket, key)
},
Restored: func(node string) bool {
key, err := delivery.RestoreAskKey(node, name, fp)
return err == nil && p.answeredWith(key, assign.OptionRestore)
},
Allowed: func(node, digest string) bool {
key, err := delivery.RecordAskKey(node, name, digest)
return err == nil && p.answeredWith(key, assign.OptionYes)
}}
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name]}
if fp != "" {
held, _ := delivery.HeldKey(name, fp)
grant, _ := delivery.ControllerGrantKey(name, fp)
hold, _ := delivery.HoldAskKey(name, fp)
m.Released = p.answeredWith(hold, "release")
m.Kept = p.answeredWith(hold, "keep")
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(delivery.ControllerGrantsBucket, grant)
if raw := p.w.value(delivery.ControllerGrantsBucket, grant); raw != nil {
if g, err := delivery.DecodeControllerGrant(raw); err == nil {
m.GrantedOn = func(node string) bool { return assign.GrantNamesNode(g, node, name) }
}
}
if m.Released {
p.clearHeld(name, fp)
}
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held), p.w.has(delivery.ControllerGrantsBucket, grant)
}
last := map[string]string{}
for _, node := range m.Nodes {
last[node] = p.w.lastHealthy(node, name, fp)
}
out := assign.Wants(m, nodes, wanted[name], last)
gatherNeither(neither, name, out)
if first {
startClears(stale, m, out)
if fp != "" && !m.Held && out.Held == nil && !out.Lifted {
p.clearHeld(name, fp)
}
}
rules[name] = out.Rules
if out.Held != nil {
key, err := delivery.HeldKey(name, fp)
if err == nil {
@@ -161,18 +137,8 @@ func (p *passer) wants(ctx context.Context) {
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeHeld(held) })
}
}
if out.Lifted {
p.lift(ctx, name, fp)
}
if out.Held != nil || (m.Held && !out.Lifted) {
p.holdAsk(ctx, name, fp, out.Held)
p.heldCondition(ctx, name, fp, out.Held, len(out.Restore) > 0)
}
for _, node := range out.Restore {
p.restoreAsk(ctx, name, node, fp)
}
for _, need := range out.Record {
p.recordAsk(ctx, name, fp, need)
if out.Held != nil || m.Held {
p.holdAsk(ctx, name, fp)
}
for node, want := range out.Wants {
key, err := delivery.WantKey(node, name)
@@ -184,74 +150,7 @@ func (p *passer) wants(ctx context.Context) {
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeWant(want) })
}
}
}
// neitherOne is a module assigned to a node that holds no want for it after a pass, and the rule holding its first want back.
type neitherOne struct {
node, module string
holding assign.Holding
}
// gatherNeither adds the nodes on which module is on neither list in its outcome, keyed by node and module.
func gatherNeither(into map[string]neitherOne, module string, out assign.Outcome) {
for node, h := range out.Neither {
into[node+"."+module] = neitherOne{node: node, module: module, holding: h}
}
}
// startClears adds, for the first pass after a start, every node of m that is not on neither list after the pass: a neither
// condition a previous process raised for it is cleared. That covers a node with a want and one with none that is not on neither
// list, such as a held module (issue 1945: the hold is said on its own, so a neither raised for it before the restart would stay up
// for ever) or a move waiting for its record ask. Clearing a condition that is not open is quiet at the router.
func startClears(into map[string]neitherOne, m assign.Module, out assign.Outcome) {
for _, node := range m.Nodes {
if _, on := out.Neither[node]; !on {
into[node+"."+m.Name] = neitherOne{node: node, module: m.Name}
}
}
}
// sayNeither raises, once, the condition of every module on neither list in this pass and writes it to the journal, and clears,
// once, the condition of each one said before that has a want again (issue 1945). A say that fails is made again at the next pass.
// stale is, on the first pass after a start, the modules whose condition a previous process may have raised and that have a want:
// each is cleared once.
func (p *passer) sayNeither(now, stale map[string]neitherOne) {
if p.neither == nil {
p.neither = map[string]neitherOne{}
for key, one := range stale {
if _, on := now[key]; !on {
p.neither[key] = one
}
}
}
for key, one := range now {
if _, said := p.neither[key]; said {
continue
}
c := assign.NeitherCondition(one.node, one.module, one.holding, true)
if p.say != nil {
if err := p.say(c); err != nil {
p.logf("the condition of %s on neither list on %s could not be said, and is said at the next pass: %v", one.module, one.node, err)
continue
}
}
p.logf("%s is assigned to %s and on neither list: no want, and not in the node's own composition; rule %d holds its first want back: %s",
one.module, one.node, one.holding.Rule, one.holding.Words)
p.neither[key] = one
}
for key, one := range p.neither {
if _, still := now[key]; still {
continue
}
if p.say != nil {
if err := p.say(assign.NeitherCondition(one.node, one.module, assign.Holding{}, false)); err != nil {
p.logf("the condition of %s on neither list on %s could not be cleared, and is cleared at the next pass: %v", one.module, one.node, err)
continue
}
}
p.logf("%s on %s has a want again", one.module, one.node)
delete(p.neither, key)
}
return rules
}
// compositions writes the composition check's verdict of every group the indexer holds a candidate for, under the
@@ -263,10 +162,6 @@ func (p *passer) compositions(ctx context.Context) {
written := p.w.read(delivery.AssignerCompositionsBucket)
trunk := p.w.trunk()
var mesh *assign.Mesh
seen := map[string]bool{}
if p.asks != nil {
defer p.asks.sweep(seen)
}
for key, value := range p.w.read(delivery.CandidatesBucket) {
c, err := delivery.DecodeCandidate(value)
if err != nil {
@@ -288,12 +183,7 @@ func (p *passer) compositions(ctx context.Context) {
m := p.w.mesh(ctx, kvGetter(p.kvs[delivery.ManifestsBucket]))
mesh = &m
}
if ch.SDKHead != "" && p.asks != nil {
// A group with a mesh-sdk member is composed by the composer built at its head (ADR 0482 clauses 3 to 5).
v = p.composeAtHead(ctx, *mesh, ch, seen, began)
} else {
v = assign.Compose(*mesh, ch, p.sdk, time.Since(began))
}
v = assign.Compose(*mesh, ch, p.sdk, time.Since(began))
}
if before, err := delivery.DecodeCompositionVerdict(written[out]); err == nil {
before.Took, before.V = v.Took, 0
@@ -322,219 +212,11 @@ func (p *passer) put(ctx context.Context, bucket, key string, encode func() ([]b
// holdAsk writes, once, the ask about a held fingerprint (ADR 0528 clause 6, ADR 0437 clause 10): the first ask that needs
// no proof of who answered, which the view may answer and the teller fills. In release 5 an answer changes nothing on a
// node (wants are only compared, ADR 0429 §7), and the options say so.
func (p *passer) holdAsk(ctx context.Context, module, fp string, fresh *delivery.Held) {
func (p *passer) holdAsk(ctx context.Context, module, fp string) {
key, err := delivery.HoldAskKey(module, fp)
if err != nil || p.w.has(delivery.AssignerAsksBucket, key) {
return
}
// The asking is the hold's time in seconds: a hold of the same fingerprint written again after a lift is another asking, under
// an id of its own, also after a restart, since the router drops a second ask under an id it keeps (issue 1906).
now := p.clock()
asking := int(now.Unix())
h := p.heldHold(module, fp, fresh)
a := assign.HoldAsk(key, module, fp, h.Node, h.Words, asking, now)
a := assign.HoldAsk(key, module, fp, time.Now())
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
}
// lift takes away a hold the assigner lifted by itself, since every node that runs the module runs the held fingerprint healthy
// (issue 1906): the held record and its ask (the teller then takes the ask back from the operator), and the held condition is
// cleared saying why, once. A hold of the same fingerprint written later is said and asked again.
func (p *passer) lift(ctx context.Context, module, fp string) {
held, err := delivery.HeldKey(module, fp)
if err != nil {
return
}
ask, err := delivery.HoldAskKey(module, fp)
if err != nil {
return
}
c := assign.LiftedCondition(module, fp)
if !p.said[c.Key+".lifted"] {
if p.say != nil {
if err := p.say(c); err != nil {
// The hold stays written, so the next pass lifts it again and says so then.
p.logf("the condition of the lifted hold of %s could not be cleared, and is cleared at the next pass: %v", module, err)
return
}
}
p.logf("%s", assign.LiftedWords(module, fp))
p.said[c.Key+".lifted"] = true
delete(p.said, c.Key+".raised") // a hold written again is said again
}
p.remove(ctx, delivery.AssignerWantsBucket, held)
p.remove(ctx, delivery.AssignerAsksBucket, ask)
}
// remove deletes one key of the assigner's own bucket when it holds it; a delete that fails is said and made again at the next
// pass.
func (p *passer) remove(ctx context.Context, bucket, key string) {
if !p.w.has(bucket, key) {
return
}
if err := p.kvs[bucket].Delete(ctx, key); err != nil && !errors.Is(err, context.Canceled) {
p.logf("%s %s could not be taken away: %v", bucket, key, err)
return
}
p.w.put(bucket, key, nil)
}
// heldHold is the hold of fp as written: the node whose health check failed and its words.
func (p *passer) heldHold(module, fp string, fresh *delivery.Held) delivery.Held {
if fresh != nil {
return *fresh
}
key, err := delivery.HeldKey(module, fp)
if err != nil {
return delivery.Held{}
}
h, _ := delivery.DecodeHeld(p.w.read(delivery.AssignerWantsBucket)[key])
return h
}
// heldCondition raises, once, the condition for a fingerprint the assigner holds (ADR 0429 §6, ADR 0539 §19): the module, the
// node whose health check failed and the check's words. A module holding data needs the operator's answer to its restore ask.
func (p *passer) heldCondition(ctx context.Context, module, fp string, fresh *delivery.Held, data bool) {
h := p.heldHold(module, fp, fresh)
c := assign.HeldCondition(module, fp, h.Node, h.Words, true, data)
if p.say == nil || p.said[c.Key+".raised"] {
return
}
if err := p.say(c); err != nil {
p.logf("the condition of the held %s could not be said, and is said at the next pass: %v", module, err)
return
}
p.said[c.Key+".raised"] = true
delete(p.said, c.Key+".lifted") // a hold raised again and lifted again is said lifted again
delete(p.said, c.Key+".cleared") // and released again is cleared again
}
// restoreAsk writes, once, the ask about a module holding data whose held fingerprint failed its health check on node.
func (p *passer) restoreAsk(ctx context.Context, module, node, fp string) {
key, err := delivery.RestoreAskKey(node, module, fp)
if err != nil || p.w.has(delivery.AssignerAsksBucket, key) {
return
}
words := p.heldHold(module, fp, nil).Words
a := assign.RestoreAsk(key, node, module, fp, words, time.Now())
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
}
// answeredWith says whether the teller wrote an answer to the ask at key as it stands now and the option chosen is option. The
// answer is the operator's, given in the view or on the phone, and stays under its key (ADR 0539 §14, §16, §19); it counts only
// for the asking it answered (its warrant names that asking's id), so an answer to an ask taken away, or to an earlier hold of
// the same fingerprint, does not carry over to a later one (issue 1906).
func (p *passer) answeredWith(key, option string) bool {
a, ok := p.w.answer(key)
if !ok || a.Option != option {
return false
}
ask, err := delivery.DecodeAssignerAsk(key, p.w.value(delivery.AssignerAsksBucket, key))
if err != nil {
return false
}
id, _, _ := strings.Cut(a.Warrant, "@")
return id == ask.Ask.ID
}
// clearHeld clears, once, the condition of a hold the operator released.
func (p *passer) clearHeld(module, fp string) {
c := assign.HeldCondition(module, fp, "", "", false, false)
if p.say == nil || p.said[c.Key+".cleared"] {
return
}
if err := p.say(c); err != nil {
p.logf("the condition of the released %s could not be cleared, and is cleared at the next pass: %v", module, err)
return
}
p.said[c.Key+".cleared"] = true
}
// recordAsk writes, once, the ask about a move of a module of policy record that waits for the operator's yes.
func (p *passer) recordAsk(ctx context.Context, module, fp string, need assign.RecordNeed) {
key, err := delivery.RecordAskKey(need.Node, module, need.Digest)
if err != nil || p.w.has(delivery.AssignerAsksBucket, key) {
return
}
a := assign.RecordAsk(key, need.Node, module, fp, need, time.Now())
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
}
// backups sees to the modules held on a node because the backup before an apply failed (the have's phase held, ADR 0539 §16):
// it asks whether to back up again or go on without a backup, raises the condition naming node, module and the backup's words,
// and writes the answer into the node's want for the module, which the node agent reads from it and from nothing else.
//
// The ask is bound to the composition (it names the digest, and a new revision at the same fingerprint is asked again under a
// new number) and an answer counts only if the router's warrant names the id of the ask now at the key (the warrant's first part,
// before the `@`). Each answer is another want value (the asking's number is in the want's words), so a backup that fails again
// after "back up again" and is answered "again" again changes the key and the node retries. That the backup failed again is seen
// in a new have value after the answer was written.
func (p *passer) backups(ctx context.Context) {
wanted := p.w.wantsOf()
for key, value := range p.w.read(delivery.HaveBucket) {
node, module, isNode, err := delivery.ParseHaveKey(key)
if err != nil || isNode {
continue
}
have, err := delivery.DecodeHave(value)
if err != nil || have.Phase != delivery.PhaseHeld {
continue
}
want, ok := wanted[module][node]
if !ok || want.Removed {
continue
}
akey, err := delivery.BackupAskKey(node, module, want.FP)
if err != nil {
continue
}
current, there := delivery.DecodeAssignerAsk(akey, p.w.read(delivery.AssignerAsksBucket)[akey])
if there != nil || assign.BackupAskDigest(current) != want.Composed {
after := -1
if there == nil {
after = current.Asking
}
p.backupAsk(ctx, akey, node, module, want, have.Detail, assign.NextBackupAsking(after))
continue
}
answer, answered := p.w.answer(akey)
if router, _, _ := strings.Cut(answer.Warrant, "@"); !answered || router != current.Ask.ID ||
(answer.Option != delivery.BackupAgain && answer.Option != delivery.BackupSkip) {
continue // not answered yet: the ask stands
}
words := assign.BackupAnswerWords(answer.Option, current.Asking)
haveChanges := p.w.changed(delivery.HaveBucket, key)
if want.Backup != answer.Option || want.Words != words {
want.Backup, want.Words = answer.Option, words
if wkey, err := delivery.WantKey(node, module); err == nil {
w := want
p.put(ctx, delivery.AssignerWantsBucket, wkey, func() ([]byte, error) { return delivery.EncodeWant(w) })
p.answeredAt[current.Ask.ID] = haveChanges
}
continue
}
if answer.Option != delivery.BackupAgain {
continue
}
// The answer is in the want. The node retried and the backup failed again when the have took another value since.
if seen, known := p.answeredAt[current.Ask.ID]; !known {
p.answeredAt[current.Ask.ID] = haveChanges // after a restart: wait for the next failure
} else if haveChanges > seen {
p.backupAsk(ctx, akey, node, module, want, have.Detail, assign.NextBackupAsking(current.Asking))
}
}
}
// backupAsk writes the backup ask at its asking and raises the condition of the failed backup, once.
func (p *passer) backupAsk(ctx context.Context, key, node, module string, want delivery.Want, words string, asking int) {
a := assign.BackupAsk(key, node, module, want.FP, want.Composed, words, asking, time.Now())
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
c := assign.BackupCondition(node, module, want.FP, words, true)
if p.say == nil || p.said[c.Key] {
return
}
if err := p.say(c); err != nil {
p.logf("the condition of the failed backup of %s on %s could not be said: %v", module, node, err)
return
}
p.said[c.Key] = true
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go
-472
View File
@@ -1,472 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provider"
"mesh-delivery-assigner/internal/assign"
)
// memKV is the part of a bucket a pass writes: puts and deletes.
type memKV struct {
jetstream.KeyValue
values map[string][]byte
}
func (m *memKV) Put(_ context.Context, key string, value []byte) (uint64, error) {
m.values[key] = value
return uint64(len(m.values)), nil
}
func (m *memKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error {
delete(m.values, key)
return nil
}
// newPasser is a pass on a fresh world, as after a start, writing to the buckets given, at the clock given.
func newPasser(kvs buckets, at time.Time) *passer {
return &passer{kvs: kvs, w: newWorld(), logf: func(string, ...any) {}, said: map[string]bool{}, answeredAt: map[string]uint64{},
now: func() time.Time { return at }}
}
var fpTest = delivery.SourcePrefix2 + strings.Repeat("b", 64)
// 1906/H2: given a hold lifted by itself, when the assigner is started again and the same fingerprint is held again, then the
// hold ask is written under another asking, with an id of its own, so the teller raises it (the router drops a second ask under
// an id it keeps).
func TestAHoldAskedAgainAfterALiftAndARestartHasANewID(t *testing.T) {
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
kvs := buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}
key, _ := delivery.HoldAskKey("web", fpTest)
held := &delivery.Held{Node: "ace", Words: "health failed"}
first := newPasser(kvs, time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC))
first.holdAsk(context.Background(), "web", fpTest, held)
a1, err := delivery.DecodeAssignerAsk(key, asksKV.values[key])
if err != nil {
t.Fatal(err)
}
first.lift(context.Background(), "web", fpTest)
if _, there := asksKV.values[key]; there {
t.Fatal("the lift left the hold ask")
}
again := newPasser(kvs, time.Date(2026, 10, 25, 2, 0, 0, 0, time.UTC)) // started again: it remembers nothing
again.holdAsk(context.Background(), "web", fpTest, held)
a2, err := delivery.DecodeAssignerAsk(key, asksKV.values[key])
if err != nil {
t.Fatal(err)
}
if a2.Ask.ID == a1.Ask.ID {
t.Errorf("the ask written again after a lift and a restart has the id %s of the first", a1.Ask.ID)
}
}
// 1906/H4: an answer counts only for the asking it answered: an answer to an earlier hold of the same fingerprint, or to an ask
// taken away, is not taken for the hold that stands now.
func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
key, _ := delivery.HoldAskKey("web", fpTest)
p := newPasser(buckets{}, time.Now())
put := func(asking int) string {
a := assign.HoldAsk(key, "web", fpTest, "ace", "health failed", asking, time.Now())
v, err := delivery.EncodeAssignerAsk(key, a)
if err != nil {
t.Fatal(err)
}
p.w.put(delivery.AssignerAsksBucket, key, v)
return a.Ask.ID
}
first := put(100)
v, _ := delivery.EncodeAnswer(delivery.Answer{Option: "release", Warrant: first + "@2026-10-25T01:00:00Z", Words: "the operator"})
p.w.put(delivery.AnswersBucket, key, v)
if !p.answeredWith(key, "release") {
t.Fatal("the answer to the ask that stands is not taken")
}
if p.answeredWith(key, "keep") {
t.Error("an answer was taken for an option not chosen")
}
put(200) // the same fingerprint held again, after a lift
if p.answeredWith(key, "release") {
t.Error("the answer to an earlier hold carried over to a later one")
}
p.w.put(delivery.AssignerAsksBucket, key, nil) // the ask taken away
if p.answeredWith(key, "release") {
t.Error("an answer to an ask taken away was taken")
}
t.Run("a put of the bus acts on the operator's warrant alone", putOfTheBus)
}
// memEntry is a value a memKV holds, as a bucket's Get answers it.
type memEntry struct {
jetstream.KeyValueEntry
value []byte
revision uint64
}
func (e memEntry) Value() []byte { return e.value }
func (e memEntry) Revision() uint64 { return e.revision }
func (m *memKV) Get(_ context.Context, key string) (jetstream.KeyValueEntry, error) {
v, ok := m.values[key]
if !ok {
return nil, jetstream.ErrKeyNotFound
}
return memEntry{value: v, revision: uint64(len(v))}, nil
}
func (m *memKV) Update(_ context.Context, key string, value []byte, revision uint64) (uint64, error) {
if uint64(len(m.values[key])) != revision {
return 0, errors.New("wrong last sequence")
}
m.values[key] = value
return uint64(len(value)), nil
}
// memStore is the runtime's states for the put's tools, over the same memKVs a pass writes.
type memStore map[string]*memKV
func (s memStore) get(state, key string) ([]byte, uint64, bool, error) {
v, ok := s[state].values[key]
return v, uint64(len(v)), ok, nil
}
func (s memStore) put(state, key string, value []byte) error {
s[state].values[key] = value
return nil
}
func (s memStore) keys(state string) ([]string, error) {
var out []string
for k := range s[state].values {
out = append(out, k)
}
return out, nil
}
// fakeBus is the assigner's process account in a test: every call recorded, each answered at once.
type fakeBus struct {
calls chan string
events []string
}
func (b *fakeBus) Call(key string, body any) (json.RawMessage, error) {
b.calls <- fmt.Sprintf("%s %v", key, body)
return json.RawMessage(`{"ok":true,"output":""}`), nil
}
func (b *fakeBus) CallOn(node, key string, body any) (json.RawMessage, error) {
return b.Call(node+"/"+key, body)
}
func (b *fakeBus) Emit(eventType string, body any) error {
b.events = append(b.events, fmt.Sprintf("%s %+v", eventType, body))
return nil
}
// ADR 0546, the first put (2026-10-25, want.novox.nats): the tool writes the put; a pass waits for the composition and grant
// the controller's take-in writes; once they are there it asks the operator with a yes that approves; only on that warrant does it have
// the streams snapshotted, then writes the want held and records the warrant; the have running it healthy holds it; a release
// gives the want back to the rules. Each step is said as the event put.
func putOfTheBus(t *testing.T) {
ctx := context.Background()
now := time.Date(2026, 10, 25, 22, 0, 0, 0, time.UTC)
fp := delivery.SourcePrefix2 + "fd497ab8" + strings.Repeat("0", 56)
digest := "sha256:" + strings.Repeat("d", 64)
mem := func() *memKV { return &memKV{values: map[string][]byte{}} }
puts, asksKV, wantsKV := mem(), mem(), mem()
kvs := buckets{delivery.AssignerPutsBucket: puts, delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}
p := newPasser(kvs, now)
bus := &fakeBus{calls: make(chan string, 8)}
woke := make(chan struct{}, 8) // each act's goroutine wakes a pass once it is done
p.acts = newPutActs(bus, func() { woke <- struct{}{} }, func(string, ...any) {})
p.acts.snapshot = func(context.Context, string, string) (string, error) {
return "novox's restore point of nats (streams) taken 22:01", nil
}
// The world: nats assigned to novox, built at fp, the bus by its manifest, novox running it from before the switch.
world := func(bucket, key string, v any) {
var data []byte
switch x := v.(type) {
case []byte:
data = x
default:
data, _ = json.Marshal(x)
}
p.w.put(bucket, key, data)
}
world(provider.AssignmentsBucket, "novox.nats", map[string]string{"node": "novox", "module": "nats"})
mdigest := strings.Repeat("e", 64)
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: mdigest, Repository: "novox/mesh-catalog", Commit: strings.Repeat("c", 40)})
world(delivery.ModulesBucket, "nats", mod)
p.w.manifests[mdigest] = parsed{module: "nats", manifest: catalogue.Manifest{Module: "nats", Provides: []catalogue.Offer{{Name: "mesh-bus"}}}}
bkey, _ := delivery.BuilderBuildKey("nats", fp)
built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: mdigest})
world(delivery.BuilderBuildsBucket, bkey, built)
hkey, _ := delivery.HaveKey("novox", "nats")
before, _ := delivery.EncodeHave(delivery.Have{FP: delivery.SourcePrefix2 + strings.Repeat("a", 64), Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
world(delivery.HaveBucket, hkey, before)
builds := mem()
builds.values[bkey] = built
modules := mem()
modules.values["nats"] = mod
s := newPutTools(memStore{putsState: puts, delivery.BuilderModule + ".builds": builds, delivery.IndexerModule + ".modules": modules})
s.now = func() time.Time { return now }
was := answerWithin
answerWithin = 0
defer func() { answerWithin = was }()
if _, err := s.doPut(map[string]any{"node": "novox", "module": "nats", "version": "fd497ab8ffff", "why": "x"}, "shanks.node-tools"); err == nil {
t.Error("a prefix of a fingerprint the builder never built was taken")
}
if got, err := s.version("nats", "fd497ab80000"); err != nil || got != fp {
t.Errorf("the prefix of the fingerprint built reads %q, %v", got, err)
}
if got, err := s.version("nats", "newest"); err != nil || got != fp {
t.Errorf("newest reads %q, %v", got, err)
}
ans, err := s.doPut(map[string]any{"node": "novox", "module": "nats", "version": fp, "hold": "true", "why": "the bus's user list"}, "shanks.node-tools")
if err != nil {
t.Fatal(err)
}
if a := ans.(putAnswer); a.Step != delivery.PutAsked || !a.Hold || !strings.Contains(a.Follow, "mesh_wait") {
t.Fatalf("put answered %+v", a)
}
pkey, _ := delivery.PutKey("novox", "nats")
step := func() delivery.Put {
t.Helper()
world(delivery.AssignerPutsBucket, pkey, puts.values[pkey])
p.pass(ctx)
got, err := delivery.DecodePut(puts.values[pkey])
if err != nil {
t.Fatal(err)
}
return got
}
if got := step(); got.Step != delivery.PutCompose || !strings.Contains(got.Words, "no composition") {
t.Fatalf("the first pass left the put at %s: %s", got.Step, got.Words)
}
select {
case call := <-bus.calls:
t.Errorf("a put asked %q: it reads the compositions and asks nobody for one", call)
default:
}
ckey, _ := delivery.ComposedKey("novox", "nats", fp)
cd, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digest, Judgement: delivery.JudgedCannot, Words: "the have names no composition"})
world(delivery.ControllerComposedBucket, ckey, cd)
gkey, _ := delivery.ControllerGrantKey("nats", fp)
g, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
world(delivery.ControllerGrantsBucket, gkey, g)
got := step()
akey, _ := delivery.RecordAskKey("novox", "nats", digest)
if got.Step != delivery.PutPerson || got.Ask != akey {
t.Fatalf("composed, the put stands at %s (%s), ask %q", got.Step, got.Words, got.Ask)
}
a, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
if err != nil || a.Ask.Options[0].Level != asks.Approve {
t.Fatalf("the ask is %+v, %v; want a yes that approves", a, err)
}
if _, there := wantsKV.values["want.novox.nats"]; there {
t.Fatal("the bus's want was written before the operator's warrant")
}
world(delivery.AssignerAsksBucket, akey, asksKV.values[akey])
yes, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a.Ask.ID + "@2026-10-25T22:01:00Z", Words: "the operator, verified"})
world(delivery.AnswersBucket, akey, yes)
if got := step(); got.Step != delivery.PutSnapshot {
t.Fatalf("on the warrant the put stands at %s: %s", got.Step, got.Words)
}
<-woke
got = step()
w, err := delivery.DecodeWant(wantsKV.values["want.novox.nats"])
if got.Step != delivery.PutWanted || err != nil || w.FP != fp || w.Composed != digest || !w.Put || !w.Hold {
t.Fatalf("snapshotted, the put stands at %s (%s); the want %+v, %v", got.Step, got.Words, w, err)
}
if call := <-bus.calls; !strings.Contains(call, "warranted") || !strings.Contains(call, a.Ask.ID) {
t.Errorf("the warrant was recorded as %q", call)
}
world(delivery.AssignerWantsBucket, "want.novox.nats", wantsKV.values["want.novox.nats"])
running, _ := delivery.EncodeHave(delivery.Have{FP: fp, Composed: digest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
world(delivery.HaveBucket, hkey, running)
if got := step(); got.Step != delivery.PutHolding {
t.Fatalf("running healthy, the put stands at %s: %s", got.Step, got.Words)
}
if _, err := s.doRelease(map[string]any{"node": "novox", "module": "nats", "why": "done"}, "shanks.node-tools"); err != nil {
t.Fatal(err)
}
if got := step(); got.Step != delivery.PutReleased {
t.Fatalf("released, the put stands at %s", got.Step)
}
listed, _ := s.doPuts(map[string]any{"module": "nats"})
if l := listed.(map[string]any)["puts"].([]putAnswer); len(l) != 1 || l[0].Step != delivery.PutReleased || len(l[0].Steps) < 6 {
t.Errorf("puts lists %+v", l)
}
if len(bus.events) < 5 || !strings.Contains(strings.Join(bus.events, "\n"), "holding") {
t.Errorf("the steps said as events: %v", bus.events)
}
// The same bus put again at the same composition after the release is its own asking: its ask is written again under an
// id of its own, and the earlier yes, still at the answers' key, does not count for it (ADR 0546 clause 8).
world(delivery.HaveBucket, hkey, before)
s.now = func() time.Time { return now.Add(time.Minute) }
if _, err := s.doPut(map[string]any{"node": "novox", "module": "nats", "version": fp, "hold": "true", "why": "again"}, "shanks.node-tools"); err != nil {
t.Fatal(err)
}
again := step()
a2, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
if again.Step != delivery.PutPerson || err != nil || a2.Ask.ID == a.Ask.ID {
t.Fatalf("put again, it stands at %s (%s); its ask %q, %v; the first ask was %q", again.Step, again.Words, a2.Ask.ID, err, a.Ask.ID)
}
world(delivery.AssignerAsksBucket, akey, asksKV.values[akey])
yes2, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a2.Ask.ID + "@2026-10-25T22:03:00Z", Words: "the operator, verified"})
world(delivery.AnswersBucket, akey, yes2)
if got := step(); got.Step != delivery.PutSnapshot {
t.Fatalf("on the second warrant the put stands at %s: %s", got.Step, got.Words)
}
<-woke
// Released between the pass's read and its write: the pass writes no want for the put that no longer stands.
oldWant, _ := delivery.EncodeWant(delivery.Want{FP: delivery.SourcePrefix2 + strings.Repeat("a", 64)})
wantsKV.values["want.novox.nats"] = oldWant
world(delivery.AssignerWantsBucket, "want.novox.nats", oldWant)
if _, err := s.doRelease(map[string]any{"node": "novox", "module": "nats", "why": "changed my mind"}, "shanks.node-tools"); err != nil {
t.Fatal(err)
}
p.pass(ctx)
if string(wantsKV.values["want.novox.nats"]) != string(oldWant) {
t.Errorf("a want was written for a put released meanwhile: %s", wantsKV.values["want.novox.nats"])
}
}
// 1945/AC21: given a module on neither list on a node, when passes follow, then the assigner raises its condition once, says it
// again after a say that failed, and clears it once when the module has a want again; a module on neither list is also written to
// the journal, so it is loud with no operator channel.
func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
p := newPasser(buckets{}, time.Date(2026, 10, 25, 15, 0, 0, 0, time.UTC))
var said []asks.Condition
failing := true
p.say = func(c asks.Condition) error {
if failing {
failing = false
return errors.New("the channel is away")
}
said = append(said, c)
return nil
}
var logged []string
p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
one := map[string]neitherOne{"g14.screen-lock": {node: "g14", module: "screen-lock",
holding: assign.Holding{Rule: delivery.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}}}
p.sayNeither(one, nil) // the say fails
p.sayNeither(one, nil)
p.sayNeither(one, nil)
if len(said) != 1 || said[0].State != asks.StateRaised || said[0].Subject.Machine != "g14" || said[0].Subject.ID != "screen-lock" {
t.Fatalf("said = %+v, want one raised condition of screen-lock on g14", said)
}
if !strings.Contains(strings.Join(logged, "\n"), "screen-lock") {
t.Errorf("the journal does not name the module on neither list: %q", logged)
}
p.sayNeither(map[string]neitherOne{}, nil)
p.sayNeither(map[string]neitherOne{}, nil)
if len(said) != 2 || said[1].State != asks.StateCleared || said[1].Key != said[0].Key {
t.Errorf("said = %+v, want the condition cleared once under its key", said)
}
}
// 1945/AC21: the pass gathers the modules on neither list of every module's outcome, keyed by node and module.
func TestThePassGathersTheModulesOnNeitherList(t *testing.T) {
got := map[string]neitherOne{}
gatherNeither(got, "screen-lock", assign.Outcome{Neither: map[string]assign.Holding{"g14": {Rule: delivery.RuleGrant, Words: "w"}}})
gatherNeither(got, "rofi", assign.Outcome{})
if len(got) != 1 || got["g14.screen-lock"].node != "g14" || got["g14.screen-lock"].module != "screen-lock" || got["g14.screen-lock"].holding.Rule != delivery.RuleGrant {
t.Errorf("gathered = %+v", got)
}
}
// A neither or held condition a previous process raised is cleared by the first pass after a start: a fresh pass remembers
// nothing it said, so it clears, once, the neither condition of every assigned module not on neither list and the held condition of
// every module that is not held. Clearing a condition that is not open is quiet at the router.
func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) {
wantsKV := &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerWantsBucket: wantsKV, delivery.AssignerAsksBucket: &memKV{values: map[string][]byte{}}}, time.Now())
var said []asks.Condition
p.say = func(c asks.Condition) error { said = append(said, c); return nil }
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
p.w.put(delivery.ModulesBucket, "web", mod)
p.w.put(provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-1"}`))
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
p.w.put(delivery.PresenceBucket, "g14", on)
have, _ := delivery.EncodeHave(delivery.Have{FP: fpTest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
p.w.put(delivery.HaveBucket, "g14.web", have)
key, _ := delivery.WantKey("g14", "web")
want, _ := delivery.EncodeWant(delivery.Want{FP: fpTest, Composed: "sha256:" + strings.Repeat("c", 64)})
p.w.put(delivery.AssignerWantsBucket, key, want)
p.wants(context.Background())
cleared := map[string]bool{}
for _, c := range said {
if c.State != asks.StateCleared {
t.Errorf("the first pass raised %+v", c)
}
cleared[c.Key] = true
}
neither := assign.NeitherCondition("g14", "web", assign.Holding{}, false).Key
held := assign.HeldCondition("web", fpTest, "", "", false, false).Key
if !cleared[neither] || !cleared[held] {
t.Errorf("cleared = %v, want %s and %s", cleared, neither, held)
}
n := len(said)
p.wants(context.Background())
if len(said) != n {
t.Errorf("a second pass said again: %+v", said[n:])
}
}
// 1945/AC24: given a module held at F under rule 2 on a node that runs it with no want, when the assigner passes, then it raises no
// neither condition for it, a neither condition this process raised for it before is cleared, and so is one a previous process
// raised (the first pass after a start), each once.
func TestAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
setup := func() (*passer, *[]asks.Condition) {
p := newPasser(buckets{delivery.AssignerWantsBucket: &memKV{values: map[string][]byte{}}, delivery.AssignerAsksBucket: &memKV{values: map[string][]byte{}}}, time.Now())
said := &[]asks.Condition{}
p.say = func(c asks.Condition) error { *said = append(*said, c); return nil }
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/resolved", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
p.w.put(delivery.ModulesBucket, "systemd-resolved", mod)
p.w.put(provider.AssignmentsBucket, "shanks.systemd-resolved", []byte(`{"node":"shanks","module":"systemd-resolved","build":"b-1"}`))
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
p.w.put(delivery.PresenceBucket, "shanks", on)
have, _ := delivery.EncodeHave(delivery.Have{FP: fpTest, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed})
p.w.put(delivery.HaveBucket, "shanks.systemd-resolved", have)
hkey, _ := delivery.HeldKey("systemd-resolved", fpTest)
held, _ := delivery.EncodeHeld(delivery.Held{Node: "shanks", Words: "health failed"})
p.w.put(delivery.AssignerWantsBucket, hkey, held)
return p, said
}
key := assign.NeitherCondition("shanks", "systemd-resolved", assign.Holding{}, false).Key
neithers := func(said []asks.Condition) (raised, cleared int) {
for _, c := range said {
if c.Key != key {
continue
}
if c.State == asks.StateRaised {
raised++
} else {
cleared++
}
}
return
}
// a standing neither this process raised
p, said := setup()
p.neither = map[string]neitherOne{"shanks.systemd-resolved": {node: "shanks", module: "systemd-resolved", holding: assign.Holding{Rule: delivery.RuleNotHeld}}}
p.wants(context.Background())
p.wants(context.Background())
if r, c := neithers(*said); r != 0 || c != 1 {
t.Errorf("standing in this process: %d raised, %d cleared, want 0 and 1: %+v", r, c, *said)
}
// one a previous process raised: the first pass after a start clears it
p, said = setup()
p.wants(context.Background())
p.wants(context.Background())
if r, c := neithers(*said); r != 0 || c != 1 {
t.Errorf("after a start: %d raised, %d cleared, want 0 and 1: %+v", r, c, *said)
}
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go
-449
View File
@@ -1,449 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-assigner/internal/assign"
)
// The puts (novox/hq ADR 0546): the tool `assigner_put` writes the key `<node>.<module>` of the assigner's puts, and each pass
// takes every put that has not ended one step further (assign.StepPut), doing the acts the step asks for: the put's ask, the
// bus's snapshot, the warranted record. Each act's outcome is heard back as a change of the world (the composition, the grant,
// the answer) or, for the calls, as a wake once they answer; nothing polls but the snapshot's read-back of the backup holder,
// as the controller's bus step reads it (ADR 0235), bounded. Every step is said as the event mesh-delivery-assigner.put, so a
// caller waits on it.
// PutEvent is the event a put's every step is said as: mesh.mod.mesh-delivery-assigner.event.put.
const PutEvent = "put"
// caller is what the puts need of the bus as the assigner's own process account: a call of a seat's verb, on one node or on
// the seat held once, and an event said. process.Conn, or a test's.
type caller interface {
Call(key string, body any) (json.RawMessage, error)
CallOn(node, key string, body any) (json.RawMessage, error)
Emit(eventType string, body any) error
}
// putActs keeps, by put (its key and when it was asked), what the acts of a put came to while this process runs: a restart
// takes the snapshot again, without harm.
type putActs struct {
mu sync.Mutex
bus caller
wake func()
logf func(string, ...any)
snapping map[string]bool
snapped map[string]string
snapNo map[string]string
warranted map[string]bool
warrantNo map[string]string
said map[string]string
// snapshot takes the bus's snapshot on node; a variable so a test takes none.
snapshot func(ctx context.Context, node, module string) (string, error)
}
func newPutActs(bus caller, wake func(), logf func(string, ...any)) *putActs {
a := &putActs{bus: bus, wake: wake, logf: logf, snapping: map[string]bool{}, snapped: map[string]string{}, snapNo: map[string]string{}, warranted: map[string]bool{}, warrantNo: map[string]string{},
said: map[string]string{}}
a.snapshot = a.snapshotTheBus
return a
}
// identity names one put: its key and when it was asked, so a put asked again under the key is another.
func identity(key string, p delivery.Put) string {
return key + "@" + p.At.UTC().Format(time.RFC3339Nano)
}
// putsByModule is every put the assigner holds, by module then node.
func (p *passer) putsByModule() map[string]map[string]delivery.Put {
out := map[string]map[string]delivery.Put{}
for key, value := range p.w.read(delivery.AssignerPutsBucket) {
node, module, err := delivery.ParsePutKey(key)
if err != nil {
continue
}
put, err := delivery.DecodePut(value)
if err != nil {
p.logf("the put %s cannot be read, and is left: %v", key, err)
continue
}
if out[module] == nil {
out[module] = map[string]delivery.Put{}
}
out[module][node] = put
}
return out
}
// puts takes every put that has not ended one step further.
func (p *passer) puts(ctx context.Context) {
if p.acts == nil {
return
}
all := p.putsByModule()
if len(all) == 0 {
return
}
assigned, _ := p.w.assigned()
trunk := p.w.trunk()
get := kvGetter(p.kvs[delivery.ManifestsBucket])
wanted := p.w.wantsOf()
modules := make([]string, 0, len(all))
for m := range all {
modules = append(modules, m)
}
sort.Strings(modules)
for _, module := range modules {
var manifest catalogue.Manifest
if mod, ok := trunk[module]; ok && mod.Manifest != "" {
if got := p.w.manifest(ctx, get, mod.Manifest); got.err == "" {
manifest = got.manifest
}
}
for node, put := range all[module] {
if delivery.PutEnded(put.Step) {
continue
}
key, _ := delivery.PutKey(node, module)
id := identity(key, put)
world := p.putWorld(node, module, put, manifest, assigned[node])
world.Snapshot, world.Snapshotting, world.SnapshotFailed = p.acts.state(id)
next := assign.StepPut(world)
// A held bus whose user list the rules moved on (ADR 0539 §7): the put follows the composition its want names.
if w, ok := wanted[module][node]; ok && w.Put && w.FP == next.Put.FP && w.Composed != "" && w.Composed != next.Put.Composed &&
(next.Put.Step == delivery.PutWanted || next.Put.Step == delivery.PutHolding) {
if d, _ := p.composedFor(node, module, w.FP); d != nil && d.UserListOnly && d.Digest == w.Composed {
next.Put = next.Put.Said(world.Now, next.Put.Step, fmt.Sprintf("the bus's user list changed: the want names composition %s", shortHex(w.Composed)))
next.Put.Composed = w.Composed
}
}
// The bus's want is written only for the act the operator's yes bound (ADR 0546 clause 8).
if next.Put.Step == delivery.PutWanted && put.Step != delivery.PutWanted && world.Bus && next.Put.Ask != "" {
if err := p.performs(next.Put.Ask, assign.PutAct(node, module, next.Put.FP, next.Put.Composed)); err != nil {
next = assign.PutNext{Put: next.Put.Said(world.Now, delivery.PutRefused, "the bus's want is not written: "+err.Error())}
}
}
if failed := p.acts.warrantFailed(id); failed != "" && (next.Put.Step == delivery.PutWanted || next.Put.Step == delivery.PutHolding) &&
!strings.Contains(next.Put.Words, failed) {
next.Put = next.Put.Said(world.Now, next.Put.Step, next.Put.Words+"; "+failed)
}
p.actOn(ctx, node, module, id, world, next)
written := samePut(put, next.Put) || p.writePut(ctx, key, put, next.Put)
if !written {
continue // released or put again meanwhile: nothing is done for the put that no longer stands
}
if next.Put.Step == delivery.PutWanted && put.Step != delivery.PutWanted {
// The want is written in this pass too, before the next one would: Module.Puts writes the same.
if wkey, err := delivery.WantKey(node, module); err == nil {
want := assign.PutWant(next.Put)
if before, ok := wanted[module][node]; !ok || !delivery.SameWant(normalWant(before), normalWant(want)) {
p.put(ctx, delivery.AssignerWantsBucket, wkey, func() ([]byte, error) { return delivery.EncodeWant(want) })
}
}
if next.Warranted {
p.acts.warrant(id, delivery.AskingID(next.Put.Ask, assign.PutAsking(next.Put)))
}
}
}
}
}
func normalWant(w delivery.Want) delivery.Want {
w.V = 0
return w
}
// putWorld is what one step of a put reads from the world.
func (p *passer) putWorld(node, module string, put delivery.Put, manifest catalogue.Manifest, assignedHere []string) assign.PutWorld {
w := assign.PutWorld{Node: node, Module: module, Put: put, Now: p.clock(), Bus: catalogue.ProvidesBus(manifest), Newest: p.w.trunk()[module].FP2}
for _, m := range assignedHere {
if m == module {
w.Assigned = true
}
}
if bkey, err := delivery.BuilderBuildKey(module, put.FP); err == nil {
if raw := p.w.value(delivery.BuilderBuildsBucket, bkey); raw != nil {
if b, err := delivery.DecodeBuilderBuild(raw); err == nil {
w.Build, w.BuildWords = b.State, b.Error
} else {
w.Build = "unreadable: " + err.Error()
}
}
}
w.Composed, w.Granted = p.composedFor(node, module, put.FP)
if d := w.Composed; d != nil {
policy := d.Policy
if policy == "" {
policy, _, _ = catalogue.DerivedUpgrade(manifest)
}
w.Recorded = policy == catalogue.PolicyRecord
if akey, err := delivery.RecordAskKey(node, module, d.Digest); err == nil && put.Ask == akey {
// Only this put's own ask counts: an ask (and its answer) of an earlier put under the same key is another asking.
if a, err := delivery.DecodeAssignerAsk(akey, p.w.value(delivery.AssignerAsksBucket, akey)); err == nil && a.Asking == assign.PutAsking(put) {
switch {
case p.answeredWith(akey, assign.OptionYes):
w.Answer = assign.OptionYes
case p.answeredWith(akey, assign.OptionNo):
w.Answer = assign.OptionNo
default:
w.AskExpired = !a.Ask.Expires.IsZero() && w.Now.After(a.Ask.Expires)
}
}
}
}
if hkey, err := delivery.HaveKey(node, module); err == nil {
if raw := p.w.value(delivery.HaveBucket, hkey); raw != nil {
if h, err := delivery.DecodeHave(raw); err == nil {
w.Have = &h
}
}
}
return w
}
// composedFor is the composition of module for node at fp and whether the grant at fp names the node's accounts: the one
// place a put reads them (ADR 0546 clause 4). Today their writer is the controller (mesh-controller_composed and
// mesh-controller_grants); when composing and grants move out of the controller into delivery's own parts, this reads them
// from their new owner and nothing else changes.
func (p *passer) composedFor(node, module, fp string) (*delivery.ComposedDigest, bool) {
var out *delivery.ComposedDigest
if d, ok := p.w.composed(node, module, fp); ok {
out = &d
}
granted := false
if gkey, err := delivery.ControllerGrantKey(module, fp); err == nil {
if raw := p.w.value(delivery.ControllerGrantsBucket, gkey); raw != nil {
if g, err := delivery.DecodeControllerGrant(raw); err == nil {
granted = g.NamesNode(node, module)
}
}
}
return out, granted
}
// actOn does the acts a step asks for.
func (p *passer) actOn(ctx context.Context, node, module, id string, w assign.PutWorld, next assign.PutNext) {
if next.Put.Step == delivery.PutPerson && next.Put.Ask != "" && w.Composed != nil {
// The put's own ask, written again over one an earlier put left under the key (its asking differs), so the teller
// raises it anew under its own id.
akey := next.Put.Ask
current, err := delivery.DecodeAssignerAsk(akey, p.w.value(delivery.AssignerAsksBucket, akey))
if err != nil || current.Asking != assign.PutAsking(next.Put) {
a, err := assign.PutAsk(akey, node, module, next.Put, *w.Composed, w.Bus, w.Now)
if err != nil {
p.logf("the ask about putting %s on %s is not written: %v", module, node, err)
} else {
p.put(ctx, delivery.AssignerAsksBucket, akey, func() ([]byte, error) { return delivery.EncodeAssignerAsk(akey, a) })
}
}
}
if next.Snapshot {
p.acts.snap(ctx, id, node, module)
}
}
// performs checks that the act about to be performed is the one the yes of the ask at key bound (asks.Option.Performs).
func (p *passer) performs(key string, act asks.Act) error {
a, err := delivery.DecodeAssignerAsk(key, p.w.value(delivery.AssignerAsksBucket, key))
if err != nil {
return fmt.Errorf("the ask %s cannot be read: %w", key, err)
}
for _, o := range a.Ask.Options {
if o.ID == assign.OptionYes {
return o.Performs(act)
}
}
return fmt.Errorf("the ask %s offers no yes", key)
}
// samePut says two puts say the same: their step, words and what they found.
func samePut(a, b delivery.Put) bool {
return a.Step == b.Step && a.Words == b.Words && a.Composed == b.Composed && a.Ask == b.Ask && a.Snapshot == b.Snapshot
}
// writePut writes a put's next value over the one the pass read, by compare-and-set, unless the tool has put again or
// released it since; and says it as the event put. It says whether it wrote.
func (p *passer) writePut(ctx context.Context, key string, read, next delivery.Put) bool {
kv := p.kvs[delivery.AssignerPutsBucket]
if kv == nil {
return false
}
e, err := kv.Get(ctx, key)
if err != nil {
p.logf("the put %s cannot be read again, and is taken further at the next pass: %v", key, err)
return false
}
now, err := delivery.DecodePut(e.Value())
if err != nil || !now.At.Equal(read.At) || now.Step != read.Step {
return false // asked again or released meanwhile: the next pass reads that
}
data, err := delivery.EncodePut(next)
if err != nil {
p.logf("the put %s is not written: %v", key, err)
return false
}
if _, err := kv.Update(ctx, key, data, e.Revision()); err != nil {
if !errors.Is(err, context.Canceled) {
p.logf("the put %s could not be written, and is taken further at the next pass: %v", key, err)
}
return false
}
p.w.put(delivery.AssignerPutsBucket, key, data)
p.logf("put %s: %s: %s", key, next.Step, next.Words)
p.acts.say(key, next)
return true
}
// state is what this process knows of a put's acts.
func (a *putActs) state(id string) (snapshot string, snapping bool, snapNo string) {
a.mu.Lock()
defer a.mu.Unlock()
return a.snapped[id], a.snapping[id], a.snapNo[id]
}
// snap has the bus's snapshot taken once for a put, in the background.
func (a *putActs) snap(ctx context.Context, id, node, module string) {
a.mu.Lock()
if a.snapping[id] || a.snapped[id] != "" || a.snapNo[id] != "" {
a.mu.Unlock()
return
}
a.snapping[id] = true
a.mu.Unlock()
go func() {
defer a.wake()
where, err := a.snapshot(ctx, node, module)
a.mu.Lock()
defer a.mu.Unlock()
a.snapping[id] = false
if err != nil {
a.snapNo[id] = err.Error()
return
}
a.snapped[id] = where
}()
}
// SnapshotWithin is how long the bus node's backup holder is given to take the bus's snapshot (as the controller's bus step).
var SnapshotWithin = 15 * time.Minute
// snapshotLook is how often the backup holder's record is read back while the snapshot is taken.
var snapshotLook = 10 * time.Second
// backedUp is one module as the backup holder's `backed-up` answers it.
type backedUp struct {
Module string `json:"module"`
Data []struct {
Item string `json:"item"`
LastBackup *time.Time `json:"last_backup"`
Error string `json:"error"`
} `json:"data"`
}
// snapshotTheBus asks the bus node's backup holder to back the bus module up now, whose dump is the streams' snapshot
// (ADR 0235), and reads its record back until it names a backup newer than the ask: where the snapshot is.
func (a *putActs) snapshotTheBus(ctx context.Context, node, module string) (string, error) {
asked := time.Now()
if _, err := a.bus.CallOn(node, "seat:"+catalogue.BackupSeat+".now", map[string]any{"module": module}); err != nil {
return "", fmt.Errorf("%s's backup holder was not asked to take the bus's snapshot: %w", node, err)
}
deadline := asked.Add(SnapshotWithin)
for {
if raw, err := a.bus.CallOn(node, "seat:"+catalogue.BackupSeat+".backed-up", map[string]any{}); err == nil {
var modules []backedUp
if json.Unmarshal(raw, &modules) == nil {
for _, m := range modules {
for _, it := range m.Data {
if m.Module == module && it.LastBackup != nil && it.LastBackup.After(asked) && it.Error == "" {
return fmt.Sprintf("%s's restore point of %s (%s) taken %s", node, module, it.Item, it.LastBackup.UTC().Format(time.RFC3339)), nil
}
}
}
}
}
if time.Now().After(deadline) {
return "", fmt.Errorf("%s's backup holder did not say the bus's snapshot was taken within %s", node, SnapshotWithin)
}
select {
case <-ctx.Done():
return "", ctx.Err()
case <-time.After(snapshotLook):
}
}
}
// warrant records once, in the controller's hand-act log, that the bus's want was written on the operator's warrant: the
// router's record of the teller's ask says who chose and how (ADR 0274).
func (a *putActs) warrant(id, ask string) {
a.mu.Lock()
if a.warranted[id] {
a.mu.Unlock()
return
}
a.warranted[id] = true
a.mu.Unlock()
go func() {
if _, err := a.bus.Call("seat:"+catalogue.ControllerSeatName+".warranted", map[string]any{"asker": delivery.TellerModule, "ask": ask}); err != nil {
a.logf("the put's warrant %s is not recorded in the hand-act log: %v", ask, err)
a.mu.Lock()
a.warrantNo[id] = fmt.Sprintf("the warrant %s is not recorded in the controller's hand-act log: %v", ask, err)
a.mu.Unlock()
a.wake()
}
}()
}
// warrantFailed says why the put's warrant was not recorded, "" when it was or is not yet known.
func (a *putActs) warrantFailed(id string) string {
a.mu.Lock()
defer a.mu.Unlock()
return a.warrantNo[id]
}
// putSaid is the body of the event put.
type putSaid struct {
Node string `json:"node"`
Module string `json:"module"`
FP string `json:"fp"`
Hold bool `json:"hold"`
Step string `json:"step"`
Words string `json:"words"`
Composed string `json:"composed,omitempty"`
}
// say says a put's step as the event put, once per step and words.
func (a *putActs) say(key string, p delivery.Put) {
node, module, err := delivery.ParsePutKey(key)
if err != nil || a.bus == nil {
return
}
a.mu.Lock()
if a.said[key] == p.Step+p.Words {
a.mu.Unlock()
return
}
a.said[key] = p.Step + p.Words
a.mu.Unlock()
if err := a.bus.Emit(PutEvent, putSaid{Node: node, Module: module, FP: p.FP, Hold: p.Hold, Step: p.Step, Words: p.Words, Composed: p.Composed}); err != nil {
a.logf("the put %s's step %s is not said as an event: %v", key, p.Step, err)
}
}
// shortHex is a digest's first twelve hex digits.
func shortHex(digest string) string {
h := strings.TrimPrefix(digest, "sha256:")
if len(h) > 12 {
return h[:12]
}
return h
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/testdata/catalogue-835bf22b-xterm.module.json
Vendored
-236
View File
@@ -1,236 +0,0 @@
{
"module": "xterm",
"version": "1",
"moved": true,
"capabilities": [
"package-manager"
],
"requires": [
"x11-display"
],
"claims": [
{
"name": "node-terminal-emulator",
"scope": "node",
"serves": [
"open"
]
}
],
"tools": [
"xterm_font",
"xterm_colours"
],
"config": {
"face": {
"type": "string",
"description": "The font family xterm draws with, as fc-list names it: JetBrainsMono Nerd Font, because research 026/04 chose it for its glyphs the prompt uses. A family the machine does not have falls back to the nearest installed face (xterm_font says which).",
"pattern": "[A-Za-z0-9 ._-]{1,64}",
"default": "JetBrainsMono Nerd Font"
},
"size": {
"type": "number",
"description": "The size of xterm's text, in points: 12 reads well on a screen of about 100 DPI, and a denser screen needs a larger size.",
"minimum": 6,
"maximum": 72,
"predefined": "font-size",
"default": 12
},
"save-lines": {
"type": "number",
"description": "How many lines of scrollback xterm keeps per terminal: 10000 holds a long build log, and each line costs memory in every open terminal.",
"minimum": 0,
"maximum": 1000000,
"default": 10000
},
"term-name": {
"type": "string",
"description": "The TERM value xterm sets for the programs it runs: xterm-256color, because the palette has 256 colours and every program knows the name.",
"pattern": "[A-Za-z0-9._-]{1,64}",
"default": "xterm-256color"
},
"render-font": {
"type": "boolean",
"description": "Whether xterm draws with fontconfig faces (true) or the old X core fonts (false): true, because the face is a fontconfig family and core fonts cannot show it.",
"default": true
},
"bold-mode": {
"type": "boolean",
"description": "Whether xterm draws bold text by overstriking the normal face (true): false, because the face has a real bold style and overstriking blurs it.",
"default": false
},
"always-bold-mode": {
"type": "boolean",
"description": "Whether xterm overstrikes bold even where the face has a bold style (true): false, for the same reason as bold-mode.",
"default": false
},
"allow-bold-fonts": {
"type": "boolean",
"description": "Whether xterm may use the face's bold style for bold text (true): true, so bold text is drawn with the real bold weight.",
"default": true
},
"background": {
"type": "string",
"description": "The terminal's background colour, as #rrggbb: black, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#000000"
},
"foreground": {
"type": "string",
"description": "The terminal's text colour, as #rrggbb: a light grey, easier on the eyes than white on black.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#a8a8a8"
},
"color0": {
"type": "string",
"description": "Palette entry 0, the normal black, as #rrggbb: #000000, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#000000"
},
"color1": {
"type": "string",
"description": "Palette entry 1, the normal red, as #rrggbb: #9e1828, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#9e1828"
},
"color2": {
"type": "string",
"description": "Palette entry 2, the normal green, as #rrggbb: #008800, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#008800"
},
"color3": {
"type": "string",
"description": "Palette entry 3, the normal yellow, as #rrggbb: #d2bb4b, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#d2bb4b"
},
"color4": {
"type": "string",
"description": "Palette entry 4, the normal blue, as #rrggbb: #414171, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#414171"
},
"color5": {
"type": "string",
"description": "Palette entry 5, the normal magenta, as #rrggbb: #963c59, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#963c59"
},
"color6": {
"type": "string",
"description": "Palette entry 6, the normal cyan, as #rrggbb: #418179, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#418179"
},
"color7": {
"type": "string",
"description": "Palette entry 7, the normal white, as #rrggbb: #bebebe, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#bebebe"
},
"color8": {
"type": "string",
"description": "Palette entry 8, the bright black, as #rrggbb: #666666, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#666666"
},
"color9": {
"type": "string",
"description": "Palette entry 9, the bright red, as #rrggbb: #bc5766, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#bc5766"
},
"color10": {
"type": "string",
"description": "Palette entry 10, the bright green, as #rrggbb: #61a171, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#61a171"
},
"color11": {
"type": "string",
"description": "Palette entry 11, the bright yellow, as #rrggbb: #e7db52, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#e7db52"
},
"color12": {
"type": "string",
"description": "Palette entry 12, the bright blue, as #rrggbb: #5085af, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#5085af"
},
"color13": {
"type": "string",
"description": "Palette entry 13, the bright magenta, as #rrggbb: #a97a99, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#a97a99"
},
"color14": {
"type": "string",
"description": "Palette entry 14, the bright cyan, as #rrggbb: #6ba4a4, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#6ba4a4"
},
"color15": {
"type": "string",
"description": "Palette entry 15, the bright white, as #rrggbb: #ffffff, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#ffffff"
},
"copy-binding": {
"type": "string",
"description": "The key combination that copies the selection to the clipboard, in X translation syntax: Ctrl Shift ~Alt <Key> C, so a plain Ctrl+C still interrupts the program. It holds no colon, backslash or line break, which would end the resource.",
"pattern": "[^:\\\\\\n]+",
"default": "Ctrl Shift ~Alt <Key> C"
},
"paste-binding": {
"type": "string",
"description": "The key combination that pastes the clipboard, in X translation syntax: Ctrl Shift ~Alt <Key> V, the pair of copy-binding. It holds no colon, backslash or line break, which would end the resource.",
"pattern": "[^:\\\\\\n]+",
"default": "Ctrl Shift ~Alt <Key> V"
}
},
"items": [
{
"collection": "environment",
"name": "terminal",
"attributes": {
"kind": "variable",
"name": "TERMINAL"
},
"template": "files/env-terminal"
},
{
"collection": "shell:xresources:normal",
"name": "xresources-normal",
"template": "files/xresources-normal"
}
],
"resources": [
{
"id": "package",
"type": "package",
"package": "xterm"
}
],
"build": {
"artifacts": [
{
"name": "tools",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/xterm-tools",
"binary": "xterm-tools",
"loads": [
"xterm-tools"
]
},
{
"name": "files",
"kind": "archive",
"from": "files"
}
]
}
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go
-364
View File
@@ -1,364 +0,0 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"os"
"regexp"
"slices"
"sort"
"strings"
"sync"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The assigner's tools (novox/hq ADR 0546), served through the tool runner from its bundle: a process never serves (ADR
// 0436, mesh-sdk process). They write and read the puts; the assigner's process takes each put further (puts.go) and says
// every step as the event put. They are the module's own tools, not a seat's verbs: a new seat needs a change of the
// controller, which takes none; they become the verbs of a seat once composing leaves the controller.
// putsState is the assigner's state of puts, as the runtime names a module's own state.
const putsState = "puts"
// answerWithin is how long assigner_put waits for the assigner's process to take the put up before it answers what stands.
var answerWithin = 20 * time.Second
// store is what the tools need of the states: the runtime's (stdio.State), or a test's.
type store interface {
get(state, key string) ([]byte, uint64, bool, error)
put(state, key string, value []byte) error
keys(state string) ([]string, error)
}
// putTools serves the tools over a store, and keeps the newest value of every put it heard, to wait on.
type putTools struct {
st store
now func() time.Time
mu sync.Mutex
// heard is every put the watch handed, by key; changed is closed and replaced at each.
heard map[string][]byte
changed chan struct{}
}
func newPutTools(st store) *putTools {
return &putTools{st: st, now: time.Now, heard: map[string][]byte{}, changed: make(chan struct{})}
}
// hear takes a put's change in from the watch.
func (s *putTools) hear(key string, value []byte) {
s.mu.Lock()
s.heard[key] = value
close(s.changed)
s.changed = make(chan struct{})
s.mu.Unlock()
}
// fullFP is a fingerprint of the new form, whole; prefix at least its first twelve hex digits.
var (
fullFP = regexp.MustCompile(`^src2:[0-9a-f]{64}$`)
prefix = regexp.MustCompile(`^(src2:)?[0-9a-f]{12,63}$`)
)
// version reads the version a put names: newest (the indexer's fingerprint at the trunk), a whole fingerprint, or the start
// of one the builder built, which must name one build alone.
func (s *putTools) version(module, version string) (string, error) {
v := strings.ToLower(strings.TrimSpace(version))
switch {
case v == "newest":
raw, _, found, err := s.st.get(delivery.IndexerModule+".modules", module)
if err != nil {
return "", fmt.Errorf("the indexer's modules cannot be read: %w", err)
}
if !found {
return "", fmt.Errorf("the indexer holds no module %s at the trunk", module)
}
m, err := delivery.DecodeModule(raw)
if err != nil {
return "", fmt.Errorf("the indexer's %s cannot be read: %w", module, err)
}
if m.FP2 == "" {
return "", fmt.Errorf("the indexer holds no fingerprint of the new form for %s: %s", module, m.None2)
}
return m.FP2, nil
case fullFP.MatchString(v):
return v, nil
case prefix.MatchString(v):
start := strings.TrimPrefix(v, delivery.SourcePrefix2)
keys, err := s.st.keys(delivery.BuilderModule + ".builds")
if err != nil {
return "", fmt.Errorf("the builder's builds cannot be listed: %w", err)
}
var found []string
for _, k := range keys {
m, fp, err := delivery.ParseBuilderBuildKey(k)
if err == nil && m == module && strings.HasPrefix(strings.TrimPrefix(fp, delivery.SourcePrefix2), start) {
found = append(found, fp)
}
}
switch len(found) {
case 0:
return "", fmt.Errorf("the builder built no %s whose fingerprint starts %s: give it whole", module, start)
case 1:
return found[0], nil
}
sort.Strings(found)
return "", fmt.Errorf("%s starts %d fingerprints of %s the builder built (%s): give more of it", start, len(found), module, strings.Join(found, ", "))
}
return "", fmt.Errorf("%q is no version: newest, a fingerprint (src2: and 64 hex digits) or at least its first 12 hex digits", version)
}
// putAnswer is what assigner_put and assigner_puts answer of one put.
type putAnswer struct {
Node string `json:"node"`
Module string `json:"module"`
delivery.Put
Follow string `json:"follow,omitempty"`
}
// doPut is the tool assigner_put.
func (s *putTools) doPut(args map[string]any, by string) (any, error) {
node, module, why := arg(args, "node"), arg(args, "module"), arg(args, "why")
key, err := delivery.PutKey(node, module)
if err != nil {
return nil, err
}
if why == "" {
return nil, errors.New("put says why: it is kept with the put and said in any ask")
}
hold := false
switch arg(args, "hold") {
case "true":
hold = true
case "", "false":
default:
return nil, fmt.Errorf("hold is true or false, not %q", arg(args, "hold"))
}
fp, err := s.version(module, arg(args, "version"))
if err != nil {
return nil, err
}
now := s.now().UTC()
words := fmt.Sprintf("asked by %s: %s", by, why)
if raw, _, found, err := s.st.get(putsState, key); err != nil {
return nil, fmt.Errorf("the put of %s on %s cannot be read: %w", module, node, err)
} else if found {
if before, err := delivery.DecodePut(raw); err == nil && !delivery.PutEnded(before.Step) {
if before.FP == fp && before.Hold == hold {
return putAnswer{Node: node, Module: module, Put: before, Follow: followPut(node, module)}, nil
}
words += fmt.Sprintf("; it replaces the put of %s at %s, which stood at %s", module, shortFP(before.FP), before.Step)
}
}
p := delivery.Put{FP: fp, Hold: hold, Why: why, By: by, At: now}.Said(now, delivery.PutAsked, words)
data, err := delivery.EncodePut(p)
if err != nil {
return nil, err
}
if err := s.st.put(putsState, key, data); err != nil {
return nil, fmt.Errorf("the put of %s on %s is not written: %w", module, node, err)
}
s.hear(key, data)
return putAnswer{Node: node, Module: module, Put: s.await(key, p), Follow: followPut(node, module)}, nil
}
// await is the put at key once the assigner's process took it up past its asking, or as it stands after answerWithin.
func (s *putTools) await(key string, p delivery.Put) delivery.Put {
deadline := time.After(answerWithin)
for {
s.mu.Lock()
raw, changed := s.heard[key], s.changed
s.mu.Unlock()
if now, err := delivery.DecodePut(raw); err == nil && now.At.Equal(p.At) {
p = now
if now.Step != delivery.PutAsked && now.Step != delivery.PutBuild && now.Step != delivery.PutCompose {
return p // it waits on a person or the node, or ended
}
}
select {
case <-changed:
case <-deadline:
return p
}
}
}
// followPut says how a caller follows a put after the answer.
func followPut(node, module string) string {
return fmt.Sprintf("the put goes on: mesh_wait on mesh-delivery-assigner.put with filter {\"node\": %q, \"module\": %q}, then "+
"mesh-delivery-assigner.assigner_puts for where it stands", node, module)
}
// doRelease is the tool assigner_release.
func (s *putTools) doRelease(args map[string]any, by string) (any, error) {
node, module, why := arg(args, "node"), arg(args, "module"), arg(args, "why")
key, err := delivery.PutKey(node, module)
if err != nil {
return nil, err
}
if why == "" {
return nil, errors.New("release says why: it is kept with the put")
}
raw, _, found, err := s.st.get(putsState, key)
if err != nil {
return nil, fmt.Errorf("the put of %s on %s cannot be read: %w", module, node, err)
}
if !found {
return nil, fmt.Errorf("%s holds no put of %s: nothing to release", node, module)
}
p, err := delivery.DecodePut(raw)
if err != nil {
return nil, fmt.Errorf("the put of %s on %s cannot be read: %w", module, node, err)
}
if delivery.PutEnded(p.Step) {
return nil, fmt.Errorf("the put of %s on %s ended already (%s: %s): nothing to release", module, node, p.Step, p.Words)
}
p = p.Said(s.now(), delivery.PutReleased, fmt.Sprintf("released by %s: %s; the assigner's rules write the want again", by, why))
data, err := delivery.EncodePut(p)
if err != nil {
return nil, err
}
if err := s.st.put(putsState, key, data); err != nil {
return nil, fmt.Errorf("the release of %s on %s is not written: %w", module, node, err)
}
s.hear(key, data)
return putAnswer{Node: node, Module: module, Put: p}, nil
}
// doPuts is the tool assigner_puts.
func (s *putTools) doPuts(args map[string]any) (any, error) {
keys, err := s.st.keys(putsState)
if err != nil {
return nil, fmt.Errorf("the puts cannot be listed: %w", err)
}
sort.Strings(keys)
out := []putAnswer{}
for _, key := range keys {
node, module, err := delivery.ParsePutKey(key)
if err != nil || (arg(args, "node") != "" && node != arg(args, "node")) || (arg(args, "module") != "" && module != arg(args, "module")) {
continue
}
raw, _, found, err := s.st.get(putsState, key)
if err != nil || !found {
continue
}
p, err := delivery.DecodePut(raw)
if err != nil {
continue
}
out = append(out, putAnswer{Node: node, Module: module, Put: p})
}
return map[string]any{"puts": out}, nil
}
func arg(args map[string]any, name string) string {
switch v := args[name].(type) {
case string:
return strings.TrimSpace(v)
case bool:
return fmt.Sprint(v)
}
return ""
}
// The tools' names, as the manifest lists them.
const (
toolPut = "assigner_put"
toolRelease = "assigner_release"
toolPuts = "assigner_puts"
)
// schema is a tool's arguments, each a string: required ones first named, and switches true or false.
func schema(properties map[string]string, required []string, switches ...string) map[string]any {
props := map[string]any{}
for name, description := range properties {
p := map[string]any{"type": "string", "description": description}
if slices.Contains(switches, name) {
p["enum"] = []string{"true", "false"}
}
props[name] = p
}
out := map[string]any{"type": "object", "properties": props}
if len(required) > 0 {
out["required"] = required
}
return out
}
// tools are the assigner's tools as the runtime serves them (ADR 0546 clause 1).
func (s *putTools) tools() []stdio.Tool {
return []stdio.Tool{
{Name: toolPut, Description: "Put one version of a module on one machine (novox/hq ADR 0546): the build at that fingerprint, " +
"the composition and grant the controller wrote for that machine, then the machine's want naming it, held there until " +
"released when hold is true. A module whose moves a person takes (the bus, a module of policy record) is asked of the " +
"operator first, and the bus has its streams snapshotted after the yes. Answers what was done or what it waits on; the " +
"put goes on after the answer, each step said as the event mesh-delivery-assigner.put, and assigner_puts says where it stands.",
Input: schema(map[string]string{
"node": "the machine, where the module is assigned",
"module": "the module",
"version": "newest, a fingerprint (src2: and 64 hex digits), or at least its first 12 hex digits",
"hold": "\"true\": the want stays at this version until released; \"false\" (the default): the rules move it again once it runs",
"why": "why, kept with the put and said in any ask",
}, []string{"node", "module", "version", "why"}, "hold"),
Run: func(a map[string]any) (any, error) { return s.doPut(a, stdio.CallerOf(a)) }},
{Name: toolRelease, Description: "End a put on one machine (novox/hq ADR 0546): its hold is taken away and the assigner's " +
"rules write the machine's want again from their next pass. Nothing changes on the machine until they move it.",
Input: schema(map[string]string{"node": "the machine", "module": "the module", "why": "why, kept with the put"},
[]string{"node", "module", "why"}),
Run: func(a map[string]any) (any, error) { return s.doRelease(a, stdio.CallerOf(a)) }},
{Name: toolPuts, Description: "Every put and where it stands (novox/hq ADR 0546): its version, hold, why, each step with " +
"its time and words. Narrowed by machine or module. Read-only.",
Input: schema(map[string]string{"node": "one machine", "module": "one module"}, nil),
Run: s.doPuts},
}
}
// runtimeStore is the runtime's states.
type runtimeStore struct{}
func (runtimeStore) get(state, key string) ([]byte, uint64, bool, error) {
e, err := stdio.State(state).Get(key)
if err != nil || e == nil {
return nil, 0, false, err
}
return e.Value, e.Revision, true, nil
}
func (runtimeStore) put(state, key string, value []byte) error {
_, err := stdio.State(state).Put(key, json.RawMessage(value))
return err
}
func (runtimeStore) keys(state string) ([]string, error) { return stdio.State(state).Keys() }
// serveTools serves the tools to the runtime until it stops, hearing every put to answer assigner_put with its first steps.
func serveTools() error {
s := newPutTools(runtimeStore{})
go func() {
<-stdio.Served()
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
err := stdio.State(putsState).Watch("", func(c stdio.StateChange) error {
s.hear(c.Key, c.Value)
return nil
})
if err == nil {
return
}
fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: the puts are not heard yet (%v); asked again in %s\n", err, wait)
time.Sleep(wait)
}
}()
return stdio.Serve("", s.tools())
}
// shortFP is a fingerprint's first twelve hex digits.
func shortFP(fp string) string {
part, err := delivery.FingerprintKeyPart(fp)
if err != nil || len(part) < 12 {
return fp
}
return part[:12]
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go
+26 -93
View File
@@ -44,8 +44,8 @@ type world struct {
manifests map[string]parsed
// healthy is, by have key, every fingerprint the node ran healthy, oldest first, each once: what a set-back goes to.
healthy map[string][]string
// changes counts, by bucket and key, the values this process has seen the key take: a value written again is another.
changes map[string]uint64
// sent is, by build id, the fingerprint of the new form the controller's build record names, once it names one.
sent map[string]string
}
// parsed is a manifest read by digest: its module and its manifest, or why it cannot be read.
@@ -56,7 +56,7 @@ type parsed struct {
}
func newWorld() *world {
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{}}
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, sent: map[string]string{}}
}
// put takes a change of a watched bucket in: a value, or nil for a key deleted or purged.
@@ -71,7 +71,6 @@ func (w *world) put(bucket, key string, value []byte) {
return
}
w.values[bucket][key] = value
w.changes[bucket+"|"+key]++
if bucket == delivery.HaveBucket {
if h, err := delivery.DecodeHave(value); err == nil && h.FP != "" && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK {
list := slices.DeleteFunc(w.healthy[key], func(fp string) bool { return fp == h.FP })
@@ -149,9 +148,7 @@ func (w *world) manifest(ctx context.Context, get getter, digest string) parsed
m, err := delivery.DecodeManifest(value)
if err != nil {
p = parsed{err: err.Error()}
} else if man, err := catalogue.ParseManifestForInputs([]byte(m.JSON)); err != nil {
// Read as the indexer reads it (issue 1751): a value falling back to a predefined key is judged by the merge gate
// in the catalogue, against the predefined.json at its root that this process never holds (ADR 0348 §7).
} else if man, err := catalogue.ParseManifest([]byte(m.JSON)); err != nil {
p = parsed{module: m.Module, err: err.Error()}
} else {
p = parsed{module: m.Module, manifest: man}
@@ -196,13 +193,12 @@ func (w *world) trunk() map[string]delivery.Module {
// mesh is the mesh as the trunk holds it, for the composition check.
func (w *world) mesh(ctx context.Context, manifests getter) assign.Mesh {
m := assign.Mesh{Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{}, Digests: map[string]string{}}
m := assign.Mesh{Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{}}
m.Assigned, _ = w.assigned()
for name, mod := range w.trunk() {
if mod.Manifest == "" {
continue
}
m.Digests[name] = mod.Manifest
p := w.manifest(ctx, manifests, mod.Manifest)
if p.err != "" {
m.Unreadable[name] = p.err
@@ -215,14 +211,12 @@ func (w *world) mesh(ctx context.Context, manifests getter) assign.Mesh {
// change is a candidate's change for the composition check, or why the composition waits for it.
func (w *world) change(ctx context.Context, manifests getter, c delivery.Candidate, trunk map[string]delivery.Module) (assign.Change, string) {
ch := assign.Change{Group: c.Group, Heads: c.HeadsHash, Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{},
Twice: map[string][]string{}, Digests: map[string]string{}, Members: map[string]string{}}
ch := assign.Change{Group: c.Group, Heads: c.HeadsHash, Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{}, Twice: map[string][]string{}}
definedIn := map[string]map[string]bool{}
for _, p := range c.Pulls {
if p.Repository == assign.SDKRepository {
ch.SDKHead = p.Head.Head
}
ch.Members[p.Repository] = p.Head.Head
for _, name := range p.Modules {
fp, ok := p.Fps[name]
if !ok || fp.Manifest == "" {
@@ -239,7 +233,6 @@ func (w *world) change(ctx context.Context, manifests getter, c delivery.Candida
if strings.HasPrefix(got.err, "the indexer holds no manifest") {
return ch, got.err
}
ch.Digests[name] = fp.Manifest
if got.err != "" {
ch.Unreadable[name] = got.err
continue
@@ -326,13 +319,6 @@ func (w *world) nodes(manifests map[string]catalogue.Manifest, assigned map[stri
return n
}
// value is what a bucket holds at a key now, nil when nothing.
func (w *world) value(bucket, key string) []byte {
w.mu.Lock()
defer w.mu.Unlock()
return w.values[bucket][key]
}
// has says whether a bucket holds a key now.
func (w *world) has(bucket, key string) bool {
w.mu.Lock()
@@ -357,81 +343,28 @@ func (w *world) built(module, fp string) bool {
return err == nil && b.State == delivery.BuildDone
}
// composed is the controller's composition of module for node at fp: its digest and the judgement of the move to it
// (mesh-controller_composed `<node>.<module>.<hex>`), and whether it exists and can be read.
func (w *world) composed(node, module, fp string) (delivery.ComposedDigest, bool) {
key, err := delivery.ComposedKey(node, module, fp)
// sentFP is the fingerprint of the new form of the build the controller's assignment names, read from its build record
// (mesh-controller_builds `<module>.build.<id>`) once and kept; empty when the record names none.
func (w *world) sentFP(ctx context.Context, get getter, module, build string) string {
if build == "" {
return ""
}
w.mu.Lock()
fp, ok := w.sent[build]
w.mu.Unlock()
if ok {
return fp
}
value, err := get(ctx, delivery.BuildKey(module, build))
if err != nil {
return delivery.ComposedDigest{}, false
return ""
}
var rec delivery.BuildRecord
if json.Unmarshal(value, &rec) != nil || !strings.HasPrefix(rec.SourceFingerprint, delivery.SourcePrefix2) {
return ""
}
w.mu.Lock()
value, ok := w.values[delivery.ControllerComposedBucket][key]
w.sent[build] = rec.SourceFingerprint
w.mu.Unlock()
if !ok {
return delivery.ComposedDigest{}, false
}
d, err := delivery.DecodeComposedDigest(value)
return d, err == nil
}
// nodeComposed is, by node, the digest of the node's own composition (mesh-controller_composed `<node>.node`).
func (w *world) nodeComposed() map[string]delivery.ComposedDigest {
out := map[string]delivery.ComposedDigest{}
for key, value := range w.read(delivery.ControllerComposedBucket) {
node, _, _, isNode, err := delivery.ParseComposedKey(key)
if err != nil || !isNode {
continue
}
if d, err := delivery.DecodeComposedDigest(value); err == nil {
out[node] = d
}
}
return out
}
// answer is the teller's answer to the ask at key (mesh-delivery-teller_answers, the same key), and whether there is one.
func (w *world) answer(key string) (delivery.Answer, bool) {
w.mu.Lock()
value, ok := w.values[delivery.AnswersBucket][key]
w.mu.Unlock()
if !ok {
return delivery.Answer{}, false
}
a, err := delivery.DecodeAnswer(value)
return a, err == nil
}
// changed is how many values the key has taken in this process (see world.changes).
func (w *world) changed(bucket, key string) uint64 {
w.mu.Lock()
defer w.mu.Unlock()
return w.changes[bucket+"|"+key]
}
// runningCommits is, of every open assignment of module, the commit of the build it names, read from the controller's
// build record (`mesh-controller_builds` `<module>.build.<id>`), distinct and sorted: the commits a reader of the
// manifests runs at (novox/hq ADR 0482 clause 5, issue 1623). A build whose record names no full commit is left out.
func (w *world) runningCommits(ctx context.Context, get getter, module string) []string {
_, all := w.assigned()
seen := map[string]bool{}
for _, a := range all {
if a.Module != module || a.Build == "" {
continue
}
value, err := get(ctx, delivery.BuildKey(module, a.Build))
if err != nil {
continue
}
var rec delivery.BuildRecord
if json.Unmarshal(value, &rec) != nil || len(rec.Commit) != 40 {
continue
}
seen[rec.Commit] = true
}
out := make([]string, 0, len(seen))
for c := range seen {
out = append(out, c)
}
sort.Strings(out)
return out
return rec.SourceFingerprint
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go
-37
View File
@@ -2,7 +2,6 @@ package main
import (
"context"
"os"
"slices"
"strings"
"testing"
@@ -89,39 +88,3 @@ func TestAChangeWaitsForItsManifests(t *testing.T) {
t.Errorf("an unreadable manifest: waits %q, change %+v", waits, ch)
}
}
// A manifest of the change whose config value falls back to a predefined key (novox/hq ADR 0348 §7) is read, not
// unreadable for the catalogue's predefined.json the assigner never holds: the merge gate judges the fallback in the
// catalogue, and the indexer reads it the same way (catalogue.ParseManifestForInputs, issue 1751). A group whose
// mesh-catalog member touched pacman and xterm failed its composition on this (group check 18df81380db3). The manifest is
// xterm's module.json from mesh-catalog at 835bf22b.
func TestAManifestFallingBackToAPredefinedKeyIsReadable(t *testing.T) {
raw, err := os.ReadFile("testdata/catalogue-835bf22b-xterm.module.json")
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(raw), `"predefined"`) {
t.Fatal("the fixture names no predefined key")
}
f := fakeManifests{}
xterm := f.add(t, "xterm", string(raw))
ch, waits := newWorld().change(context.Background(), f.get, delivery.Candidate{Group: "g", HeadsHash: "0123456789ab",
Pulls: []delivery.CandidatePull{pull("novox/mesh-catalog", 1, map[string]delivery.CandidateModule{"xterm": {FP2: fp('b'), Manifest: xterm}})}}, nil)
if waits != "" || ch.Unreadable["xterm"] != "" || ch.Manifests["xterm"].Module != "xterm" {
t.Errorf("xterm: waits %q, unreadable %q, want read", waits, ch.Unreadable["xterm"])
}
}
// Review round 1 (items 3 and 4): the world counts the values a key takes, so a have written again after an answer is seen.
func TestTheWorldCountsTheValuesAKeyTakes(t *testing.T) {
w := newWorld()
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
w.put(delivery.HaveBucket, "ace.mail", []byte(`{"v":1}`))
if got := w.changed(delivery.HaveBucket, "ace.web"); got != 2 {
t.Errorf("changed = %d, want 2", got)
}
if got := w.changed(delivery.HaveBucket, "gone"); got != 0 {
t.Errorf("changed = %d for a key never seen", got)
}
}
modules/mesh-delivery-assigner/go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
modules/mesh-delivery-assigner/go.sum
+2 -8
View File
@@ -1,11 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025195051-20a6dc4920f8 h1:jA7V7hlfJAigVUvJDpKwfgd9ylQ9bqppg3rK9XgLWoA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025195051-20a6dc4920f8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8 h1:vhYk/iv43u2WGSTrFyQGDvGBxLaRITgMAiiI9DjuA10=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 h1:gE80bR0tPME7MTMBLcdP62v+F/9soVXmdcBYNMOye44=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
modules/mesh-delivery-assigner/internal/assign/asks.go
+9 -186
View File
@@ -2,28 +2,24 @@ package assign
import (
"fmt"
"regexp"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (delivery.HoldAskKey): keep it held or
// release it. Releasing writes the held fingerprint's want again by the rules (ADR 0539 §19). node and words are the held
// record's: the machine whose health check failed and the hold's words (issue 1906). asking numbers the writing: 0 the first, a
// larger number an ask written again after a hold lifted by itself, so the router takes it as new.
func HoldAsk(key, module, fp, node, words string, asking int, now time.Time) delivery.AssignerAsk {
return delivery.AssignerAsk{Asking: asking, Ask: asks.Ask{
ID: delivery.AskingID(key, asking),
// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (delivery.HoldAskKey), first asking: keep
// it held or release it. Both options acknowledge and bind nothing: in release 5 the wants are only compared, so no answer
// changes what a node runs (ADR 0429 §7); the node agent acts on the hold from release 6.
func HoldAsk(key, module, fp string, now time.Time) delivery.AssignerAsk {
return delivery.AssignerAsk{Ask: asks.Ask{
ID: delivery.AskingID(key, 0),
Headline: fmt.Sprintf("%s is held at %s", module, short(fp)),
Explanation: fmt.Sprintf("The health check of %s failed on %s at %s (%s), so the assigner holds that version and wants "+
"no node to move to it. It lifts the hold by itself once every node that runs %s runs it healthy; releasing it "+
"lets the wants move to it now.", module, node, short(fp), words, module),
Explanation: fmt.Sprintf("A node's health check of %s failed at %s, so the assigner holds that version and wants "+
"no node to move to it. Until release 6 this changes nothing a node runs: the controller still sends.", module, short(fp)),
Options: []asks.Option{
{ID: "keep", Label: "Keep it held", Does: "the version stays held", Level: asks.Acknowledge},
{ID: "release", Label: "Release it", Does: "the version is released and the wants move to it by the rules again", Level: asks.Acknowledge},
{ID: "release", Label: "Release it", Does: "the version is released when wants are applied, from release 6", Level: asks.Acknowledge},
},
Who: asks.Operator,
Expires: now.Add(7 * 24 * time.Hour),
@@ -31,176 +27,3 @@ func HoldAsk(key, module, fp, node, words string, asking int, now time.Time) del
About: key,
}}
}
// The options of the restore ask (ADR 0537 §3, ADR 0539 §19), and what each does on the node.
const (
OptionRestore = "restore"
OptionLeaveStopped = "leave-stopped"
)
// RestoreAsk is the ask about a module holding data whose fingerprint failed its health check on a node (ADR 0429 §6): it is
// not set back by itself, since a set-back would lose the writes made since. words is the check's words. Restore sets the
// node back to the last healthy fingerprint and loses the newer writes; leaving it stopped keeps the data as it is.
func RestoreAsk(key, node, module, fp, words string, now time.Time) delivery.AssignerAsk {
return delivery.AssignerAsk{Ask: asks.Ask{
ID: delivery.AskingID(key, 0),
Headline: fmt.Sprintf("%s failed its health check on %s", module, node),
Explanation: fmt.Sprintf("%s holds data, and its health check failed on %s at %s: %s. Setting it back to the version that ran healthy "+
"would lose the writes made since; leaving it stopped keeps the data as it is.", module, node, short(fp), words),
Options: []asks.Option{
{ID: OptionRestore, Label: "Restore it", Does: "the node is set back to the last healthy version and the newer writes are lost", Level: asks.Acknowledge},
{ID: OptionLeaveStopped, Label: "Leave it stopped", Does: "the module stays as it is on the node and the data is kept", Level: asks.Acknowledge},
},
Who: asks.Operator,
Expires: now.Add(7 * 24 * time.Hour),
OnExpiry: "the module stays as it is on the node",
About: key,
}}
}
// HeldCondition is the condition the assigner raises when it holds a fingerprint of a module (ADR 0429 §6, ADR 0539 §19), or
// clears (raised false) when the hold is released. It names the module, the node whose health check failed and the check's
// words, under the assigner's own name (ADR 0437 clause 9), keyed by the held fingerprint. needsYou is true for a module holding
// data, whose restore ask waits for the operator.
func HeldCondition(module, fp, node, words string, raised, needsYou bool) asks.Condition {
key := delivery.AssignerModule + ".held." + module + "." + strings.TrimPrefix(fp, delivery.SourcePrefix2)
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
if !raised {
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared}
}
c := asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
Summary: fmt.Sprintf("%s is held at %s", module, short(fp)),
Headline: fmt.Sprintf("%s is held at %s", module, short(fp))}
what := fmt.Sprintf("The health check of %s failed on %s at %s: %s. The assigner holds that version and wants no node to move to it.", module, node, short(fp), words)
if needsYou {
c.Needs = "answer whether to restore " + module + " on " + node
c.Explanation = asks.NeedsYou + c.Needs + ". " + what
} else {
c.Explanation = asks.NothingToDo + " " + what + " Nodes that wanted it are set back to the version that ran healthy."
}
return c
}
// NeitherCondition is the condition of a module assigned to node that holds no want for it (issue 1945): the node's own composition
// gives up every module with a fingerprint of the new form, so nothing applies it there and its files are gone. It is raised (urgent)
// naming the module, the node and the rule holding its first want back with the rule's words, under the assigner's own name keyed by
// node and module, and cleared (raised false) when the node has a want for it again.
func NeitherCondition(node, module string, h Holding, raised bool) asks.Condition {
key := delivery.AssignerModule + ".neither." + node + "." + module
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
if !raised {
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared,
Resolved: fmt.Sprintf("%s on %s has a want again, so its own composition is applied there.", module, node)}
}
headline := fmt.Sprintf("%s on %s is on no list", module, node)
if n := len([]rune(headline)); n > asks.HeadlineLength {
headline = string([]rune(headline)[:asks.HeadlineLength])
}
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
Summary: headline, Headline: headline,
Explanation: fmt.Sprintf("%s %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
"hold it either, so nothing applies it there. The assigner's rule %d holds its first want back: %s. The assigner writes the want once the "+
"rule holds, and clears this.", asks.NothingToDo, module, node, h.Rule, h.Words)}
}
// LiftedCondition clears the condition of a held fingerprint the assigner lifted by itself, saying why (issue 1906).
func LiftedCondition(module, fp string) asks.Condition {
c := HeldCondition(module, fp, "", "", false, false)
c.Resolved = LiftedWords(module, fp) + "."
return c
}
// The options of the record ask (ADR 0539 §14) and of the backup ask (§16).
const (
OptionYes = "yes"
OptionNo = "no"
OptionBackupAgain = delivery.BackupAgain
OptionBackupSkip = delivery.BackupSkip
)
// RecordAsk is the ask about a move of a module of policy record on node to the composition with the digest, judged by the
// controller to recreate a container, to move data, or not to be judgeable (ADR 0539 §14). The want names the composition only
// after the answer yes. The way the move was judged is in the ask.
func RecordAsk(key, node, module, fp string, need RecordNeed, now time.Time) delivery.AssignerAsk {
what := map[delivery.Judgement]string{
delivery.JudgedRecreates: "recreates a container",
delivery.JudgedMovesData: "moves the module's data",
delivery.JudgedCannot: "cannot be judged: " + need.Words,
}[need.Judgement]
return delivery.AssignerAsk{Ask: asks.Ask{
ID: delivery.AskingID(key, 0),
Headline: fmt.Sprintf("%s waits for your yes on %s", module, node),
Explanation: fmt.Sprintf("%s is recorded: a person takes each of its moves. The next one on %s (version %s) %s. "+
"The node is told to apply it only after you say yes.", module, node, short(fp), what),
Options: []asks.Option{
{ID: OptionYes, Label: "Yes, move it", Does: "the node is told to apply the new composition of " + module, Level: asks.Acknowledge},
{ID: OptionNo, Label: "No, not now", Does: "the node keeps what it runs", Level: asks.Acknowledge},
},
Who: asks.Operator,
Expires: now.Add(7 * 24 * time.Hour),
OnExpiry: "the node keeps what it runs",
About: key,
}}
}
// NextBackupAsking is the number of the asking that follows the one numbered after (a negative number: none yet). Numbers 0 and 1
// give one id (delivery.AskingID), so the first asking is 0 and the next 2.
func NextBackupAsking(after int) int {
if after < 0 {
return 0
}
return max(after+1, 2)
}
var backupDigest = regexp.MustCompile(`\(composition (sha256:[0-9a-f]{64})\)`)
// BackupAskDigest is the digest of the composition a backup ask was made about, from its explanation; empty for an ask that names
// none. An answer is taken for the composition of the ask it answers, and a new revision at the same fingerprint is asked again
// under a new number (ADR 0539 §16).
func BackupAskDigest(a delivery.AssignerAsk) string {
if m := backupDigest.FindStringSubmatch(a.Ask.Explanation); m != nil {
return m[1]
}
return ""
}
// BackupAnswerWords are the words of a want carrying the answer to the asking-th ask: each asking's answer is another want
// value, so a backup that fails again after "back up again" and is answered "again" again changes the key and the node retries.
func BackupAnswerWords(option string, asking int) string {
return fmt.Sprintf("the operator answered %q to the backup ask (asking %d)", option, asking+1)
}
// BackupAsk is the ask about a backup of a module on node that failed before an apply (ADR 0539 §16): back up again, or go on
// without a backup. digest is the composition about to be applied and asking the number of the asking, 0 the first; words are the
// backup's own words, from the node's have.
func BackupAsk(key, node, module, fp, digest, words string, asking int, now time.Time) delivery.AssignerAsk {
return delivery.AssignerAsk{Asking: asking, Ask: asks.Ask{
ID: delivery.AskingID(key, asking),
Headline: fmt.Sprintf("The backup of %s failed on %s", module, node),
Explanation: fmt.Sprintf("The backup before applying %s at %s on %s failed: %s. %s keeps running what it has until you answer. (composition %s)",
module, short(fp), node, words, module, digest),
Options: []asks.Option{
{ID: OptionBackupAgain, Label: "Back up again", Does: "the node tries the backup again before it applies", Level: asks.Acknowledge},
{ID: OptionBackupSkip, Label: "Go on without a backup", Does: "the node applies the new version without a backup", Level: asks.Acknowledge},
},
Who: asks.Operator,
Expires: now.Add(7 * 24 * time.Hour),
OnExpiry: "the module keeps running what it has",
About: key,
}}
}
// BackupCondition is the condition raised when a backup before an apply fails and the module is held on the node (ADR 0539 §16),
// naming the node, the module and the backup's words; cleared under the same key.
func BackupCondition(node, module, fp, words string, raised bool) asks.Condition {
key := delivery.AssignerModule + ".backup." + node + "." + module + "." + strings.TrimPrefix(fp, delivery.SourcePrefix2)
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
if !raised {
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared}
}
headline := fmt.Sprintf("The backup of %s failed on %s", module, node)
needs := "answer whether to back up again or go on without a backup"
return asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
Summary: headline, Headline: headline, Needs: needs,
Explanation: asks.NeedsYou + needs + ". The backup before applying " + module + " at " + short(fp) + " on " + node + " failed: " + words + "."}
}
modules/mesh-delivery-assigner/internal/assign/asks_test.go
+1 -176
View File
@@ -1,11 +1,9 @@
package assign
import (
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -17,39 +15,7 @@ func TestTheHoldAskIsValid(t *testing.T) {
t.Fatal(err)
}
now := time.Now()
a := HoldAsk(key, "mail", fNew, "ace", "health failed, phase running: connection refused", 0, now)
if err := a.Ask.Check(now); err != nil {
t.Errorf("the router refuses the ask: %v", err)
}
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
t.Errorf("the ask cannot be written: %v", err)
}
// 1906/H3: the ask names the machine whose check failed and the hold's words.
if !strings.Contains(a.Ask.Explanation, "on ace") || !strings.Contains(a.Ask.Explanation, "connection refused") {
t.Errorf("the ask does not name the node and the hold's words: %q", a.Ask.Explanation)
}
// 1906/H2: an ask written again after a lift is another asking, under an id of its own.
again := HoldAsk(key, "mail", fNew, "ace", "x", 7, now)
if again.Asking != 7 || again.Ask.ID != delivery.AskingID(key, 7) || again.Ask.ID == a.Ask.ID {
t.Errorf("the asking after a lift = %d %s", again.Asking, again.Ask.ID)
}
if _, err := delivery.EncodeAssignerAsk(key, again); err != nil {
t.Errorf("the ask asked again cannot be written: %v", err)
}
if _, err := delivery.ViewAnswerKey(key); err != nil {
t.Errorf("the view may not answer it: %v", err)
}
}
// 1906/D3 (ADR 0539 §19): the ask about a data module held on a node is a valid ask of the router and of mesh-sdk's layout at
// its restore key, of a kind the view may answer; it offers restoring (losing the newer writes) and leaving it stopped.
func TestTheRestoreAskIsValid(t *testing.T) {
key, err := delivery.RestoreAskKey("ace", "mail", fNew)
if err != nil {
t.Fatal(err)
}
now := time.Now()
a := RestoreAsk(key, "ace", "mail", fNew, "connection refused", now)
a := HoldAsk(key, "mail", fNew, now)
if err := a.Ask.Check(now); err != nil {
t.Errorf("the router refuses the ask: %v", err)
}
@@ -59,145 +25,4 @@ func TestTheRestoreAskIsValid(t *testing.T) {
if _, err := delivery.ViewAnswerKey(key); err != nil {
t.Errorf("the view may not answer it: %v", err)
}
var ids []string
for _, o := range a.Ask.Options {
ids = append(ids, o.ID)
}
if len(ids) != 2 || ids[0] != OptionRestore || ids[1] != OptionLeaveStopped {
t.Errorf("options = %v", ids)
}
if !strings.Contains(a.Ask.Explanation, "connection refused") || !strings.Contains(a.Ask.Explanation, "ace") {
t.Errorf("the ask does not name the node and the check's words: %q", a.Ask.Explanation)
}
}
// 1906/D3: the condition for a held fingerprint names the module, the node and the check's words, under the assigner's own
// name (ADR 0437 clause 9), and is cleared under the same key.
func TestTheHeldConditionNamesModuleNodeAndWords(t *testing.T) {
c := HeldCondition("mail", fNew, "ace", "connection refused", true, true)
if _, err := asks.EncodeCondition(c); err != nil {
t.Fatalf("the condition is refused: %v", err)
}
if c.State != asks.StateRaised || c.Subject.ID != "mail" || c.Subject.Machine != "ace" || !strings.Contains(c.Explanation, "connection refused") || !strings.Contains(c.Key, "mail") {
t.Errorf("condition = %+v", c)
}
if got := HeldCondition("mail", fNew, "", "", false, false); got.State != asks.StateCleared || got.Key != c.Key {
t.Errorf("cleared = %+v", got)
}
}
// 1906/H2: a hold lifted by itself clears the held condition under its key, saying why.
func TestALiftedHoldClearsItsConditionSayingWhy(t *testing.T) {
c := LiftedCondition("mail", fNew)
if _, err := asks.EncodeCondition(c); err != nil {
t.Fatalf("the condition is refused: %v", err)
}
if c.State != asks.StateCleared || c.Key != HeldCondition("mail", fNew, "ace", "x", true, false).Key || !strings.Contains(c.Resolved, "lifted its hold by itself") {
t.Errorf("condition = %+v", c)
}
}
// 1906/D4: the record ask and the backup ask are valid asks of the router and of mesh-sdk's layout at their keys, answerable in
// the view, and name the move and the node.
func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
now := time.Now()
rkey, err := delivery.RecordAskKey("ace", "mail", "sha256:"+strings.Repeat("b", 64))
if err != nil {
t.Fatal(err)
}
r := RecordAsk(rkey, "ace", "mail", fNew, RecordNeed{Node: "ace", Digest: "sha256:" + strings.Repeat("b", 64), Judgement: delivery.JudgedCannot, Words: "the composition it runs is gone"}, now)
bkey, err := delivery.BackupAskKey("ace", "mail", fNew)
if err != nil {
t.Fatal(err)
}
dgst := "sha256:" + strings.Repeat("b", 64)
b := BackupAsk(bkey, "ace", "mail", fNew, dgst, "no space left", 0, now)
for key, a := range map[string]delivery.AssignerAsk{rkey: r, bkey: b} {
if err := a.Ask.Check(now); err != nil {
t.Errorf("%s: the router refuses the ask: %v", key, err)
}
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
t.Errorf("%s: cannot be written: %v", key, err)
}
if _, err := delivery.ViewAnswerKey(key); err != nil {
t.Errorf("%s: the view may not answer it: %v", key, err)
}
}
if !strings.Contains(r.Ask.Explanation, "gone") || r.Ask.Options[0].ID != OptionYes {
t.Errorf("the record ask = %+v", r.Ask)
}
if !strings.Contains(b.Ask.Explanation, "no space left") || b.Ask.Options[0].ID != delivery.BackupAgain || b.Ask.Options[1].ID != delivery.BackupSkip {
t.Errorf("the backup ask = %+v", b.Ask)
}
// the id binds the ask to the composition and to its number (review round 1, items 3 and 4)
if again := BackupAsk(bkey, "ace", "mail", fNew, dgst, "x", NextBackupAsking(0), now); again.Ask.ID == b.Ask.ID {
t.Errorf("asked again under the same id")
}
if NextBackupAsking(-1) != 0 || NextBackupAsking(0) != 2 || NextBackupAsking(2) != 3 {
t.Errorf("the numbers of the askings: %d %d %d", NextBackupAsking(-1), NextBackupAsking(0), NextBackupAsking(2))
}
if BackupAskDigest(b) != dgst {
t.Errorf("the ask names composition %q, want %q", BackupAskDigest(b), dgst)
}
if BackupAnswerWords("again", 0) == BackupAnswerWords("again", 1) {
t.Errorf("two answers of the same option are one want value")
}
c := BackupCondition("ace", "mail", fNew, "no space left", true)
if _, err := asks.EncodeCondition(c); err != nil {
t.Errorf("the backup condition is refused: %v", err)
}
// ADR 0546 clauses 7 and 8: a put's ask is valid at the record ask's key; for the bus its yes approves and binds the act,
// so the view cannot answer it, and it lasts no longer than an approving ask may.
put := delivery.Put{FP: fNew, Why: "the bus's user list", Hold: true}
d := delivery.ComposedDigest{Digest: dgst, Judgement: delivery.JudgedCannot, Words: "the have names no composition"}
for _, bus := range []bool{false, true} {
a, err := PutAsk(rkey, "ace", "mail", put, d, bus, now)
if err != nil {
t.Fatalf("bus %v: %v", bus, err)
}
if err := a.Ask.Check(now); err != nil {
t.Errorf("bus %v: the router refuses the put's ask: %v", bus, err)
}
if _, err := delivery.EncodeAssignerAsk(rkey, a); err != nil {
t.Errorf("bus %v: the put's ask cannot be written: %v", bus, err)
}
viewed := delivery.ViewAnswerFits(delivery.ViewAnswer{Asking: a.Ask.ID, Option: OptionYes}, a)
yes := a.Ask.Options[0]
switch {
case !strings.Contains(a.Ask.Explanation, "the bus's user list"):
t.Errorf("bus %v: the put's ask does not say why: %s", bus, a.Ask.Explanation)
case bus && (yes.Level != asks.Approve || yes.Binds == "" || viewed == nil || a.Ask.Expires.After(now.Add(asks.ApproveLasts))):
t.Errorf("bus: the yes is %+v, the view's answer %v, expiring %s", yes, viewed, a.Ask.Expires)
case !bus && (yes.Level != asks.Acknowledge || viewed != nil):
t.Errorf("a recorded module: the yes is %+v, the view's answer %v", yes, viewed)
}
later := put
later.At = now.Add(time.Minute)
b, _ := PutAsk(rkey, "ace", "mail", later, d, bus, now)
if b.Ask.ID == a.Ask.ID || b.Asking != PutAsking(later) {
t.Errorf("bus %v: a later put's ask is asking %d under id %q, the earlier's %q", bus, b.Asking, b.Ask.ID, a.Ask.ID)
}
if err := yes.Performs(PutAct("ace", "mail", fNew, dgst)); err != nil {
t.Errorf("bus %v: the yes does not perform the act it bound: %v", bus, err)
}
if err := yes.Performs(PutAct("ace", "mail", fNew, dig(fOld))); bus && err == nil {
t.Errorf("bus: the yes performs another composition's act")
}
}
}
// 1945/AC21: the condition of a module on neither list is urgent, names the module, the node and the rule holding its want back,
// under the assigner's own name keyed by node and module, and is cleared under the same key.
func TestTheNeitherConditionNamesModuleNodeAndRule(t *testing.T) {
c := NeitherCondition("g14", "i3status-rust", Holding{Rule: delivery.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}, true)
if _, err := asks.EncodeCondition(c); err != nil {
t.Fatalf("the condition is refused: %v", err)
}
if c.State != asks.StateRaised || c.Severity != asks.SeverityUrgent || c.Subject.ID != "i3status-rust" || c.Subject.Machine != "g14" ||
!strings.Contains(c.Explanation, "x11-display") || !strings.Contains(c.Explanation, "rule 5") || c.Key != "mesh-delivery-assigner.neither.g14.i3status-rust" {
t.Errorf("condition = %+v", c)
}
if got := NeitherCondition("g14", "i3status-rust", Holding{}, false); got.State != asks.StateCleared || got.Key != c.Key {
t.Errorf("cleared = %+v", got)
}
}
modules/mesh-delivery-assigner/internal/assign/compare.go
+52
View File
@@ -0,0 +1,52 @@
package assign
import (
"fmt"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// Sent is what the controller sends a node of a module: whether it sends it at all (an open assignment), the build its
// assignment names, and that build's fingerprint of the new form (empty when its record names none).
type Sent struct {
Sends bool
Build string
FP string
}
// Compare is the difference between a node's want for a module and what the controller sends it (ADR 0528 clause 4),
// and whether there is one. want is nil when the assigner wrote none; newest is the module's newest fingerprint and
// hold the rule that holds the node back from it (Rule 0 when none does). A difference is explained by a rule when the
// controller sent the newest fingerprint while that rule holds the node's want back; every other is unexplained, said
// in words, for acceptance test 17 of design 47 to read.
func Compare(want *delivery.Want, sent Sent, newest string, hold Holding) (delivery.Difference, bool) {
var d delivery.Difference
removed := want != nil && want.Removed
if want != nil && !removed {
d.Want = want.FP
}
d.WantRemoved = removed
if !sent.Sends {
if want == nil || removed {
return delivery.Difference{}, false
}
d.Unexplained, d.Words = true, "the node wants "+short(want.FP)+" and the controller sends it nothing"
return d, true
}
d.Sent, d.SentBuild = sent.FP, sent.Build
switch {
case sent.FP == "":
d.Unexplained, d.Words = true, fmt.Sprintf("the build %s the controller sends names no fingerprint of the new form", sent.Build)
case removed:
d.Unexplained, d.Words = true, fmt.Sprintf("the want is removed and the controller still sends %s", short(sent.FP))
case sent.FP == d.Want:
return delivery.Difference{}, false
case hold.Rule != 0 && sent.FP == newest:
d.Rule, d.Words = hold.Rule, fmt.Sprintf("the controller sent %s while %s", short(sent.FP), hold.Words)
case d.Want == "":
d.Unexplained, d.Words = true, fmt.Sprintf("the controller sends %s and the node has no want", short(sent.FP))
default:
d.Unexplained, d.Words = true, fmt.Sprintf("the controller sends %s and the node wants %s", short(sent.FP), short(d.Want))
}
return d, true
}
modules/mesh-delivery-assigner/internal/assign/compare_test.go
+59
View File
@@ -0,0 +1,59 @@
package assign
import (
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// 1297/B5: given the controller sent F to a node while the canary is not yet healthy on F, the difference names rule 3.
func TestADifferenceExplainedByTheCanary(t *testing.T) {
m, n := ready()
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
out := Wants(m, n, wantedOld(), nil)
want := delivery.Want{FP: fOld, Newest: fNew, Rule: delivery.RuleCanary}
d, ok := Compare(&want, Sent{Sends: true, Build: "b-9", FP: fNew}, fNew, out.Rules["g14"])
if !ok || d.Rule != delivery.RuleCanary || d.Unexplained || d.Want != fOld || d.Sent != fNew {
t.Fatalf("difference = %+v, %v; want rule 3", d, ok)
}
if _, err := delivery.EncodeDifference(d); err != nil {
t.Errorf("the difference cannot be written: %v", err)
}
}
// 1297/B5: a want equal to what is sent is no difference; nothing sent and no want is none; every other is written
// unexplained, with words, and is writable.
func TestCompare(t *testing.T) {
w := delivery.Want{FP: fNew}
if _, ok := Compare(&w, Sent{Sends: true, Build: "b", FP: fNew}, fNew, Holding{}); ok {
t.Errorf("an equal want and send is a difference")
}
if _, ok := Compare(nil, Sent{}, fNew, Holding{}); ok {
t.Errorf("no want and nothing sent is a difference")
}
removed := delivery.Want{Removed: true}
if _, ok := Compare(&removed, Sent{}, fNew, Holding{}); ok {
t.Errorf("a removed want and nothing sent is a difference")
}
old := delivery.Want{FP: fOld}
for name, c := range map[string]struct {
want *delivery.Want
sent Sent
}{
"sends an older one": {&w, Sent{Sends: true, Build: "b", FP: fOld}},
"names no fingerprint": {&w, Sent{Sends: true, Build: "b"}},
"sends a removed module": {&removed, Sent{Sends: true, Build: "b", FP: fNew}},
"sends nothing": {&w, Sent{}},
"no want": {nil, Sent{Sends: true, Build: "b", FP: fNew}},
"no rule holds it back": {&old, Sent{Sends: true, Build: "b", FP: fNew}},
} {
d, ok := Compare(c.want, c.sent, fNew, Holding{})
if !ok || !d.Unexplained || d.Words == "" {
t.Errorf("%s: difference = %+v, %v; want it unexplained, with words", name, d, ok)
continue
}
if _, err := delivery.EncodeDifference(d); err != nil {
t.Errorf("%s: the difference cannot be written: %v", name, err)
}
}
}
modules/mesh-delivery-assigner/internal/assign/compose.go
+3 -16
View File
@@ -32,9 +32,6 @@ type Mesh struct {
Manifests map[string]catalogue.Manifest
Unreadable map[string]string
Assigned map[string][]string
// Digests is every trunk module's manifest digest in the indexer's manifests bucket, readable or not: what a
// composer at another mesh-sdk reads (issue 1622).
Digests map[string]string
}
// Change is a group's change as the indexer's candidate holds it: the manifest of every module it moves or adds, those
@@ -48,10 +45,6 @@ type Change struct {
Twice map[string][]string
SDKHead string
Rebuilt int
// Digests is the manifest digest of every module the change moves or adds, readable by the assigner or not, and
// Members the head of each member by repository: what a composer at the group's mesh-sdk reads (issues 1622, 1623).
Digests map[string]string
Members map[string]string
}
// Touched is the machines a change touches (ADR 0446 clause 3): those running a module the change moves, adds or
@@ -174,19 +167,13 @@ func Compose(m Mesh, c Change, sdk string, took time.Duration) delivery.Composit
failAt(node, fmt.Sprintf("%s composes on the trunk and not with the change: %v", node, now))
}
}
if c.Rebuilt > delivery.CompositionRebuildWarning {
v.Warnings = append(v.Warnings, fmt.Sprintf("the merge rebuilds %d modules, more than %d", c.Rebuilt, delivery.CompositionRebuildWarning))
}
if c.SDKHead != "" {
v.Warnings = append(v.Warnings, fmt.Sprintf("composed with the assigner's own mesh-sdk %s, not the group's mesh-sdk head %s, "+
"and a field the running controller cannot read is not judged (issues 1621, 1622, 1623)", sdk, c.SDKHead))
}
return finish(v, m, c, machines, fails)
}
// finish adds what every composition says beside its own judgement (more modules rebuilt than CompositionRebuildWarning,
// what is not judged, the machines not composed, a change that touches none) and its state from fails.
func finish(v delivery.CompositionVerdict, m Mesh, c Change, machines []string, fails []string) delivery.CompositionVerdict {
if c.Rebuilt > delivery.CompositionRebuildWarning {
v.Warnings = append(v.Warnings, fmt.Sprintf("the merge rebuilds %d modules, more than %d", c.Rebuilt, delivery.CompositionRebuildWarning))
}
v.Warnings = append(v.Warnings, NotJudged)
if could := CouldRun(m, c, machines); len(could) > 0 {
v.Warnings = append(v.Warnings, "not judged until the facts snapshot is on the bus (issue 1461, ADR 0537 clause 1): "+
modules/mesh-delivery-assigner/internal/assign/composer.go
-241
View File
@@ -1,241 +0,0 @@
package assign
import (
"encoding/json"
"fmt"
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/compose"
"git.novox.be/novox/mesh-sdk/go/composer"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The cases of a group's composition at its mesh-sdk head: the trunk as it is, and the trunk with the change.
const (
CaseTrunk = "trunk"
CaseChange = "change"
)
// SDKDir is the directory of mesh-sdk's Go module in its repository, where the composer is built at a head; a reader's
// composer is built at its repository's root, at the mesh-sdk its go.mod pins.
const SDKDir = "go"
// The modules whose mesh-sdk reads the manifests a plan sends (ADR 0482 clause 5): the controller composes and sends
// them, the node-engine applies them. Each is judged at its repository.
var ReaderModules = map[string]string{"mesh-controller": "novox/mesh-controller", "mesh-host": "novox/mesh-host"}
// HeadJob is the compose job of a group with a mesh-sdk member (ADR 0482 clauses 3 and 4, issue 1622): the composer
// built at the member's head composes the machines the change touches, as the trunk holds them and as the change leaves
// them, every manifest named by its digest in the indexer's bucket, the seats held from the claims (the assigner's input).
func HeadJob(m Mesh, c Change) (delivery.ComposeJob, error) {
trunk, change := map[string]composer.Manifest{}, map[string]composer.Manifest{}
for name, digest := range m.Digests {
trunk[name] = composer.Manifest{Digest: digest}
change[name] = composer.Manifest{Digest: digest}
}
for name, digest := range c.Digests {
change[name] = composer.Manifest{Digest: digest}
}
for _, name := range c.Removed {
delete(change, name)
}
req := composer.Request{V: composer.Version, Input: plainInput(m.Assigned), Depth: composer.DepthResolve,
HoldingsFromClaims: true, Machines: Touched(m, c),
Cases: []composer.Case{{Name: CaseTrunk, Manifests: trunk}, {Name: CaseChange, Manifests: change}}}
return job(SDKRepository, c.SDKHead, SDKDir, req, "the mesh-sdk head of "+c.Group)
}
// ReaderJob is the compose job that reads the manifests a group changes with the mesh-sdk a reader module pins at
// commit (ADR 0482 clause 5, issue 1623): no machine is composed, every changed manifest is only read.
func ReaderJob(c Change, module, repository, commit string) (delivery.ComposeJob, error) {
changed := map[string]composer.Manifest{}
for name, digest := range c.Digests {
changed[name] = composer.Manifest{Digest: digest}
}
req := composer.Request{V: composer.Version, Cases: []composer.Case{{Name: CaseChange, Manifests: changed}}}
return job(repository, commit, delivery.RootPath, req, module+" at "+commit)
}
func job(repository, commit, dir string, req composer.Request, reader string) (delivery.ComposeJob, error) {
raw, err := json.Marshal(req)
if err != nil {
return delivery.ComposeJob{}, err
}
return delivery.ComposeJob{Fetch: map[string]any{"repository": repository, "commit": commit}, Dir: dir,
Package: composer.Package, Request: raw, Reader: reader}, nil
}
// plainInput is the composition's input without its catalogue: the nodes by name with what is assigned to them (the
// composer derives the seats held from the claims, as input does).
func plainInput(assigned map[string][]string) compose.Input {
var in compose.Input
nodes := make([]string, 0, len(assigned))
for n := range assigned {
nodes = append(nodes, n)
}
sort.Strings(nodes)
for _, n := range nodes {
in.Nodes = append(in.Nodes, compose.Node{Name: n, Assigned: slices.Clone(assigned[n])})
}
return in
}
// Ran is one compose ask's end: its step's state and reason, and the composer's verdict when the step passed.
type Ran struct {
State string
Reason string
Verdict *composer.Verdict
}
// RanOf reads a compose ask's end from its step value; a step that passed with an answer that cannot be read is a
// failure, named.
func RanOf(step delivery.Step) Ran {
r := Ran{State: step.State, Reason: step.Reason}
if step.State != delivery.StepPassed {
return r
}
var a delivery.ComposeAnswer
var v composer.Verdict
if err := json.Unmarshal(step.Result, &a); err != nil {
return Ran{State: delivery.StepFailed, Reason: "the compose step's answer cannot be read: " + err.Error()}
}
if err := json.Unmarshal(a.Verdict, &v); err != nil || v.V != composer.Version {
return Ran{State: delivery.StepFailed, Reason: fmt.Sprintf("the composer's verdict cannot be read (version %d): %v", v.V, err)}
}
r.Verdict = &v
return r
}
func caseOf(v *composer.Verdict, name string) (composer.CaseVerdict, bool) {
if v == nil {
return composer.CaseVerdict{}, false
}
for _, c := range v.Cases {
if c.Name == name {
return c, true
}
}
return composer.CaseVerdict{}, false
}
// Reader is one module whose mesh-sdk reads the group's manifests, at the commit judged: the group's member head when
// the group holds one in its repository, else a build its node runs.
type Reader struct {
Module, Commit string
Member bool
Ran Ran
}
// FromComposer is the composition check of a group with a mesh-sdk member, judged from the composer built at its head
// (ADR 0482 clauses 3 to 5, issues 1622 and 1623): the same failures as Compose, read from the composer's two cases, the
// verdict naming the mesh-sdk the composer was built with; a trunk manifest the head's mesh-sdk refuses fails too (the
// group would deliver a reader that refuses it); and each reader that refuses a changed manifest the head reads fails
// it, naming the module and its words, unless the reader's mesh-sdk has no composer to ask (before issue 1621), which is
// said and judged nothing.
func FromComposer(m Mesh, c Change, head Ran, readers []Reader, took time.Duration) delivery.CompositionVerdict {
v := delivery.CompositionVerdict{Group: c.Group, Heads: c.Heads, SDK: "mesh-sdk " + c.SDKHead, Took: took.Milliseconds()}
machines := Touched(m, c)
v.Machines = machines
var fails []string
if head.State != delivery.StepPassed {
v.Words = fmt.Sprintf("the composer at mesh-sdk %s did not run: %s %s", c.SDKHead, head.State, head.Reason)
v.State = delivery.CompositionFail
return v
}
if head.Verdict.SDK != "" {
v.SDK = head.Verdict.SDK
}
trunk, okT := caseOf(head.Verdict, CaseTrunk)
change, okC := caseOf(head.Verdict, CaseChange)
if !okT || !okC {
v.State, v.Words = delivery.CompositionFail, "the composer's verdict holds no trunk or no change case"
return v
}
failAt := func(machine, words string) {
if v.Machine == "" && machine != "" {
v.Machine = machine
}
fails = append(fails, words)
}
for _, name := range sortedKeys(c.Digests) {
if r := change.Manifests[name]; !r.Accepted {
failAt("", fmt.Sprintf("the manifest of %s cannot be read by mesh-sdk %s: %s", name, c.SDKHead, r.Refused))
}
}
for _, name := range sortedKeys(trunk.Manifests) {
if _, changed := c.Digests[name]; changed || slices.Contains(c.Removed, name) || trunk.Manifests[name].Accepted {
continue
}
if _, before := m.Unreadable[name]; before {
continue // already unreadable on the trunk: not the group's doing
}
failAt("", fmt.Sprintf("mesh-sdk %s refuses the trunk's manifest of %s, which the group does not change: %s", c.SDKHead, name, trunk.Manifests[name].Refused))
}
for _, name := range sortedKeys(c.Twice) {
failAt("", fmt.Sprintf("%s is defined twice, in %s", name, strings.Join(c.Twice[name], " and ")))
}
for _, node := range machines {
for _, name := range c.Removed {
if slices.Contains(m.Assigned[node], name) {
failAt(node, fmt.Sprintf("%s runs %s, which the change removes from its source", node, name))
}
}
before, now := trunk.Machines[node], change.Machines[node]
switch {
case now.Composes:
case !before.Composes:
v.Warnings = append(v.Warnings, fmt.Sprintf("%s does not compose on the trunk either, so the change is not blamed: %s", node, before.Reason))
default:
failAt(node, fmt.Sprintf("%s composes on the trunk and not with the change: %s", node, now.Reason))
}
}
readerFails, readerWarnings := judgeReaders(c, change, readers)
fails = append(fails, readerFails...)
v.Warnings = append(v.Warnings, readerWarnings...)
return finish(v, m, c, machines, fails)
}
// NoComposer says whether a compose step failed because the mesh-sdk it was built at has no composer command (a mesh-sdk
// from before issue 1621): Go's words for a package its module does not hold.
func NoComposer(reason string) bool {
for _, words := range []string{"cannot find package", "no required module provides package", "is not in std", "cannot find module providing package", "does not contain package"} {
if strings.Contains(reason, words) {
return true
}
}
return false
}
func judgeReaders(c Change, head composer.CaseVerdict, readers []Reader) (fails, warnings []string) {
for _, r := range readers {
at := r.Module + " at " + r.Commit
if r.Member {
at = r.Module + "'s member head " + r.Commit
}
if r.Ran.State != delivery.StepPassed {
if NoComposer(r.Ran.Reason) {
warnings = append(warnings, fmt.Sprintf("not judged whether %s reads the group's manifests: the mesh-sdk it pins has no "+
"composer command (before issue 1621)", at))
continue
}
fails = append(fails, fmt.Sprintf("the composer of the mesh-sdk %s pins did not run: %s %s", at, r.Ran.State, r.Ran.Reason))
continue
}
read, ok := caseOf(r.Ran.Verdict, CaseChange)
if !ok {
fails = append(fails, fmt.Sprintf("the composer of the mesh-sdk %s pins answered no change case", at))
continue
}
for _, name := range sortedKeys(c.Digests) {
if !head.Manifests[name].Accepted || read.Manifests[name].Accepted {
continue
}
fails = append(fails, fmt.Sprintf("the manifest of %s uses what %s (mesh-sdk %s) cannot read: %s; the group must move %s onto "+
"mesh-sdk %s", name, at, r.Ran.Verdict.SDK, read.Manifests[name].Refused, r.Module, c.SDKHead))
}
}
return fails, warnings
}
modules/mesh-delivery-assigner/internal/assign/composer_test.go
-184
View File
@@ -1,184 +0,0 @@
package assign
import (
"encoding/json"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/composer"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
var testTime = time.Date(2026, 10, 24, 3, 0, 0, 0, time.UTC)
const sdkHead = "1111111111111111111111111111111111111111"
// raws are the manifests of meshOf as their repositories hold them, by digest, as the indexer keeps them; `web2` is web
// with a field `shares` that the assigner's own mesh-sdk does not read (the shape of issue 1593).
var raws = map[string]string{
"d-mail": `{"module": "mail", "version": "1", "requires": ["postgres-database"]}`,
"d-store": `{"module": "store", "version": "1", "provides": [{"name": "postgres-database"}]}`,
"d-web": `{"module": "web", "version": "1"}`,
"d-web2": `{"module": "web", "version": "2", "shares": ["photos"]}`,
}
// asTheHeadReads is how mesh-sdk at the group's head reads each manifest: as the assigner's own reads it, but for the
// field it adds, which it accepts. The head's composer is built and run by the Go build module (ADR 0482 clause 4); here
// its verdict is made by mesh-sdk's composer.Run over the same request, with `web2` read as the head reads it.
var asTheHeadReads = map[string]string{"d-web2": `{"module": "web", "version": "2"}`}
func digestedMesh() Mesh {
m := meshOf()
m.Digests = map[string]string{"mail": "d-mail", "store": "d-store", "web": "d-web"}
return m
}
// webShares is a group of mesh-sdk and mesh-catalog whose catalogue member gives web the field only the SDK head reads:
// the assigner's own reader refuses it (Unreadable), as the indexer's manifest is read in world.change.
func webShares(members map[string]string) Change {
_, err := catalogue.ParseManifest([]byte(raws["d-web2"]))
return Change{Group: "g", Heads: "0123456789ab", SDKHead: sdkHead, Manifests: map[string]catalogue.Manifest{},
Unreadable: map[string]string{"web": err.Error()}, Digests: map[string]string{"web": "d-web2"}, Members: members}
}
// run is the composer's step for a job, as the build module answers it: the request filled from the indexer's digests
// (with read standing in for how that mesh-sdk reads a digest), run, and its verdict put in a passed step.
func run(t *testing.T, job delivery.ComposeJob, read map[string]string) Ran {
t.Helper()
req, err := composer.Decode(job.Request)
if err != nil {
t.Fatal(err)
}
if err := composer.Fill(&req, func(d string) ([]byte, error) {
if r, ok := read[d]; ok {
return []byte(r), nil
}
return []byte(raws[d]), nil
}); err != nil {
t.Fatal(err)
}
v, err := composer.Run(req)
if err != nil {
t.Fatal(err)
}
v.SDK = "(devel) " + job.Fetch["commit"].(string)
vr, _ := json.Marshal(v)
ans, _ := json.Marshal(delivery.ComposeAnswer{Built: true, Key: "composer-0", Verdict: vr})
return RanOf(delivery.Step{State: delivery.StepPassed, Result: ans})
}
// 1622/AC1: a group holding a mesh-sdk pull request is composed by the composer built at the member's head: the job
// fetches mesh-sdk at that head, builds mesh-sdk's composer command in its Go module, and hands it the trunk and the
// change by digest, never the manifests' bytes, composing the machines the change touches.
func TestAGroupWithAnSDKMemberAsksTheComposerAtItsHead(t *testing.T) {
c := webShares(nil)
job, err := HeadJob(digestedMesh(), c)
if err != nil {
t.Fatal(err)
}
if job.Fetch["repository"] != SDKRepository || job.Fetch["commit"] != sdkHead || job.Dir != "go" || job.Package != composer.Package {
t.Fatalf("job = %+v", job)
}
ask := delivery.StepAsk{ID: "mh3k2x9qa1b2c5", Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: "mesh-delivery-assigner",
At: testTime, Compose: &job}
if _, err := delivery.EncodeStepAsk(ask); err != nil {
t.Fatalf("the ask is refused by its layout: %v", err)
}
req, err := composer.Decode(job.Request)
if err != nil {
t.Fatal(err)
}
if len(req.Cases) != 2 || req.Cases[0].Name != CaseTrunk || req.Cases[1].Name != CaseChange {
t.Fatalf("cases = %+v", req.Cases)
}
if req.Cases[0].Manifests["web"].Digest != "d-web" || req.Cases[1].Manifests["web"].Digest != "d-web2" || len(req.Cases[1].Manifests["web"].Raw) != 0 {
t.Errorf("the cases do not name web by its trunk and its change digest: %+v", req.Cases)
}
if !slices.Equal(req.Machines, []string{"g14"}) || req.Depth != composer.DepthResolve || !req.HoldingsFromClaims {
t.Errorf("request = machines %v, depth %q, holdings from claims %v", req.Machines, req.Depth, req.HoldingsFromClaims)
}
}
// 1622/AC2, AC3: a catalogue member using a field only the SDK head reads passes with the head's composer, and the
// verdict names the mesh-sdk the composer was built with; with the assigner's own mesh-sdk it fails.
func TestAFieldTheGroupAddsPassesWithTheHeadsComposer(t *testing.T) {
c := webShares(map[string]string{"novox/mesh-controller": "2222222222222222222222222222222222222222"})
job, _ := HeadJob(digestedMesh(), c)
head := run(t, job, asTheHeadReads)
v := FromComposer(digestedMesh(), c, head, nil, 0)
if v.State != delivery.CompositionPass || v.SDK != "(devel) "+sdkHead || !slices.Equal(v.Machines, []string{"g14"}) {
t.Fatalf("with the head's composer: %+v", v)
}
if _, err := delivery.EncodeCompositionVerdict(v); err != nil {
t.Errorf("the verdict cannot be written: %v", err)
}
own := Compose(digestedMesh(), c, "v0.1.14-own", 0)
if own.State != delivery.CompositionFail || !strings.Contains(own.Words, "shares") {
t.Fatalf("with the assigner's own mesh-sdk: %+v, want a fail naming the field", own)
}
}
// 1623/AC1, AC2: the group of mesh-sdk and mesh-catalog adding and using a field, without a controller member, fails
// naming the field and mesh-controller, whose running build pins an older mesh-sdk; with a controller member pinned to
// the SDK head (its composer reads what the head reads) it passes.
func TestAFieldTheRunningControllerCannotReadFailsTheGroup(t *testing.T) {
c := webShares(nil)
job, _ := HeadJob(digestedMesh(), c)
head := run(t, job, asTheHeadReads)
running := "3333333333333333333333333333333333333333"
rjob, err := ReaderJob(c, "mesh-controller", "novox/mesh-controller", running)
if err != nil {
t.Fatal(err)
}
if rjob.Fetch["commit"] != running || rjob.Dir != delivery.RootPath {
t.Fatalf("reader job = %+v", rjob)
}
old := run(t, rjob, nil) // the running controller's mesh-sdk reads web2 as the assigner's own does: refused
v := FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-controller", Commit: running, Ran: old}}, 0)
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "shares") || !strings.Contains(v.Words, "move mesh-controller onto mesh-sdk "+sdkHead) {
t.Fatalf("without a controller member: %+v, want a fail naming the field and mesh-controller", v)
}
member := "2222222222222222222222222222222222222222"
pinned := run(t, rjob, asTheHeadReads)
v = FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-controller", Commit: member, Member: true, Ran: pinned}}, 0)
if v.State != delivery.CompositionPass {
t.Fatalf("with a controller member pinned to the head: %+v, want a pass", v)
}
}
// 1623: a reader whose mesh-sdk has no composer command (before issue 1621) is said, not failed; a reader's composer that
// failed for another cause fails the group, loud.
func TestAReaderWithNoComposerIsSaidAndAnotherFailureFails(t *testing.T) {
c := webShares(nil)
job, _ := HeadJob(digestedMesh(), c)
head := run(t, job, asTheHeadReads)
none := Ran{State: delivery.StepFailed, Reason: "go build: no required module provides package git.novox.be/novox/mesh-sdk/go/cmd/mesh-compose"}
v := FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-host", Commit: "abc", Ran: none}}, 0)
if v.State != delivery.CompositionPass || !hasWarning(v, "mesh-host at abc") || !hasWarning(v, "before issue 1621") {
t.Fatalf("a reader with no composer: %+v", v)
}
broke := Ran{State: delivery.StepFailed, Reason: "the fetch could not reach the forge"}
v = FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-host", Commit: "abc", Ran: broke}}, 0)
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "could not reach the forge") {
t.Fatalf("a reader whose composer failed: %+v", v)
}
}
// 1622: the head's composer that did not run fails the composition naming why, never passes it; a trunk manifest the
// head's mesh-sdk refuses fails it too, since the group would deliver a reader that refuses it.
func TestTheHeadsComposerFailingOrRefusingTheTrunkFails(t *testing.T) {
c := webShares(nil)
v := FromComposer(digestedMesh(), c, Ran{State: delivery.StepRefused, Reason: "the guard refuses it"}, nil, 0)
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "the guard refuses it") || !strings.Contains(v.Words, sdkHead) {
t.Fatalf("a head composer that did not run: %+v", v)
}
job, _ := HeadJob(digestedMesh(), c)
strict := map[string]string{"d-web2": asTheHeadReads["d-web2"], "d-mail": `{"module": "mail", "version": "1", "gone": true}`}
v = FromComposer(digestedMesh(), c, run(t, job, strict), nil, 0)
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "refuses the trunk's manifest of mail") {
t.Fatalf("a head that refuses a trunk manifest: %+v", v)
}
}
modules/mesh-delivery-assigner/internal/assign/put.go
-252
View File
@@ -1,252 +0,0 @@
package assign
import (
"fmt"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// A put (novox/hq ADR 0546): one version of a module put on one node by the assigner's tool assigner_put. The tool writes the put
// key; each pass takes it one step further from what the world holds (StepPut), and the rules pass over the node's want for
// the module from its asking until it ends (Module.Puts).
// ComposeWithin is how long a put waits for the composition and grant it needs (a take-in of a build just done writes them)
// before it is refused.
const ComposeWithin = 10 * time.Minute
// PutWorld is what one step of a put reads.
type PutWorld struct {
Node, Module string
Put delivery.Put
// Assigned says the module stands assigned to the node.
Assigned bool
// Bus says the module provides the mesh's bus (ADR 0539 §15); Recorded that it is of policy record (§14).
Bus, Recorded bool
// Build is the state of the builder's build at the put's fingerprint, "" when no key is written; BuildWords its error.
Build, BuildWords string
// Newest is the module's newest fingerprint at the trunk, which the builder builds by itself.
Newest string
// Composed is the composition of the module for the node at the fingerprint, as its writer holds it; nil when there is
// none. Granted says the grant at the fingerprint names the node's accounts. Both are read, never asked for (clause 4).
Composed *delivery.ComposedDigest
Granted bool
// Answer is the option the operator chose on this put's own ask (its asking, PutAsking), "" when none; AskExpired says
// that ask expired unanswered.
Answer string
AskExpired bool
// Snapshot says where the bus's streams were snapshotted for this put, once they were; Snapshotting that it was
// asked; SnapshotFailed why it was not taken.
Snapshot string
Snapshotting bool
SnapshotFailed string
// Have is what the node says it runs of the module; nil when it says nothing.
Have *delivery.Have
Now time.Time
}
// PutNext is what a step decides: the put as it stands after it, and the acts the pass does for it.
type PutNext struct {
Put delivery.Put
// Ask writes the put's ask about the move (PutAsk).
Ask bool
// Snapshot has the bus's node back the bus up now, its dump the streams' snapshot (ADR 0235).
Snapshot bool
// Warranted records, once, in the controller's hand-act log that the bus's want was written on the operator's warrant.
Warranted bool
}
// StepPut takes a put one step further from what the world holds (ADR 0546 clauses 3 to 8).
func StepPut(w PutWorld) PutNext {
p := w.Put
if delivery.PutEnded(p.Step) {
return PutNext{Put: p}
}
say := func(step, format string, args ...any) PutNext {
return PutNext{Put: p.Said(w.Now, step, fmt.Sprintf(format, args...))}
}
fp := short(p.FP)
if !w.Assigned {
return say(delivery.PutRefused, "%s is not assigned to %s: a put moves a module where it stands assigned (mesh-controller.assign first)", w.Module, w.Node)
}
switch w.Build {
case delivery.BuildDone:
case delivery.BuildFailed:
return say(delivery.PutRefused, "the builder's build of %s at %s failed: %s", w.Module, fp, w.BuildWords)
case "":
if p.FP != w.Newest {
// The builder builds the trunk's newest fingerprint and the candidates', and takes no ask for another yet (ADR 0546
// clause 3): an older fingerprint it never built, or whose key it removed, cannot be put.
return say(delivery.PutRefused, "the builder holds no build of %s at %s, and builds by itself only the trunk's newest (%s): put that, or a version it built", w.Module, fp, short(w.Newest))
}
return say(delivery.PutBuild, "waits for the builder's build of %s at %s, the trunk's newest, which it builds by itself", w.Module, fp)
default:
return say(delivery.PutBuild, "waits for the builder's build of %s at %s, which is %s", w.Module, fp, w.Build)
}
if w.Composed == nil || !w.Granted {
missing := fmt.Sprintf("no composition of %s for %s at %s is written", w.Module, w.Node, fp)
if w.Composed != nil {
missing = fmt.Sprintf("the grant of %s at %s does not name %s's accounts", w.Module, fp, w.Node)
}
since := w.Now
if p.Step == delivery.PutCompose && len(p.Steps) > 0 {
since = p.Steps[len(p.Steps)-1].At
}
if w.Now.Sub(since) > ComposeWithin {
return say(delivery.PutRefused, "%s after %s: only a composition written by the take-in of its build (or one of the five kept) is put; "+
"for another the hand path is mesh-hand-compose (ADR 0539 §10)", missing, ComposeWithin)
}
return say(delivery.PutCompose, "waits up to %s: %s yet", ComposeWithin, missing)
}
d := *w.Composed
if p.Step == delivery.PutWanted || p.Step == delivery.PutHolding {
return PutNext{Put: applied(w, p)}
}
p.Composed = d.Digest
if needsPerson(w, d) {
switch w.Answer {
case OptionNo:
return say(delivery.PutRefused, "the operator answered no to putting %s at %s on %s", w.Module, fp, w.Node)
case OptionYes:
default:
if w.AskExpired {
return say(delivery.PutRefused, "the operator's ask about putting %s at %s on %s expired unanswered: the node keeps what it runs", w.Module, fp, w.Node)
}
next := say(delivery.PutPerson, "waits for the operator's yes to putting %s at %s on %s (composition %s, judged %s)", w.Module, fp, w.Node, short(d.Digest), d.Judgement)
if p.Ask == "" {
next.Ask = true
next.Put.Ask, _ = delivery.RecordAskKey(w.Node, w.Module, d.Digest)
}
return next
}
if w.Bus {
switch {
case w.Snapshot != "":
p.Snapshot = w.Snapshot
case w.SnapshotFailed != "":
return say(delivery.PutRefused, "the bus's streams were not snapshotted, so the bus is not put: %s", w.SnapshotFailed)
case w.Snapshotting:
return say(delivery.PutSnapshot, "the operator said yes; waits for %s's backup holder to snapshot the bus's streams", w.Node)
default:
next := say(delivery.PutSnapshot, "the operator said yes; asked %s's backup holder to snapshot the bus's streams", w.Node)
next.Snapshot = true
return next
}
}
}
words := fmt.Sprintf("the want of %s on %s names %s (composition %s)", w.Module, w.Node, fp, short(d.Digest))
if p.Hold {
words += ", held until released"
}
if p.Snapshot != "" {
words += "; the streams were snapshotted: " + p.Snapshot
}
next := PutNext{Put: applied(w, p.Said(w.Now, delivery.PutWanted, words))}
next.Warranted = w.Bus && p.Ask != ""
return next
}
// applied is a wanted put once the node's have says how it went: holding or done when it runs the composition healthy.
func applied(w PutWorld, p delivery.Put) delivery.Put {
h := w.Have
if h == nil || h.FP != p.FP || h.Composed != p.Composed {
return p
}
switch {
case h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK && p.Hold:
return p.Said(w.Now, delivery.PutHolding, fmt.Sprintf("%s runs %s at %s healthy; held there until released", w.Node, w.Module, short(p.FP)))
case h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK:
return p.Said(w.Now, delivery.PutDone, fmt.Sprintf("%s runs %s at %s healthy; the rules write its want again", w.Node, w.Module, short(p.FP)))
case failed(*h):
return p.Said(w.Now, p.Step, fmt.Sprintf("%s runs %s at %s and its health check failed: %s", w.Node, w.Module, short(p.FP), h.Detail))
}
return p
}
// needsPerson says whether the put's move waits for the operator: any move of the bus but a revision at the fingerprint the
// node runs that changes only its user list (ADR 0539 §7, §15), and a move of a module of policy record judged to recreate a
// container, move data, or that cannot be judged (§14).
func needsPerson(w PutWorld, d delivery.ComposedDigest) bool {
runs := w.Have != nil && w.Have.FP == w.Put.FP
if runs && w.Have.Composed == d.Digest {
return false // the node runs this composition already: nothing moves
}
if w.Bus {
return !(runs && d.UserListOnly)
}
if !w.Recorded {
return false
}
return d.Judgement == delivery.JudgedRecreates || d.Judgement == delivery.JudgedMovesData || d.Judgement == delivery.JudgedCannot
}
// PutWant is the want a put writes once it is wanted: the fingerprint and composition, marked put, and held when it holds.
func PutWant(p delivery.Put) delivery.Want {
return delivery.Want{FP: p.FP, Composed: p.Composed, Put: true, Hold: p.Hold, Words: "put: " + p.Why}
}
// PutOwns says what a put does to the node's want: owns says the rules pass over it, want the want it writes (nil: what
// stands stays). A put owns the want from its asking until it ends; a held put ends only when released.
func PutOwns(p delivery.Put) (want *delivery.Want, owns bool) {
switch p.Step {
case delivery.PutWanted, delivery.PutHolding:
w := PutWant(p)
return &w, true
case delivery.PutDone, delivery.PutReleased, delivery.PutRefused:
return nil, false
}
return nil, true
}
// PutAsking is a put's own asking number: when it was asked, in milliseconds. Each put asked under the same key is another
// asking, so the ask is written again under an id of its own and an answer to an earlier put, whose warrant names that put's
// asking, never counts for a later one (ADR 0546 clause 8: one approval binds one act).
func PutAsking(p delivery.Put) int {
if n := p.At.UnixMilli(); n > 0 {
return int(n)
}
return 0
}
// PutAct is the act a bus put's yes binds and that is performed when its want is written.
func PutAct(node, module, fp, composed string) asks.Act {
return asks.Act{"put": "want", "node": node, "module": module, "fp": fp, "composed": composed}
}
// PutAsk is the ask about a put's move (ADR 0546 clauses 7 and 8), under the record ask's key of the composition so the
// teller relays it and its answer as any record ask's. For the bus its yes is of level approve and binds the act, so only a
// channel that proves who answered takes it (ADR 0259), never the view.
func PutAsk(key, node, module string, p delivery.Put, d delivery.ComposedDigest, bus bool, now time.Time) (delivery.AssignerAsk, error) {
yes := asks.Option{ID: OptionYes, Label: "Yes, put it", Does: "the node is told to apply " + module + " at " + short(p.FP), Level: asks.Acknowledge}
what := map[delivery.Judgement]string{
delivery.JudgedRecreates: "recreates a container",
delivery.JudgedMovesData: "moves the module's data",
delivery.JudgedCannot: "cannot be judged: " + d.Words,
delivery.JudgedNeither: "recreates no container and moves no data",
delivery.JudgedUnchanged: "changes nothing the node runs",
}[d.Judgement]
explanation := fmt.Sprintf("Asked to put %s at version %s on %s: %s. The move %s.", module, short(p.FP), node, p.Why, what)
expires := now.Add(7 * 24 * time.Hour)
if bus {
binds, err := asks.ActDigest(PutAct(node, module, p.FP, d.Digest))
if err != nil {
return delivery.AssignerAsk{}, err
}
yes.Level, yes.Binds = asks.Approve, binds
yes.Does = "the bus's streams are snapshotted, then " + node + " is told to apply the bus at " + short(p.FP)
explanation += " It is the mesh's bus: every module reconnects while it restarts, and its streams are snapshotted first."
expires = now.Add(asks.ApproveLasts)
}
return delivery.AssignerAsk{Ask: asks.Ask{
ID: delivery.AskingID(key, PutAsking(p)),
Headline: fmt.Sprintf("Put %s on %s?", module, node),
Explanation: explanation,
Options: []asks.Option{yes,
{ID: OptionNo, Label: "No", Does: "the node keeps what it runs, and the put is refused", Level: asks.Acknowledge}},
Who: asks.Operator,
Expires: expires,
OnExpiry: "the node keeps what it runs",
About: key,
}, Asking: PutAsking(p)}, nil
}
modules/mesh-delivery-assigner/internal/assign/wants.go
+30 -324
View File
@@ -4,8 +4,6 @@ import (
"fmt"
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
@@ -15,44 +13,13 @@ import (
// trunk (the indexer's modules), whether the builder's build of F is done, whether F is held, whether the controller
// issued its credential for F, its manifest at F, and the nodes it is assigned to.
type Module struct {
Name string
FP string
Built bool
Held bool
Granted bool
// GrantedAt says whether the controller issued the module's credential for another fingerprint than FP: the first want of a node
// that runs an older one names it (issue 1945). Nil: none.
GrantedAt func(fp string) bool
// GrantedOn says whether the controller's grant of FP names the module's accounts on a node, when the grant names any:
// a grant written for the module's other nodes says nothing of a node whose account is not issued and sealed (issue
// 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus). Nil: the grant alone.
GrantedOn func(node string) bool
Manifest catalogue.Manifest
Nodes []string
// Composed answers the controller's composition of the module for a node at a fingerprint (mesh-controller_composed): its
// digest and the judgement of the move to it, and whether it exists. Nil: none exists (ADR 0539 §1, §6).
Composed func(node, fp string) (delivery.ComposedDigest, bool)
// Allowed says whether the operator answered yes to the record ask about the move of this module on node to the composition
// with this digest (ADR 0539 §14). Nil: none did.
Allowed func(node, digest string) bool
// Released is a hold on FP the operator released (ADR 0539 §19): FP is not held again by a health check that failed.
Released bool
// Restored says whether the operator answered "restore" to the restore ask of this data module on node: it is then set back
// as any module is, losing the writes made since (ADR 0539 §19). Nil: none did.
Restored func(node string) bool
// Kept says the operator answered "keep" to the hold ask of FP: a hold kept so is not lifted by itself.
Kept bool
// Now is when this pass reads the haves: the held record names it as when the health check was seen to fail. Zero: not said.
Now time.Time
// Puts are the puts of the module, by node (ADR 0546): while one owns a node's want (PutOwns), the rules pass over it.
Puts map[string]delivery.Put
}
func (m Module) composition(node, fp string) (delivery.ComposedDigest, bool) {
if m.Composed == nil {
return delivery.ComposedDigest{}, false
}
return m.Composed(node, fp)
Name string
FP string
Built bool
Held bool
Granted bool
Manifest catalogue.Manifest
Nodes []string
}
// Node is what the rules read of the nodes: whether each is online (the watchman's presence) and each module's have.
@@ -84,43 +51,27 @@ func Canary(nodes []string, online map[string]bool) string {
// with why (ADR 0429 §6). Rule 4, at most `step` others ahead, always holds: the mesh keeps no step, and step 0 is no
// limit (ADR 0528 clause 3).
func Holds(m Module, node string, n Nodes) (bool, int, string) {
return holds(m, node, n, true)
}
// holds is Holds; with ordering false it leaves out the two rules that order a move among nodes and modules, the canary (3)
// and the providers (5), and keeps those without which there is nothing to want: the build done, not held, the grant and the
// composition.
func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
switch {
case !m.Built:
return false, delivery.RuleBuildDone, fmt.Sprintf("the build of %s at %s is not done", m.Name, short(m.FP))
case m.Held:
return false, delivery.RuleNotHeld, fmt.Sprintf("%s is held at %s", m.Name, short(m.FP))
}
if ordering {
canary := Canary(m.Nodes, n.Online)
switch {
case canary == "":
return false, delivery.RuleCanary, fmt.Sprintf("none of the nodes of %s is online, so it has no canary", m.Name)
case node != canary && !healthy(n.Haves[canary][m.Name], m.FP):
return false, delivery.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP))
}
for _, provision := range m.Manifest.Requires {
if ok, why := providedHealthy(provision, node, n); !ok {
return false, delivery.RuleProviders, why
}
canary := Canary(m.Nodes, n.Online)
switch {
case canary == "":
return false, delivery.RuleCanary, fmt.Sprintf("none of the nodes of %s is online, so it has no canary", m.Name)
case node != canary && !healthy(n.Haves[canary][m.Name], m.FP):
return false, delivery.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP))
}
for _, provision := range m.Manifest.Requires {
if ok, why := providedHealthy(provision, node, n); !ok {
return false, delivery.RuleProviders, why
}
}
if !m.Granted {
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s for %s", m.Name, short(m.FP))
}
if m.GrantedOn != nil && !m.GrantedOn(node) {
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s on %s for %s", m.Name, node,
short(m.FP))
}
if _, ok := m.composition(node, m.FP); !ok {
return false, delivery.RuleComposed, fmt.Sprintf("the controller has not composed %s for %s at %s", m.Name, node, short(m.FP))
}
return true, 0, ""
}
@@ -166,30 +117,8 @@ type Outcome struct {
Wants map[string]delivery.Want
// Held is the hold of m.FP to write, when a node's health check failed on it and it is not held yet.
Held *delivery.Held
// Lifted is the hold of m.FP lifted by itself: every node that runs the module runs m.FP healthy (issue 1906). The held
// record and its ask are taken away, and the wants move by the rules again in this pass.
Lifted bool
// Record is the moves a person must allow before the want names them: each is asked once (ADR 0539 §14).
Record []RecordNeed
// Restore is the nodes of a module holding data whose held fingerprint failed its health check there and that are not set
// back: the operator is asked whether to restore each (ADR 0539 §19).
Restore []string
// Rules is, by node, the rule holding the node back from m.FP and why.
// Rules is, by node, the rule holding the node back from m.FP and why, for the comparison.
Rules map[string]Holding
// Neither is, by node, the nodes the module is assigned to that run it (or whose have names it), hold no want for it after this
// pass and are not held (a held module is on the held list), with the rule holding the first want back: the node's own composition does not hold a module with a fingerprint of the new form, so such a module is on
// neither list and nothing applies it on that node (issue 1945). A module with no fingerprint of the new form goes with the node's
// own composition and is never here.
Neither map[string]Holding
}
// RecordNeed is a move of a module of policy record on a node that waits for the operator's yes: the composition's digest, the
// controller's judgement of the move and its words.
type RecordNeed struct {
Node string
Digest string
Judgement delivery.Judgement
Words string
}
// Holding is a rule holding a node back, and why.
@@ -201,18 +130,15 @@ type Holding struct {
// Wants is the pass of one module m: for each of its nodes, the want by the six rules, starting from the want written
// before (wants, by node); lastHealthy is, by node, the newest fingerprint other than m.FP the node ran healthy. A failed
// health check of m.FP on a node holds it: every node wanting m.FP is set back to its last healthy fingerprint, but a
// module holding data is not set back but asked about (Restore). A
// module holding data is not set back (in release 5 it is asked about from release 6 on: wants are only compared). A
// node that wants m but is no longer among its nodes gets its want written removed, never deleted (ADR 0408 §4).
func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[string]string) Outcome {
out := Outcome{Wants: map[string]delivery.Want{}, Rules: map[string]Holding{}, Neither: map[string]Holding{}}
out := Outcome{Wants: map[string]delivery.Want{}, Rules: map[string]Holding{}}
held := m.Held
if held && !m.Kept && m.FP != "" && allRunHealthy(m, n) {
out.Lifted, held = true, false
}
if !held && !out.Lifted && !m.Released && m.FP != "" {
if !held && m.FP != "" {
for _, node := range slices.Sorted(slices.Values(m.Nodes)) {
if h, ok := n.Haves[node][m.Name]; ok && h.FP == m.FP && failed(h) {
out.Held = &delivery.Held{Node: node, Words: HeldWords(h, m.Now)}
if h, ok := n.Haves[node][m.Name]; ok && h.FP == m.FP && h.Health == delivery.HealthFailed {
out.Held = &delivery.Held{Node: node, Words: h.Detail}
held = true
break
}
@@ -226,128 +152,30 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
}
before, had := wants[node]
var next delivery.Want
if p, put := m.Puts[node]; put {
if want, owns := PutOwns(p); owns {
if want != nil {
w := *want
// A held bus still takes a revision of its composition that changes only its user list (ADR 0539 §7).
if ul, ok := m.userListRevision(node, before, had, n); ok && ul.FP == w.FP {
w.Composed = ul.Composed
}
if !had || !delivery.SameWant(normal(before), normal(w)) {
out.Wants[node] = w
}
}
continue
}
}
// A revision of the bus module's composition that differs only in its user list is written at once, at the fingerprint the
// node runs, outside the rules, the record hold and the bar on a new fingerprint: a new module's credential must reach the
// bus before the module can connect (ADR 0539 §7). The controller marks it; whatever it judges of the file's mount stands.
if w, ok := m.userListRevision(node, before, had, n); ok {
if !had || !delivery.SameWant(normal(before), normal(w)) {
out.Wants[node] = w
}
continue
}
ok, rule, words := Holds(mm, node, n)
target := m.FP
if have, ran := n.Haves[node][m.Name]; !ok && (!had || before.Removed) && (rule == delivery.RuleCanary || rule == delivery.RuleProviders) && runs(have) {
// A node that runs the module and has no want for it has no want the ordering rules could hold it back to: holding its first
// want back leaves the module on neither list, since the node's own composition gives up every module with a fingerprint of
// the new form (issue 1945: on g14 the bundles of clipmenu, screen-lock and every module requiring x11-display were removed
// and never made again, because xorg's health is unknown and screen-lock's canary never ran it healthy). Its first want names
// what it runs, never a newer fingerprint, so the canary is never passed over: F when it runs F (by the rules without which
// there is nothing to want), or an older fingerprint of the new form whose composition and grant exist. Rules 3 and 5 then
// order the move from it as from any want.
switch _, composed := m.composition(node, have.FP); {
case !ran:
case have.FP == m.FP:
ok, rule, words = holds(mm, node, n, false)
case !strings.HasPrefix(have.FP, delivery.SourcePrefix2):
// An old-form build (not src2:) has no composition, so no want can ever name it: its first want names F, and only by every
// rule as usual, the canary (3) and the providers (5) included, which here hold it back. Nothing is written and the module
// stays on neither list until the canary runs F healthy; then Holds passes and the want names F (issue 1945, a design
// limit set by the lead: the canary stays in force and old-form modules leave neither list by themselves).
case composed && m.GrantedAt != nil && m.GrantedAt(have.FP):
ok, target = true, have.FP
}
}
switch {
switch ok, rule, words := Holds(mm, node, n); {
case ok:
d, _ := m.composition(node, target)
next = delivery.Want{FP: target, Composed: d.Digest}
at := m
if target != m.FP {
// the first want at the older fingerprint the node runs: held back from F by the rule, and judged as a want at it
next.Newest, next.Rule, next.Words = m.FP, rule, words
at.FP = target
}
if had && !before.Removed && !before.Put && before.FP == target && before.Composed == d.Digest {
next.Backup, next.Words = before.Backup, before.Words // the answer to a failed backup stays while its target stays (ADR 0539 §16)
}
running, ran := n.Haves[node][m.Name]
// A removed want is no want: a first want after it goes through the record ask as a first want does.
asked, hadLive := before, had
if before.Removed {
asked, hadLive = delivery.Want{}, false
}
if need, wait := at.mustAsk(node, asked, hadLive, ran, running.FP, running.Composed == "", d); wait {
if need != nil {
out.Record = append(out.Record, *need)
}
continue
}
next = delivery.Want{FP: m.FP}
case held && had && before.FP == m.FP:
out.Rules[node] = Holding{Rule: rule, Words: words}
last := lastHealthy[node]
switch {
case keepsData(m.Manifest) && (m.Restored == nil || !m.Restored(node)):
case keepsData(m.Manifest):
next = before
next.Words = fmt.Sprintf("%s holds data, so it is not set back from %s: the operator is asked whether to restore it", m.Name, short(m.FP))
out.Restore = append(out.Restore, node)
next.Words = fmt.Sprintf("%s holds data, so it is not set back from %s: the operator is asked from release 6", m.Name, short(m.FP))
case last == "":
next = before
next.Words = fmt.Sprintf("%s is held at %s and %s ran no other fingerprint healthy to set it back to", m.Name, short(m.FP), node)
default:
d, there := m.composition(node, last)
if !there {
next = before
next.Words = fmt.Sprintf("%s is held at %s, and the composition of %s at %s is gone, so %s is not set back", m.Name, short(m.FP), m.Name, short(last), node)
break
}
next = delivery.Want{FP: last, Composed: d.Digest, Newest: m.FP, Rule: rule, Words: words}
next = delivery.Want{FP: last, Newest: m.FP, Rule: rule, Words: words}
}
default:
out.Rules[node] = Holding{Rule: rule, Words: words}
if !had || before.Removed {
// The node runs the module, or its have names it, and gets no want for it: the module is on neither list (issue 1945), and
// the pass says so. A new assignment, with no have, waits for the rules and is on no list yet by design. A held module (rule 2,
// held before or by this pass) is on the held list instead, whose hold is said and answered on its own: it is never on neither
// list (issue 1945: systemd-resolved, held on shanks by the operator, was said as urgent every hour).
if h, there := n.Haves[node][m.Name]; there && h.Phase != delivery.PhaseRemoved && !held {
out.Neither[node] = Holding{Rule: rule, Words: words}
}
continue
}
next = delivery.Want{FP: before.FP, Composed: before.Composed, Newest: m.FP, Rule: rule, Words: words}
next = delivery.Want{FP: before.FP, Newest: m.FP, Rule: rule, Words: words}
if before.FP == m.FP {
// Held at F by the rules: a new revision of F's composition waits for them, as a new fingerprint does (ADR 0539 §5).
next = delivery.Want{FP: m.FP, Composed: before.Composed}
} else if d, there := m.composition(node, before.FP); there && d.Digest != "" && d.Digest != before.Composed {
// Held back at an earlier fingerprint whose kept composition the controller composed again with today's config
// (novox/hq ADR 0545): the want follows its new revision, so the node does not run that build with the config of
// the moment it was made (on g14 a capacity of 0 after a rollback). Asked as any move of a record module is.
at := m
at.FP = before.FP
running, ran := n.Haves[node][m.Name]
need, wait := at.mustAsk(node, before, true, ran, running.FP, running.Composed == "", d)
if need != nil {
out.Record = append(out.Record, *need)
}
if !wait {
next.Composed = d.Digest
}
next = delivery.Want{FP: m.FP}
}
}
if !had || !delivery.SameWant(normal(before), normal(next)) {
@@ -362,58 +190,6 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
return out
}
// failed says whether a have's health check failed once the node is done moving: phase running or failed (never removed, held, or
// moving). A failed health while it
// backs up, applies or checks is a step of the move, not its outcome (on novox at 15:27:36 on 2026-10-25, "health failed, phase
// applying" held the controller's canary two seconds before the new build was healthy). Every healthy read (healthy, providedHealthy,
// the world's lastHealthy) already asks for phase running.
func failed(h delivery.Have) bool {
return h.Health == delivery.HealthFailed && (h.Phase == delivery.PhaseRunning || h.Phase == delivery.PhaseFailed)
}
// runs says whether a have is of a module the node runs: running, whatever its health.
func runs(h delivery.Have) bool {
return h.Phase == delivery.PhaseRunning
}
// allRunHealthy says whether every node of m that runs the module (its have's phase is running, or failed) runs m.FP healthy, and
// at least one does: a hold whose fingerprint runs healthy wherever the module runs holds nothing back any more (issue 1906: seven
// modules stayed held on every machine after one transient failed check). A node set back to another fingerprint does not run
// m.FP, and a failed one does not run it healthy, so the hold stays; a have that is removed, held for its backup or still moving
// (backing up, applying, checking) is not counted, or it would keep the hold up for ever.
func allRunHealthy(m Module, n Nodes) bool {
ran := 0
for _, node := range m.Nodes {
h, ok := n.Haves[node][m.Name]
if !ok || (h.Phase != delivery.PhaseRunning && h.Phase != delivery.PhaseFailed) {
continue
}
if !healthy(h, m.FP) {
return false
}
ran++
}
return ran > 0
}
// HeldWords are the words a hold keeps of its cause (issue 1906): the health and phase the node reported, when the assigner saw
// it, and the node's own words. The node is the held record's own field.
func HeldWords(h delivery.Have, at time.Time) string {
words := fmt.Sprintf("health %s, phase %s", h.Health, h.Phase)
if !at.IsZero() {
words += ", seen at " + at.UTC().Format(time.RFC3339)
}
if d := strings.TrimSpace(h.Detail); d != "" {
return words + ": " + d
}
return words + ": the node gave no words"
}
// LiftedWords say why a hold was lifted by itself.
func LiftedWords(module, fp string) string {
return fmt.Sprintf("every node that runs %s runs %s healthy, so the assigner lifted its hold by itself", module, short(fp))
}
// normal is a want without its version, as a decoded and a composed one are compared.
func normal(w delivery.Want) delivery.Want {
w.V = 0
@@ -427,73 +203,3 @@ func short(fp string) string {
}
return part[:12]
}
// mustAsk says whether the move to the composition d on node waits: for the operator's yes (need names the ask) or, for the bus,
// for the operator's hand (ADR 0539 §14, §15). A module the node never ran (it has no want and no have), a move judged neither, a module that is not of
// policy record and a want that stays are written as any want; so is a revision of the bus module's composition at the
// same fingerprint judged neither (a user list only, §7).
func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, running string, haveNamesNoComposition bool, d delivery.ComposedDigest) (*RecordNeed, bool) {
if (!had && !ran) || before.Removed || (before.FP == m.FP && before.Composed == d.Digest) {
return nil, false
}
// Right after the switch no node has a readable want, and a have from the old node-engine names a fingerprint and no composition
// (ADR 0539 §9, §11): a node running this fingerprint is given its want at it with no hold when the controller judges the composition
// unchanged, equal to the declaration the node last applied (ADR 0540 §9), since the node agent then applies nothing. A composition that
// differs from it (a changed setting or secret), or that the controller could not judge, waits for the ask. A new fingerprint still waits.
if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == delivery.JudgedUnchanged {
return nil, false
}
// The fingerprint the bus ran before: its want's, or with no readable want the one its have names (ADR 0539 §15 bars a new one).
prev := before.FP
if !had {
prev = running
}
if catalogue.ProvidesBus(m.Manifest) && prev != m.FP {
return nil, true // the bus is never moved by the assigner; the operator writes its want (§15)
}
if !m.recorded(d) || d.Judgement == delivery.JudgedNeither || d.Judgement == delivery.JudgedUnchanged {
return nil, false
}
if m.Allowed != nil && m.Allowed(node, d.Digest) {
return nil, false
}
return &RecordNeed{Node: node, Digest: d.Digest, Judgement: d.Judgement, Words: d.Words}, true
}
// recorded says whether the module is of policy record: as the controller holds it beside the composition's digest, the
// operator's own choice included (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it.
func (m Module) recorded(d delivery.ComposedDigest) bool {
if d.Policy != "" {
return d.Policy == delivery.PolicyRecord
}
policy, _, _ := catalogue.DerivedUpgrade(m.Manifest)
return policy == catalogue.PolicyRecord
}
// userListRevision is the want that names the bus module's composition at the fingerprint the node runs when that composition
// differs from the one the node runs only in the bus's user list (ComposedDigest.UserListOnly), and whether there is one. A node
// whose want names another fingerprint than the one it runs, or a module that does not provide the bus, has none.
func (m Module) userListRevision(node string, before delivery.Want, had bool, n Nodes) (delivery.Want, bool) {
if !catalogue.ProvidesBus(m.Manifest) {
return delivery.Want{}, false
}
running, ran := n.Haves[node][m.Name]
if !ran || running.FP == "" || (had && (before.Removed || before.FP != running.FP)) {
return delivery.Want{}, false
}
d, ok := m.composition(node, running.FP)
if !ok || !d.UserListOnly {
return delivery.Want{}, false
}
w := delivery.Want{FP: running.FP, Composed: d.Digest}
if had && before.Composed == d.Digest {
w.Backup, w.Words = before.Backup, before.Words
}
return w, true
}
// GrantNamesNode says whether a grant of module names its accounts on node (mesh-sdk's ControllerGrant.NamesNode, which a
// put reads by too).
func GrantNamesNode(g delivery.ControllerGrant, node, module string) bool {
return g.NamesNode(node, module)
}
modules/mesh-delivery-assigner/internal/assign/wants_test.go
+7 -911
View File
@@ -1,10 +1,8 @@
package assign
import (
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
@@ -20,7 +18,7 @@ var (
// ready is a module mail at fNew on ace and g14 for which every rule holds: built, not held, granted, its provider
// store healthy on ace and the canary ace running fNew healthy.
func ready() (Module, Nodes) {
m := Module{Name: "mail", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"g14", "ace"},
m := Module{Name: "mail", FP: fNew, Built: true, Granted: true, Nodes: []string{"g14", "ace"},
Manifest: catalogue.Manifest{Module: "mail", Requires: []string{"database"}}}
n := Nodes{
Online: map[string]bool{"ace": true, "g14": true},
@@ -33,16 +31,8 @@ func ready() (Module, Nodes) {
return m, n
}
// dig is the digest of the composition at a fingerprint in these tests: the fingerprint's last character repeated.
func dig(fp string) string { return "sha256:" + strings.Repeat(fp[len(fp)-1:], 64) }
// composedEverywhere is a Module.Composed with a composition for every node at every fingerprint, judged neither.
func composedEverywhere(_, fp string) (delivery.ComposedDigest, bool) {
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedNeither}, true
}
func wantedOld() map[string]delivery.Want {
return map[string]delivery.Want{"ace": {FP: fOld, Composed: dig(fOld)}, "g14": {FP: fOld, Composed: dig(fOld)}}
return map[string]delivery.Want{"ace": {FP: fOld}, "g14": {FP: fOld}}
}
// 1297/B4: with every rule holding, both nodes' wants move to the new fingerprint (the canary ace runs it healthy, so
@@ -51,7 +41,7 @@ func TestEveryRuleHoldsSoTheWantsMove(t *testing.T) {
m, n := ready()
out := Wants(m, n, wantedOld(), nil)
for _, node := range []string{"ace", "g14"} {
if out.Wants[node].FP != fNew || out.Wants[node].Rule != 0 || out.Wants[node].Composed != dig(fNew) {
if out.Wants[node].FP != fNew || out.Wants[node].Rule != 0 {
t.Errorf("%s's want = %+v, want %s", node, out.Wants[node], fNew)
}
}
@@ -69,9 +59,6 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
n.Haves["ace"]["store"] = delivery.Have{FP: fpOf('s'), Phase: delivery.PhaseRunning, Health: delivery.HealthWaits}
},
delivery.RuleGrant: func(m *Module, _ *Nodes) { m.Granted = false },
delivery.RuleComposed: func(m *Module, _ *Nodes) {
m.Composed = func(string, string) (delivery.ComposedDigest, bool) { return delivery.ComposedDigest{}, false }
},
} {
m, n := ready()
breakIt(&m, &n)
@@ -86,33 +73,6 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
t.Errorf("rule %d broken: g14's want is %+v", rule, w)
}
}
// Rule 6 per node: a grant written for the module's other nodes does not grant a node whose account it does not name
// (issue 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus).
m, n := ready()
grant := delivery.ControllerGrant{Accounts: []string{"ace.mail", "ace.mail.send"}}
m.GrantedOn = func(node string) bool { return GrantNamesNode(grant, node, "mail") }
out := Wants(m, n, wantedOld(), nil)
if w := out.Wants["ace"]; w.FP != fNew {
t.Errorf("ace, whose account the grant names, wants %+v", w)
}
if got := out.Rules["g14"]; got.Rule != delivery.RuleGrant || !strings.Contains(got.Words, "on g14") {
t.Errorf("g14, whose account the grant does not name, is held by %+v", got)
}
for _, c := range []struct {
accounts []string
node, module string
named bool
}{
{nil, "g14", "mail", true},
{[]string{"g14.mail.send"}, "g14", "mail", true},
{[]string{"g14.mailer"}, "g14", "mail", false},
{[]string{"ace.mail"}, "g14", "mail", false},
} {
if got := GrantNamesNode(delivery.ControllerGrant{Accounts: c.accounts}, c.node, c.module); got != c.named {
t.Errorf("%v names %s of %s: %v, want %v", c.accounts, c.module, c.node, got, c.named)
}
}
}
// 1297/B4, ADR 0435 §3: a canary that is offline is passed over: the next node by name is the canary, and moves first.
@@ -138,13 +98,11 @@ func TestAnOfflineCanaryIsPassedOver(t *testing.T) {
func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
m, n := ready()
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"}
m.Now = time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC)
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fOld, Composed: dig(fOld)}}, map[string]string{"ace": fOld})
// 1906/H3: the held record keeps its cause in words: which machine (its node), which health, when, and the node's words.
if out.Held == nil || out.Held.Node != "ace" || out.Held.Words != "health failed, phase running, seen at 2026-10-25T01:00:00Z: connection refused" {
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew}, "g14": {FP: fOld}}, map[string]string{"ace": fOld})
if out.Held == nil || out.Held.Node != "ace" || out.Held.Words != "connection refused" {
t.Fatalf("held = %+v", out.Held)
}
if w := out.Wants["ace"]; w.FP != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule != delivery.RuleNotHeld {
if w := out.Wants["ace"]; w.FP != fOld || w.Newest != fNew || w.Rule != delivery.RuleNotHeld {
t.Errorf("ace's want = %+v, want set back to the last healthy fingerprint", w)
}
if w, ok := out.Wants["g14"]; ok && w.FP == fNew {
@@ -164,20 +122,6 @@ func TestADataModuleIsNotSetBack(t *testing.T) {
if w := out.Wants["ace"]; w.FP != fNew || !strings.Contains(w.Words, "holds data") {
t.Errorf("ace's want = %+v, want it left at F, saying it holds data", w)
}
// 1906/D3: and the node is to be asked whether to restore it
if !slices.Equal(out.Restore, []string{"ace"}) {
t.Errorf("restore asks = %v, want ace", out.Restore)
}
}
// 1906/D3: a module without data is set back and asked about nothing.
func TestAModuleWithoutDataIsNotAskedToRestore(t *testing.T) {
m, n := ready()
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew}}, map[string]string{"ace": fOld})
if len(out.Restore) != 0 {
t.Errorf("restore asks = %v", out.Restore)
}
}
// 1297/B4, ADR 0408 §4: a node that leaves the module's nodes gets its want written removed, never deleted; a want that
@@ -185,7 +129,7 @@ func TestAModuleWithoutDataIsNotAskedToRestore(t *testing.T) {
func TestALeavingNodeIsRemovedAndNothingIsWrittenTwice(t *testing.T) {
m, n := ready()
m.Nodes = []string{"ace"}
out := Wants(m, n, map[string]delivery.Want{"ace": {V: 1, FP: fNew, Composed: dig(fNew)}, "g14": {FP: fOld, Composed: dig(fOld)}}, nil)
out := Wants(m, n, map[string]delivery.Want{"ace": {V: 1, FP: fNew}, "g14": {FP: fOld}}, nil)
if w := out.Wants["g14"]; !w.Removed {
t.Errorf("g14's want = %+v, want removed", w)
}
@@ -193,851 +137,3 @@ func TestALeavingNodeIsRemovedAndNothingIsWrittenTwice(t *testing.T) {
t.Errorf("ace's want, unchanged, was written again: %+v", out.Wants["ace"])
}
}
// 1906/D2 (ADR 0539 §6): the seventh rule. A want waits for the composition of the module for that node at F, and names its
// digest once it exists; a node with no want before still gets none until then.
func TestAWantWaitsForItsCompositionAndNamesItsDigest(t *testing.T) {
m, n := ready()
have := map[string]bool{}
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
if !have[node+"|"+fp] {
return delivery.ComposedDigest{}, false
}
return composedEverywhere(node, fp)
}
out := Wants(m, n, wantedOld(), nil)
for node, w := range out.Wants {
if w.FP == fNew {
t.Errorf("%s's want moved to F before its composition exists: %+v", node, w)
}
}
if r := out.Rules["ace"]; r.Rule != delivery.RuleComposed || !strings.Contains(r.Words, "ace") {
t.Errorf("ace is held back by %+v, want rule 7 naming the node", r)
}
// and a node that never had a want gets none
if out := Wants(m, n, nil, nil); len(out.Wants) != 0 {
t.Errorf("wants without a composition: %+v", out.Wants)
}
have["ace|"+fNew] = true
out = Wants(m, n, wantedOld(), nil)
if w := out.Wants["ace"]; w.FP != fNew || w.Composed != dig(fNew) {
t.Errorf("ace's want = %+v, want F naming the composition's digest", w)
}
if w, ok := out.Wants["g14"]; ok && w.FP == fNew {
t.Errorf("g14, whose composition does not exist, moved: %+v", w)
}
}
// 1906/D2 (ADR 0539 §5): a new revision of the composition at the same fingerprint reaches the node only through a new want,
// by the same rules: written when they hold, and not while one does not.
func TestARevisionAtTheSameFingerprintIsAWantByTheSameRules(t *testing.T) {
m, n := ready()
revised := "sha256:" + strings.Repeat("9", 64)
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
return delivery.ComposedDigest{Digest: revised, Judgement: delivery.JudgedNeither}, true
}
running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fNew, Composed: dig(fNew)}}
out := Wants(m, n, running, nil)
for _, node := range []string{"ace", "g14"} {
if w := out.Wants[node]; w.FP != fNew || w.Composed != revised {
t.Errorf("%s's want = %+v, want F naming the revised digest", node, w)
}
}
m.Granted = false
out = Wants(m, n, running, nil)
if w, ok := out.Wants["ace"]; ok && w.Composed == revised {
t.Errorf("a revision was written while a rule holds: %+v", w)
}
// A want held back at an earlier fingerprint follows the new revision of that fingerprint's kept composition, which the
// controller composed again with today's config (novox/hq ADR 0545: on g14 a capacity of 0 after a rollback), while
// the rules still hold F; for a record module whose revision recreates a container, only after the ask.
out = Wants(m, n, wantedOld(), nil)
if w := out.Wants["g14"]; w.FP != fOld || w.Composed != revised || w.Newest != fNew {
t.Errorf("g14, held back at its earlier fingerprint, wants %+v; want it at that fingerprint naming the revised digest", w)
}
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
return delivery.ComposedDigest{Digest: revised, Judgement: delivery.JudgedRecreates, Policy: delivery.PolicyRecord,
Against: dig(fOld), Words: "recreates"}, true
}
out = Wants(m, n, wantedOld(), nil)
if w := out.Wants["g14"]; w.Composed == revised {
t.Errorf("a recorded module's revision that recreates a container was written with no ask: %+v", w)
}
if !slices.ContainsFunc(out.Record, func(r RecordNeed) bool { return r.Node == "g14" && r.Digest == revised }) {
t.Errorf("the revision was not asked: %+v", out.Record)
}
}
// 1906/D2: a set-back names the composition of the fingerprint it sets back to, and when that is gone the want stays and says so.
func TestASetBackNamesTheOlderCompositionOrStays(t *testing.T) {
m, n := ready()
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "x"}
running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}}
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
if fp == fOld {
return delivery.ComposedDigest{}, false
}
return composedEverywhere(node, fp)
}
out := Wants(m, n, running, map[string]string{"ace": fOld})
if w := out.Wants["ace"]; w.FP == fOld {
t.Errorf("set back to a fingerprint whose composition is gone: %+v", w)
}
m.Composed = composedEverywhere
out = Wants(m, n, running, map[string]string{"ace": fOld})
if w := out.Wants["ace"]; w.FP != fOld || w.Composed != dig(fOld) {
t.Errorf("set-back want = %+v", w)
}
}
// movingTo is a module ready to move, whose composition at F is judged j against the one it runs, and whose digest is dig(F).
func movingTo(j delivery.Judgement) (Module, Nodes, map[string]delivery.Want) {
m, n := ready()
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
d := delivery.ComposedDigest{Digest: dig(fp), Judgement: j}
if j != delivery.JudgedNeither {
d.Against, d.Words = dig(fOld), "judged"
}
return d, true
}
return m, n, wantedOld()
}
func recording(m *Module) {
m.Manifest.Upgrade = &catalogue.UpgradePolicy{Policy: catalogue.PolicyRecord, Why: "a person takes each build"}
}
// 1906/D4 (ADR 0539 §14): a module of policy record whose move is judged to recreate a container, to move data, or cannot be
// judged, is not written until its ask is answered yes; the same at a new revision of the same fingerprint.
func TestARecordModuleWaitsForItsAskWhenTheMoveIsNotJudgedNeither(t *testing.T) {
for _, j := range []delivery.Judgement{delivery.JudgedRecreates, delivery.JudgedMovesData, delivery.JudgedCannot} {
m, n, wanted := movingTo(j)
recording(&m)
out := Wants(m, n, wanted, nil)
if w, ok := out.Wants["ace"]; ok {
t.Errorf("%s: ace's want was written without the ask's yes: %+v", j, w)
}
var ace *RecordNeed
for i := range out.Record {
if out.Record[i].Node == "ace" {
ace = &out.Record[i]
}
}
if ace == nil || ace.Digest != dig(fNew) || ace.Judgement != j {
t.Errorf("%s: the asks = %+v, want one for ace naming the digest and the judgement", j, out.Record)
}
// a new revision at the same fingerprint is asked too
same := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fOld)}}
if out := Wants(m, n, same, nil); len(out.Record) == 0 || len(out.Wants) != 0 && out.Wants["ace"].Composed == dig(fNew) {
t.Errorf("%s: a new revision at the same fingerprint: %+v", j, out)
}
m.Allowed = func(node, digest string) bool { return node == "ace" && digest == dig(fNew) }
out = Wants(m, n, wanted, nil)
if w := out.Wants["ace"]; w.FP != fNew || w.Composed != dig(fNew) {
t.Errorf("%s: after the yes ace's want = %+v", j, w)
}
}
}
// 1906/D4: a move judged neither is written at once, whatever the policy; so is a module the node never ran; and a module
// that rolls (not record) is never asked.
func TestAMoveJudgedNeitherOrNeverRunOrNotRecordedIsWrittenAtOnce(t *testing.T) {
m, n, wanted := movingTo(delivery.JudgedNeither)
recording(&m)
if out := Wants(m, n, wanted, nil); out.Wants["ace"].FP != fNew || len(out.Record) != 0 {
t.Errorf("neither: %+v", out)
}
m, n, _ = movingTo(delivery.JudgedRecreates)
recording(&m)
m.Nodes = []string{"ace"} // g14 runs it with no want, so its first want is asked (1945/AC19): this case is ace's alone
delete(n.Haves["ace"], "mail")
if out := Wants(m, n, nil, nil); out.Wants["ace"].FP != fNew || len(out.Record) != 0 {
t.Errorf("a node that never ran it: %+v", out)
}
// but one that runs an older fingerprint with no want yet is asked like any other (one running F is not: N1 below)
m, n, _ = movingTo(delivery.JudgedCannot)
recording(&m)
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
if out := Wants(m, n, nil, nil); out.Wants["ace"].FP == fNew || len(out.Record) == 0 {
t.Errorf("a node that runs it with no want: %+v", out)
}
m, n, wanted = movingTo(delivery.JudgedRecreates)
if out := Wants(m, n, wanted, nil); out.Wants["ace"].FP != fNew || len(out.Record) != 0 {
t.Errorf("a rolled module: %+v", out)
}
}
// 1906/D4 (ADR 0539 §7, §15): the bus module's new fingerprint is never written by the assigner, but a revision of its
// composition at the same fingerprint that only changes the user list (judged neither) is written at once.
func TestTheBusIsNotMovedButItsUserListRevisionIsWrittenAtOnce(t *testing.T) {
m, n, _ := movingTo(delivery.JudgedNeither)
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
if out := Wants(m, n, wantedOld(), nil); out.Wants["ace"].FP == fNew {
t.Errorf("the bus moved to a new fingerprint: %+v", out.Wants["ace"])
}
running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fOld)}, "g14": {FP: fNew, Composed: dig(fOld)}}
out := Wants(m, n, running, nil)
if w := out.Wants["ace"]; w.FP != fNew || w.Composed != dig(fNew) {
t.Errorf("a user-list revision of the bus: ace's want = %+v", w)
}
if len(out.Record) != 0 {
t.Errorf("a user-list revision was asked: %+v", out.Record)
}
// ADR 0546: a put owns the node's want from its asking until it ends. While it waits the rules write nothing there; once
// wanted its want stands, marked put and held, and a held bus still takes its user-list revision (ADR 0539 §7).
put := delivery.Put{FP: fNew, Hold: true, Why: "the bus's user list", Composed: dig(fNew)}
for _, c := range []struct {
name string
step string
wanted map[string]delivery.Want
want *delivery.Want // nil: nothing written on ace
}{
{"waiting for the operator", delivery.PutPerson, wantedOld(), nil},
{"wanted", delivery.PutWanted, wantedOld(), &delivery.Want{FP: fNew, Composed: dig(fNew), Put: true, Hold: true, Words: "put: the bus's user list"}},
{"holding, written already", delivery.PutHolding, map[string]delivery.Want{"ace": PutWant(put)}, nil},
{"released: the rules again", delivery.PutReleased, map[string]delivery.Want{"ace": PutWant(put)}, &delivery.Want{FP: fNew, Composed: dig(fNew)}},
} {
p := put
p.Step = c.step
m.Puts = map[string]delivery.Put{"ace": p}
out := Wants(m, n, c.wanted, nil)
got, wrote := out.Wants["ace"]
switch {
case c.want == nil && wrote:
t.Errorf("put %s: the rules wrote ace's want %+v", c.name, got)
case c.want != nil && (!wrote || !delivery.SameWant(normal(got), *c.want)):
t.Errorf("put %s: ace's want = %+v (written %v), want %+v", c.name, got, wrote, *c.want)
}
}
userList := "sha256:" + strings.Repeat("7", 64)
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
return delivery.ComposedDigest{Digest: userList, Judgement: delivery.JudgedNeither, UserListOnly: true}, true
}
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Composed: dig(fNew), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
held := put
held.Step = delivery.PutHolding
m.Puts = map[string]delivery.Put{"ace": held}
if w := Wants(m, n, map[string]delivery.Want{"ace": PutWant(held)}, nil).Wants["ace"]; w.Composed != userList || !w.Put || !w.Hold {
t.Errorf("a held bus did not take its user-list revision as a put: %+v", w)
}
m.Puts = nil
stepsOfAPut(t)
}
// stepsOfAPut takes a put through its steps (ADR 0546 clauses 3 to 8): the build, the controller's composition, the
// operator's yes, the bus's snapshot, the want, and the have.
func stepsOfAPut(t *testing.T) {
t.Helper()
now := time.Date(2026, 10, 25, 22, 0, 0, 0, time.UTC)
base := delivery.Put{FP: fNew, Hold: true, Why: "the bus's user list", At: now}.Said(now, delivery.PutAsked, "asked")
d := delivery.ComposedDigest{Digest: dig(fNew), Judgement: delivery.JudgedCannot, Words: "the have names no composition"}
running := delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
bus := func() PutWorld {
return PutWorld{Node: "novox", Module: "nats", Put: base, Assigned: true, Bus: true, Build: delivery.BuildDone, Newest: fNew,
Composed: &d, Granted: true, Have: &running, Now: now}
}
cases := []struct {
name string
world func() PutWorld
step string
says string
ask, snap bool
warranted bool
}{
{"not assigned", func() PutWorld { w := bus(); w.Assigned = false; return w }, delivery.PutRefused, "not assigned", false, false, false},
{"no build yet of the newest", func() PutWorld { w := bus(); w.Build = ""; return w }, delivery.PutBuild, "builds by itself", false, false, false},
{"no build of an older one", func() PutWorld { w := bus(); w.Build, w.Newest = "", fpOf('c'); return w }, delivery.PutRefused, "holds no build", false, false, false},
{"a failed build", func() PutWorld { w := bus(); w.Build, w.BuildWords = delivery.BuildFailed, "go vet"; return w }, delivery.PutRefused, "go vet", false, false, false},
{"no composition yet: waits", func() PutWorld { w := bus(); w.Composed = nil; return w }, delivery.PutCompose, "no composition of nats for novox", false, false, false},
{"composed, grant not naming the node", func() PutWorld { w := bus(); w.Granted = false; return w }, delivery.PutCompose, "does not name novox", false, false, false},
{"none within the bound: refused", func() PutWorld {
w := bus()
w.Composed = nil
w.Put = w.Put.Said(now.Add(-ComposeWithin-time.Second), delivery.PutCompose, "waits")
return w
}, delivery.PutRefused, "mesh-hand-compose", false, false, false},
{"the bus asks the operator", bus, delivery.PutPerson, "waits for the operator's yes", true, false, false},
{"the operator said no", func() PutWorld { w := bus(); w.Answer = OptionNo; return w }, delivery.PutRefused, "answered no", false, false, false},
{"the ask expired unanswered", func() PutWorld { w := bus(); w.AskExpired = true; return w }, delivery.PutRefused, "expired unanswered", false, false, false},
{"yes: the snapshot is asked", func() PutWorld { w := bus(); w.Answer = OptionYes; return w }, delivery.PutSnapshot, "asked novox's backup holder", false, true, false},
{"the snapshot is taken", func() PutWorld { w := bus(); w.Answer, w.Snapshotting = OptionYes, true; return w }, delivery.PutSnapshot, "waits for novox's backup holder", false, false, false},
{"the snapshot failed", func() PutWorld { w := bus(); w.Answer, w.SnapshotFailed = OptionYes, "no restic"; return w }, delivery.PutRefused, "no restic", false, false, false},
{"snapshotted: the want, warranted", func() PutWorld {
w := bus()
w.Answer, w.Snapshot = OptionYes, "novox's restore point"
w.Put.Ask, _ = delivery.RecordAskKey("novox", "nats", d.Digest)
return w
}, delivery.PutWanted, "held until released; the streams were snapshotted: novox's restore point", false, false, true},
{"a user-list revision of the bus needs nobody", func() PutWorld {
w := bus()
ul := d
ul.UserListOnly = true
w.Composed, w.Have = &ul, &delivery.Have{FP: fNew, Composed: dig(fOld), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
return w
}, delivery.PutWanted, "held until released", false, false, false},
{"a module not recorded needs nobody", func() PutWorld { w := bus(); w.Bus = false; return w }, delivery.PutWanted, "names", false, false, false},
{"a recorded module judged to recreate asks", func() PutWorld {
w := bus()
w.Bus, w.Recorded = false, true
r := d
r.Judgement = delivery.JudgedRecreates
w.Composed = &r
return w
}, delivery.PutPerson, "waits for the operator's yes", true, false, false},
{"wanted, the node runs it: holding", func() PutWorld {
w := bus()
w.Put = w.Put.Said(now, delivery.PutWanted, "wanted")
w.Put.Composed = d.Digest
w.Have = &delivery.Have{FP: fNew, Composed: d.Digest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
return w
}, delivery.PutHolding, "held there until released", false, false, false},
{"wanted without hold: done", func() PutWorld {
w := bus()
w.Put.Hold = false
w.Put = w.Put.Said(now, delivery.PutWanted, "wanted")
w.Put.Composed = d.Digest
w.Have = &delivery.Have{FP: fNew, Composed: d.Digest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
return w
}, delivery.PutDone, "the rules write its want again", false, false, false},
{"released: nothing more", func() PutWorld { w := bus(); w.Put = w.Put.Said(now, delivery.PutReleased, "released"); return w }, delivery.PutReleased, "released", false, false, false},
}
for _, c := range cases {
next := StepPut(c.world())
if next.Put.Step != c.step || !strings.Contains(next.Put.Words, c.says) {
t.Errorf("put, %s: at %s, %q; want %s saying %q", c.name, next.Put.Step, next.Put.Words, c.step, c.says)
}
if next.Ask != c.ask || next.Snapshot != c.snap || next.Warranted != c.warranted {
t.Errorf("put, %s: acts ask %v snapshot %v warranted %v; want %v %v %v", c.name, next.Ask, next.Snapshot, next.Warranted,
c.ask, c.snap, c.warranted)
}
if next.Ask && next.Put.Ask == "" {
t.Errorf("put, %s: an ask without its key", c.name)
}
}
}
// 1906/D4 (ADR 0539 §16): the answer to a failed backup stays in the want while its target stays.
func TestTheBackupAnswerStaysInTheWantWhileItsTargetStays(t *testing.T) {
m, n := ready()
running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew), Backup: delivery.BackupSkip, Words: BackupAnswerWords("skip", 0)}}
if w, ok := Wants(m, n, running, nil).Wants["ace"]; ok {
t.Errorf("the answer was erased: %+v", w)
}
}
// 1906/D4 (ADR 0539 §19): a hold released by the operator is not held again by a health check that failed before, and the
// wants move by the rules again.
func TestAReleasedHoldIsNotHeldAgain(t *testing.T) {
m, n := ready()
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "x"}
m.Released = true
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fOld, Composed: dig(fOld), Newest: fNew, Rule: delivery.RuleNotHeld}}, map[string]string{"ace": fOld})
if out.Held != nil {
t.Errorf("a released fingerprint was held again: %+v", out.Held)
}
if w := out.Wants["ace"]; w.FP != fNew || w.Composed != dig(fNew) || w.Rule != 0 {
t.Errorf("ace's want = %+v, want the held fingerprint written again", w)
}
}
// 1906/D4: a data module whose restore ask the operator answered "restore" is set back like any other, losing the newer writes;
// answered "leave stopped" (or not answered) it stays.
func TestARestoreAnswerSetsADataModuleBack(t *testing.T) {
m, n := ready()
m.Manifest.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Class: catalogue.ClassIrreplaceable}}}
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}}
m.Restored = func(node string) bool { return false }
if w, ok := Wants(m, n, running, map[string]string{"ace": fOld}).Wants["ace"]; ok && w.FP == fOld {
t.Errorf("set back without the operator's restore: %+v", w)
}
m.Restored = func(node string) bool { return node == "ace" }
if w := Wants(m, n, running, map[string]string{"ace": fOld}).Wants["ace"]; w.FP != fOld || w.Composed != dig(fOld) {
t.Errorf("after restore ace's want = %+v", w)
}
}
// 1906/D4 (ADR 0539 §14): the policy the controller holds beside the composition decides, the operator's own choice included;
// a composition with none (made by hand) falls back to what the manifest derives.
func TestTheRecordPolicyIsTheOneTheControllerHolds(t *testing.T) {
m, n, wanted := movingTo(delivery.JudgedMovesData)
with := func(policy, from string) {
inner := m.Composed
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
d, ok := inner(node, fp)
d.Policy, d.PolicyFrom = policy, from
return d, ok
}
}
with(delivery.PolicyRecord, "person") // a roll module the operator set to record
if out := Wants(m, n, wanted, nil); len(out.Record) == 0 || out.Wants["ace"].FP == fNew {
t.Errorf("the operator's record policy was not honoured: %+v", out)
}
recording(&m) // the manifest says record, the operator chose roll
with("roll", "person")
if out := Wants(m, n, wanted, nil); len(out.Record) != 0 || out.Wants["ace"].FP != fNew {
t.Errorf("the operator's roll policy was not honoured: %+v", out)
}
with("", "") // made by hand: the manifest decides
if out := Wants(m, n, wanted, nil); len(out.Record) == 0 {
t.Errorf("a hand-made composition fell through the manifest's record policy: %+v", out)
}
}
// Review round 1, item 2 (ADR 0539 §15): a node that runs the bus with no readable want compares the new fingerprint with the one
// it runs, from its have: the same fingerprint is not a new one (written by the rules), another is barred.
func TestTheBusWithNoWantIsComparedWithWhatItRuns(t *testing.T) {
m, n, _ := movingTo(delivery.JudgedNeither)
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
if w, ok := Wants(m, n, nil, nil).Wants["ace"]; !ok || w.FP != fNew {
t.Errorf("the bus runs F with no want and was not given one: %+v %v", w, ok)
}
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
if w, ok := Wants(m, n, nil, nil).Wants["ace"]; ok {
t.Errorf("the bus was moved to a new fingerprint with no want: %+v", w)
}
}
// Review round 1, item 1 (ADR 0539 §7): a revision of the bus module's composition that differs only in its user list is written
// at once, at the fingerprint the node runs, before the record hold, whatever the controller judges of the file's mount (it
// says recreates) and whatever policy the bus has (record, from the bus). The shape is the controller's: judged recreates against
// the composition the node runs, policy record, UserListOnly set.
func TestAUserListOnlyRevisionOfTheBusIsWrittenAtOnceBeforeTheRecordHold(t *testing.T) {
m, n, _ := movingTo(delivery.JudgedRecreates)
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
userList := "sha256:" + strings.Repeat("7", 64)
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
d := delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedRecreates, Against: dig(fOld), Words: "the user list file is mounted",
Policy: delivery.PolicyRecord, PolicyFrom: "the bus"}
if fp == fOld {
d.Digest, d.UserListOnly = userList, true // the composition at the fingerprint the node runs, with the new user list
}
return d, true
}
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
for name, wanted := range map[string]map[string]delivery.Want{
"a want at the fingerprint it runs": {"ace": {FP: fOld, Composed: dig(fOld)}},
"no readable want": nil,
} {
out := Wants(m, n, wanted, nil)
if w := out.Wants["ace"]; w.FP != fOld || w.Composed != userList {
t.Errorf("%s: ace's want = %+v, want the running fingerprint naming the user-list revision", name, w)
}
if len(out.Record) != 0 {
t.Errorf("%s: the user-list revision was asked: %+v", name, out.Record)
}
}
// the same flag on a module that is not the bus changes nothing
m.Manifest.Provides = nil
if out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fOld, Composed: dig(fOld)}}, nil); out.Wants["ace"].Composed == userList {
t.Errorf("a module that is not the bus was written at once: %+v", out.Wants["ace"])
}
}
// Review round 2, N1 (ADR 0539 §9, §11): right after the switch no node has a readable want and every have comes from the old node-engine
// with a fingerprint and no composition, so every move is judged cannot-judge. A node running F with such a have gets its want at F
// with no record hold, for a record module and for the bus alike (the node agent applies nothing when the composition equals the
// declaration it kept); a new fingerprint still waits for its ask.
func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) {
for name, bus := range map[string]bool{"a record module": false, "the bus": true} {
// only a composition the controller judged unchanged (it equals the declaration the node last applied, ADR 0540 §9) skips the hold
unchanged := func(node, fp string) (delivery.ComposedDigest, bool) {
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedUnchanged}, true
}
m, n, _ := movingTo(delivery.JudgedCannot)
recording(&m)
m.Nodes = []string{"ace"}
if bus {
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
}
m.Composed = unchanged
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK} // no composition named
out := Wants(m, n, nil, nil)
if w := out.Wants["ace"]; w.FP != fNew || w.Composed != dig(fNew) {
t.Errorf("%s: ace's want = %+v, want F naming its composition", name, w)
}
if len(out.Record) != 0 {
t.Errorf("%s: asked at the switch: %+v", name, out.Record)
}
// a composition that differs from the declaration the node kept (a changed setting or secret), and one the controller did not judge,
// still wait for the ask
for what, j := range map[string]delivery.Judgement{"differing": delivery.JudgedRecreates, "unjudged": delivery.JudgedCannot, "judged neither": delivery.JudgedNeither} {
m2, n2, _ := movingTo(j)
recording(&m2)
m2.Nodes = []string{"ace"}
if bus {
m2.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
}
n2.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
out := Wants(m2, n2, nil, nil)
if j == delivery.JudgedNeither {
if out.Wants["ace"].FP != fNew || len(out.Record) != 0 {
t.Errorf("%s, %s: %+v", name, what, out)
}
continue
}
if w, ok := out.Wants["ace"]; ok || len(out.Record) == 0 {
t.Errorf("%s, %s: written at the switch with no yes: %+v %+v", name, what, w, out.Record)
}
}
// a have that names a composition is judged as before
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedRecreates, Against: dig(fOld), Words: "judged"}, true
}
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Composed: dig(fOld), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
if out := Wants(m, n, nil, nil); out.Wants["ace"].FP == fNew && len(out.Record) == 0 {
t.Errorf("%s: a have naming a composition skipped the hold: %+v", name, out)
}
// a new fingerprint still asks (the bus is barred instead)
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
out = Wants(m, n, nil, nil)
if w, ok := out.Wants["ace"]; ok {
t.Errorf("%s: a new fingerprint was written with no yes: %+v", name, w)
}
if !bus && len(out.Record) == 0 {
t.Errorf("%s: a new fingerprint was not asked", name)
}
}
}
// 1906/H3: a failed check whose node gave no words is held saying so, never with empty words.
func TestAHoldOfAFailureWithoutWordsSaysSo(t *testing.T) {
m, n := ready()
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}}, map[string]string{"ace": fOld})
if out.Held == nil || out.Held.Words != "health failed, phase running: the node gave no words" {
t.Fatalf("held = %+v", out.Held)
}
}
// 1906/H2: given a held fingerprint that every node running the module now runs healthy, when the assigner passes, then the
// hold is lifted by itself and the wants move to it by the rules again. A node set back to another fingerprint, a node whose
// check fails, or a hold the operator answered "keep" leaves it held; a node that does not run the module at all does not count.
func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
everywhere := func() (Module, Nodes) {
m, n := ready()
m.Held = true
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
return m, n
}
m, n := everywhere()
out := Wants(m, n, wantedOld(), map[string]string{"ace": fOld, "g14": fOld})
if !out.Lifted || out.Held != nil {
t.Fatalf("lifted = %v, held = %+v: want the hold lifted", out.Lifted, out.Held)
}
for _, node := range []string{"ace", "g14"} {
if w := out.Wants[node]; w.FP != fNew || w.Rule != 0 {
t.Errorf("%s's want = %+v, want it moved to the lifted fingerprint", node, w)
}
}
m, n = everywhere()
delete(n.Haves["g14"], "mail")
if out := Wants(m, n, wantedOld(), nil); !out.Lifted {
t.Error("a node that does not run the module kept the hold")
}
for name, change := range map[string]func(*Module, *Nodes){
"set back": func(_ *Module, n *Nodes) {
n.Haves["g14"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
},
"failing": func(_ *Module, n *Nodes) {
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
},
"kept": func(m *Module, _ *Nodes) { m.Kept = true },
"no haves": func(_ *Module, n *Nodes) { delete(n.Haves["g14"], "mail"); delete(n.Haves["ace"], "mail") },
"not held": func(m *Module, _ *Nodes) { m.Held = false },
} {
m, n := everywhere()
change(&m, &n)
if out := Wants(m, n, wantedOld(), map[string]string{"ace": fOld, "g14": fOld}); out.Lifted {
t.Errorf("%s: the hold was lifted", name)
}
}
}
// x11 is screen-lock as on g14 and shanks on 2026-10-25 (issue 1945): built, granted and composed at F and at the older fOld
// on both, requiring x11-display, whose provider xorg runs on each node with its health unknown; each node runs screen-lock at fOld
// (health unknown) and holds no want for it. The canary g14 does not run F.
func x11() (Module, Nodes) {
m := Module{Name: "screen-lock", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"g14", "shanks"},
GrantedAt: func(fp string) bool { return fp == fNew || fp == fOld },
Manifest: catalogue.Manifest{Module: "screen-lock", Requires: []string{"x11-display"}}}
n := Nodes{
Online: map[string]bool{"g14": true, "shanks": true},
Haves: map[string]map[string]delivery.Have{
"g14": {"screen-lock": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown},
"xorg": {FP: fpOf('x'), Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}},
"shanks": {"screen-lock": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown},
"xorg": {FP: fpOf('x'), Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}},
},
Providers: map[string]map[string][]string{"x11-display": {"xorg": {"g14", "shanks"}}},
}
return m, n
}
// 1945/AC19: given a node that runs an older fingerprint of the new form, composed and granted, with no want, held back from F by
// the providers or the canary rule, when the assigner passes, then its first want names the fingerprint it runs with that
// composition, never the newest, and says the newest it is held back from; the canary is never passed over.
func TestANodeRunningAModuleWithNoWantGetsItsFirstWantAtWhatItRuns(t *testing.T) {
m, n := x11()
out := Wants(m, n, nil, nil)
for _, node := range []string{"g14", "shanks"} {
if w := out.Wants[node]; w.FP != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule == 0 {
t.Errorf("%s's want = %+v, want its first want at the fingerprint it runs, held back from F", node, w)
}
}
if len(out.Neither) != 0 {
t.Errorf("left on neither list: %+v", out.Neither)
}
// with the provider healthy, only the canary holds shanks back: still at what it runs
m, n = x11()
n.Haves["g14"]["xorg"] = delivery.Have{FP: fpOf('x'), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
n.Haves["shanks"]["xorg"] = n.Haves["g14"]["xorg"]
if w := Wants(m, n, nil, nil).Wants["shanks"]; w.FP != fOld || w.Rule != delivery.RuleCanary {
t.Errorf("shanks held back by the canary with no want: %+v, want at fOld by rule 3", w)
}
// a node already running F gets its want at F; a removed want is no want either
m, n = x11()
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
out = Wants(m, n, map[string]delivery.Want{"g14": {Removed: true}}, nil)
if w := out.Wants["g14"]; w.FP != fNew || w.Composed != dig(fNew) || w.Removed {
t.Errorf("g14 running F with a removed want = %+v, want F", w)
}
}
// 1945/AC19: a node running a fingerprint that is not of the new form, or one whose composition or grant does not exist, gets no
// want past the rules, and is named on neither list; the ordering rules still order every move from a want.
func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) {
for what, change := range map[string]func(*Module, *Nodes){
"an old-form build": func(_ *Module, n *Nodes) {
n.Haves["shanks"]["screen-lock"] = delivery.Have{FP: "build-2", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
},
"no grant": func(m *Module, _ *Nodes) { m.GrantedAt = func(fp string) bool { return fp == fNew } },
"no composition": func(m *Module, _ *Nodes) {
m.Composed = func(_, fp string) (delivery.ComposedDigest, bool) {
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedNeither}, fp == fNew
}
},
} {
m, n := x11()
change(&m, &n)
out := Wants(m, n, nil, nil)
if w, ok := out.Wants["shanks"]; ok {
t.Errorf("%s: shanks got a want: %+v", what, w)
}
if h, ok := out.Neither["shanks"]; !ok || h.Rule == 0 {
t.Errorf("%s: shanks on neither list = %+v (named %v)", what, h, ok)
}
}
m, n := x11()
before := map[string]delivery.Want{"g14": {FP: fOld, Composed: dig(fOld)}, "shanks": {FP: fOld, Composed: dig(fOld)}}
out := Wants(m, n, before, nil)
if len(out.Neither) != 0 {
t.Errorf("nodes holding a want named on neither list: %+v", out.Neither)
}
for node, w := range out.Wants {
if w.FP != fOld {
t.Errorf("%s's want moved from its want past the ordering rules: %+v", node, w)
}
}
}
// 1945/AC20: a new assignment (a node with no have of the module) waiting for the canary, the build or the grant gets no want and is
// never named on neither list: only a node that runs the module, or whose have names it, is; a module with no fingerprint of the new
// form goes with the node's own composition and is never named.
func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) {
m, n := x11()
delete(n.Haves["shanks"], "screen-lock")
out := Wants(m, n, nil, nil)
if w, ok := out.Wants["shanks"]; ok {
t.Errorf("a new assignment waiting for the canary got a want: %+v", w)
}
if _, ok := out.Neither["shanks"]; ok {
t.Errorf("a new assignment waiting for the canary is named on neither list: %+v", out.Neither)
}
m.Built = false
if out := Wants(m, n, nil, nil); len(out.Neither) != 1 || out.Neither["g14"].Rule != delivery.RuleBuildDone {
t.Errorf("with no build: neither = %+v, want g14 alone (it runs the module), by rule 1", out.Neither)
}
m.FP = ""
if out := Wants(m, n, nil, nil); len(out.Neither) != 0 {
t.Errorf("a module of the node's own composition named on neither list: %+v", out.Neither)
}
}
// 1945/AC21: a node whose have names an old-form build (no fingerprint of the new form, so no composition of it) can never be
// named in a want at what it runs; its first want names F, under the canary rule as usual and with everything F needs and the
// providers rule. Given the canary runs F healthy, shanks's first want names F; given it does not yet, shanks gets no want and
// stays on neither list. The old-form node that is itself the canary gets F once its providers run healthy.
func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) {
oldForm := func(n Nodes, node string) {
n.Haves[node]["screen-lock"] = delivery.Have{FP: "build-1792260876956849831", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
}
providersHealthy := func(n Nodes) {
for _, node := range []string{"g14", "shanks"} {
n.Haves[node]["xorg"] = delivery.Have{FP: fpOf('x'), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
}
}
// the canary g14 runs F healthy: shanks's first want names F
m, n := x11()
providersHealthy(n)
oldForm(n, "shanks")
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
out := Wants(m, n, nil, nil)
if w := out.Wants["shanks"]; w.FP != fNew || w.Composed != dig(fNew) || w.Newest != "" || w.Rule != 0 {
t.Errorf("canary healthy on F: shanks's want = %+v, want F", w)
}
if _, ok := out.Neither["shanks"]; ok {
t.Errorf("canary healthy on F: shanks still on neither list: %+v", out.Neither)
}
// the canary does not run F healthy yet: no want, and shanks stays on neither list by rule 3
m, n = x11()
providersHealthy(n)
oldForm(n, "shanks")
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
out = Wants(m, n, nil, nil)
if w, ok := out.Wants["shanks"]; ok {
t.Errorf("canary not healthy on F: shanks got a want: %+v", w)
}
if h, ok := out.Neither["shanks"]; !ok || h.Rule != delivery.RuleCanary {
t.Errorf("canary not healthy on F: shanks on neither list = %+v (named %v), want rule 3", h, ok)
}
// the canary healthy on F but the providers not healthy on shanks: rule 5 still holds it back
m, n = x11()
oldForm(n, "shanks")
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
out = Wants(m, n, nil, nil)
if w, ok := out.Wants["shanks"]; ok {
t.Errorf("providers not healthy: shanks got a want: %+v", w)
}
if h := out.Neither["shanks"]; h.Rule != delivery.RuleProviders {
t.Errorf("providers not healthy: shanks on neither list = %+v, want rule 5", h)
}
// F not granted: nothing, even with the canary healthy
m, n = x11()
providersHealthy(n)
oldForm(n, "shanks")
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
m.Granted = false
if w, ok := Wants(m, n, nil, nil).Wants["shanks"]; ok {
t.Errorf("F not granted: shanks got a want: %+v", w)
}
// the old-form node is the canary itself: F once its providers run healthy
m, n = x11()
providersHealthy(n)
oldForm(n, "g14")
if w := Wants(m, n, nil, nil).Wants["g14"]; w.FP != fNew || w.Composed != dig(fNew) {
t.Errorf("old-form canary: g14's want = %+v, want F", w)
}
}
// A failed health counts only once the node is done moving: phase running or failed, never backing-up, applying or checking.
// The live row on novox at 15:27:36 on 2026-10-25 ("health failed, phase applying" at the controller's new fingerprint) held the
// canary two seconds before the new build was healthy.
func TestAFailedHealthWhileTheNodeMovesHoldsNothing(t *testing.T) {
for _, phase := range []delivery.Phase{delivery.PhaseBackingUp, delivery.PhaseApplying, delivery.PhaseChecking} {
m := Module{Name: "mesh-controller", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"novox"},
Manifest: catalogue.Manifest{Module: "mesh-controller"}}
n := Nodes{Online: map[string]bool{"novox": true}, Haves: map[string]map[string]delivery.Have{
"novox": {"mesh-controller": {FP: fNew, Phase: phase, Health: delivery.HealthFailed}}}}
before := map[string]delivery.Want{"novox": {FP: fNew, Composed: dig(fNew)}}
if out := Wants(m, n, before, map[string]string{"novox": fOld}); out.Held != nil {
t.Errorf("phase %s, health failed: held %+v, want no hold while the node moves", phase, out.Held)
}
}
for _, phase := range []delivery.Phase{delivery.PhaseRunning, delivery.PhaseFailed} {
m := Module{Name: "mesh-controller", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"novox"},
Manifest: catalogue.Manifest{Module: "mesh-controller"}}
n := Nodes{Online: map[string]bool{"novox": true}, Haves: map[string]map[string]delivery.Have{
"novox": {"mesh-controller": {FP: fNew, Phase: phase, Health: delivery.HealthFailed}}}}
before := map[string]delivery.Want{"novox": {FP: fNew, Composed: dig(fNew)}}
if out := Wants(m, n, before, map[string]string{"novox": fOld}); out.Held == nil || out.Held.Node != "novox" {
t.Errorf("phase %s, health failed: held %+v, want a hold on novox", phase, out.Held)
}
}
}
// A hold is lifted by itself once every node that runs the module runs F healthy: a have of a node that is not running the module
// (removed, held for its backup, or still moving) is not counted, so it does not keep the hold up for ever; a node running another
// fingerprint, or failed, still does.
func TestAHoldIsLiftedOverHavesThatAreNotRunning(t *testing.T) {
setup := func(phase delivery.Phase, fp string, health delivery.Health) (Module, Nodes) {
m := Module{Name: "web", FP: fNew, Built: true, Granted: true, Held: true, Composed: composedEverywhere, Nodes: []string{"ace", "g14"}}
n := Nodes{Online: map[string]bool{"ace": true, "g14": true}, Haves: map[string]map[string]delivery.Have{
"ace": {"web": {FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}},
"g14": {"web": {FP: fp, Phase: phase, Health: health}}}}
return m, n
}
for _, phase := range []delivery.Phase{delivery.PhaseRemoved, delivery.PhaseHeld, delivery.PhaseApplying, delivery.PhaseBackingUp, delivery.PhaseChecking} {
m, n := setup(phase, fOld, delivery.HealthUnknown)
if out := Wants(m, n, nil, nil); !out.Lifted {
t.Errorf("g14's have in phase %s kept the hold up", phase)
}
}
for what, h := range map[string]delivery.Have{
"running another fingerprint": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK},
"failed": {FP: fNew, Phase: delivery.PhaseFailed, Health: delivery.HealthFailed},
} {
m, n := setup(h.Phase, h.FP, h.Health)
if out := Wants(m, n, nil, nil); out.Lifted {
t.Errorf("g14 %s: the hold was lifted", what)
}
}
// a failed health on a removed have holds nothing
m, n := setup(delivery.PhaseRemoved, fNew, delivery.HealthFailed)
m.Held = false
if out := Wants(m, n, nil, nil); out.Held != nil {
t.Errorf("a removed have with a failed health held F: %+v", out.Held)
}
}
// A module of policy record whose want was removed but which the node still runs gets its first want through the record ask: a
// removed want is no want, so a move the controller judges to recreate or move data waits for the operator's yes.
func TestAFirstWantAfterARemovedWantGoesThroughTheRecordAsk(t *testing.T) {
m, n := x11()
recording(&m)
m.Composed = func(_, fp string) (delivery.ComposedDigest, bool) {
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedRecreates, Policy: delivery.PolicyRecord}, true
}
out := Wants(m, n, map[string]delivery.Want{"shanks": {Removed: true}}, nil)
if w, ok := out.Wants["shanks"]; ok {
t.Errorf("shanks's first want after a removed want was written with no record ask: %+v", w)
}
asked := false
for _, r := range out.Record {
asked = asked || (r.Node == "shanks" && r.Digest == dig(fOld) && r.Judgement == delivery.JudgedRecreates)
}
if !asked {
t.Errorf("the record asks = %+v, want one for shanks at the composition it runs", out.Record)
}
}
// 1945/AC23: given a module held at F under rule 2 (the operator holds systemd-resolved on shanks, 2026-10-25), or held by the
// assigner in this pass after a failed health check of F, when a node that runs it holds no want for it, then the node is not on
// neither list: a held module is on the held list, whose hold is said and answered on its own.
func TestAHeldModuleWithNoWantIsNotOnNeitherList(t *testing.T) {
for what, change := range map[string]func(*Module, *Nodes){
"held at F": func(m *Module, _ *Nodes) { m.Held, m.Kept = true, true },
"held at F, running an old-form build": func(m *Module, n *Nodes) {
m.Held, m.Kept = true, true
n.Haves["shanks"]["screen-lock"] = delivery.Have{FP: "build-2", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
},
"held by this pass": func(_ *Module, n *Nodes) {
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
n.Haves["shanks"]["screen-lock"] = delivery.Have{FP: "build-2", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
},
} {
m, n := x11()
change(&m, &n)
out := Wants(m, n, nil, nil)
if len(out.Neither) != 0 {
t.Errorf("%s: a held module is on neither list: %+v", what, out.Neither)
}
if out.Rules["shanks"].Rule != delivery.RuleNotHeld {
t.Errorf("%s: shanks is held back by %+v, want rule 2", what, out.Rules["shanks"])
}
}
}
modules/mesh-delivery-assigner/manifest_test.go
+6 -20
View File
@@ -24,7 +24,7 @@ func load(t *testing.T) catalogue.Manifest {
// 1297/B2a: the controller's reader (mesh-sdk catalogue, which the controller reads manifests with) takes the
// manifest; it reads exactly the buckets the assigner opens (the program's list, held equal in its own test), keeps
// the states it declares (the differences kept read-only, 1906/D5), and holds a credential of its own.
// its three states, and holds a credential of its own.
func TestManifest(t *testing.T) {
m := load(t)
if m.Module != "mesh-delivery-assigner" {
@@ -32,9 +32,9 @@ func TestManifest(t *testing.T) {
}
reads := slices.Clone(m.Reads)
slices.Sort(reads)
want := []string{"mesh-build-go.steps", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants",
want := []string{"mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.grants",
"mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests",
"mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "node-engine.have"}
"mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}
if !reflect.DeepEqual(reads, want) {
t.Errorf("reads = %v, want exactly %v", reads, want)
}
@@ -42,7 +42,7 @@ func TestManifest(t *testing.T) {
for _, s := range m.State {
states[s.Name] = s.History
}
if want := map[string]int{"asks": 5, "compositions": 5, "differences": 64, "puts": 20, "wants": 20}; !reflect.DeepEqual(states, want) {
if want := map[string]int{"asks": 5, "compositions": 5, "differences": 64, "wants": 20}; !reflect.DeepEqual(states, want) {
t.Errorf("state = %v, want %v", states, want)
}
if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 {
@@ -51,21 +51,7 @@ func TestManifest(t *testing.T) {
if m.SecretsOwner != "mesh-delivery-assigner" {
t.Errorf("secrets-owner = %q", m.SecretsOwner)
}
// 1906/D3: it says its conditions on the operator channel; 1622: it puts its compose asks on the Go build seat, as the
// checker puts its checks.
if !slices.Equal(m.Uses, []string{"node-build-go", "operator-channel"}) {
t.Errorf("uses = %v, want exactly node-build-go and operator-channel", m.Uses)
}
// ADR 0546: it serves its own tools (put, release, puts) from its bundle through the tool runner (the artifact loads the
// module), claims no seat (a new seat needs a change of the controller), says each put's step as the event put, and
// calls the controller's warranted and the bus node's backup holder.
if len(m.Claims) != 0 || !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts"}) {
t.Errorf("the assigner claims %v and serves tools %v, want no claim and its three put tools", m.Claims, m.Tools)
}
if want := []string{"seat:mesh-controller.warranted", "seat:node-backup.backed-up", "seat:node-backup.now"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) {
t.Errorf("invokes = %v, want %v", m.Invokes, want)
}
if !slices.Equal(m.Emits, []string{"put"}) {
t.Errorf("emits = %v, want put", m.Emits)
if len(m.Claims) != 0 || len(m.Tools) != 0 || len(m.Invokes) != 0 {
t.Errorf("the assigner claims, serves or calls something: claims %v, tools %v, invokes %v", m.Claims, m.Tools, m.Invokes)
}
}
modules/mesh-delivery-assigner/module.json
+1 -25
View File
@@ -3,41 +3,20 @@
"version": "1",
"slug": "assigner",
"reads": [
"mesh-build-go.steps",
"mesh-controller.assignments",
"mesh-controller.builds",
"mesh-controller.composed",
"mesh-controller.grants",
"mesh-delivery-builder.builds",
"mesh-delivery-indexer.candidates",
"mesh-delivery-indexer.manifests",
"mesh-delivery-indexer.modules",
"mesh-delivery-teller.answers",
"mesh-delivery-watchman.presence",
"node-engine.have"
],
"uses": [
"node-build-go",
"operator-channel"
],
"invokes": [
"seat:mesh-controller.warranted",
"seat:node-backup.now",
"seat:node-backup.backed-up"
],
"emits": [
"put"
],
"tools": [
"assigner_put",
"assigner_release",
"assigner_puts"
],
"state": [
{ "name": "asks", "history": 5 },
{ "name": "compositions", "history": 5 },
{ "name": "differences", "history": 64 },
{ "name": "puts", "history": 20 },
{ "name": "wants", "history": 20 }
],
"own-secrets": {
@@ -85,10 +64,7 @@
"language": "go",
"system": "arch",
"from": "cmd/mesh-delivery-assigner",
"binary": "mesh-delivery-assigner",
"loads": [
"mesh-delivery-assigner"
]
"binary": "mesh-delivery-assigner"
}
]
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/bus.go
-5
View File
@@ -66,11 +66,6 @@ func settle(ctx context.Context, kv jetstream.KeyValue, each func(jetstream.KeyV
if err != nil {
return nil, err
}
return settled(ctx, w, each)
}
// settled hands each entry the watch delivers before its end-of-initial marker to each, and answers the watch.
func settled(ctx context.Context, w jetstream.KeyWatcher, each func(jetstream.KeyValueEntry)) (jetstream.KeyWatcher, error) {
for {
select {
case <-ctx.Done():
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/gone_integration_test.go
-185
View File
@@ -1,185 +0,0 @@
//go:build integration
package main
import (
"encoding/json"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// A done key whose blob the artifact store no longer holds is built again, by the builder whole against the bus
// (novox/hq issue 1937): on 2026-10-24 the store's 03:30 collection deleted the bare blobs of three done keys the
// controller had never taken in, and the builder kept them done.
// heldVerb answers the controller seat's verb held on the bus, as the serving controller does: {result: {output, ok,
// answer}}; a reference is gone while gone says so.
type heldVerb struct {
mu sync.Mutex
gone func(reference string) bool
asked []string
}
func (h *heldVerb) serve(t *testing.T, b *bed) {
t.Helper()
sub, err := b.js.Conn().Subscribe("mesh.seat.mesh-controller.call.held.>", func(m *nats.Msg) {
var args struct {
References []string `json:"references"`
}
_ = json.Unmarshal(m.Data, &args)
a := delivery.HeldAnswer{Store: "store.test:5100", Held: []string{}, Gone: []string{}}
h.mu.Lock()
for _, r := range args.References {
h.asked = append(h.asked, r)
if h.gone(r) {
a.Gone = append(a.Gone, r)
} else {
a.Held = append(a.Held, r)
}
}
h.mu.Unlock()
reply, _ := json.Marshal(map[string]any{"result": map[string]any{"output": "held", "ok": true, "answer": a}, "node": "ctl"})
_ = m.Respond(reply)
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
}
func (h *heldVerb) asks() int {
h.mu.Lock()
defer h.mu.Unlock()
return len(h.asked)
}
// eventsConsumer makes EVENTS and the builder's durable consumer of artifacts-gone on it, as the controller makes them
// from the manifest's consumes.
func eventsConsumer(t *testing.T, b *bed, node string) {
t.Helper()
if _, err := b.js.CreateStream(b.ctx, jetstream.StreamConfig{Name: eventsStream, Subjects: []string{delivery.ArtifactsGoneSubject}}); err != nil {
t.Fatal(err)
}
if _, err := b.js.CreateOrUpdateConsumer(b.ctx, eventsStream, jetstream.ConsumerConfig{Durable: node + "_mesh-delivery-builder",
AckPolicy: jetstream.AckExplicitPolicy, FilterSubjects: []string{delivery.ArtifactsGoneSubject}}); err != nil {
t.Fatal(err)
}
}
// doneKey writes builds.<module>.<fp> done, naming one archive at reference, and answers its revision.
func doneKey(t *testing.T, b *bed, module, manifest, reference, digest string) uint64 {
t.Helper()
key, _ := delivery.BuilderBuildKey(module, fp)
data, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "n2.mesh-delivery-builder",
Manifest: manifest, Step: "built mesh-build-go on n2",
Artifacts: []delivery.BuiltArtifact{{Name: "files", Kind: "archive", Digest: digest, Reference: reference}}})
if err != nil {
t.Fatal(err)
}
rev, err := b.builds.Put(b.ctx, key, data)
if err != nil {
t.Fatal(err)
}
return rev
}
func revisionOf(t *testing.T, b *bed, module string) uint64 {
t.Helper()
key, _ := delivery.BuilderBuildKey(module, fp)
e, err := b.builds.Get(b.ctx, key)
if err != nil {
t.Fatal(err)
}
return e.Revision()
}
// 1937/AC3 (integration): the builder, started whole, finds the done key of the trunk's fingerprint naming a blob held
// answers gone, builds it again as a new revision whose step says so, and leaves the rebuilt key done.
func TestADoneKeyWhoseBlobIsGoneIsBuiltAgainAtStart(t *testing.T) {
b := startBed(t)
eventsConsumer(t, b, "n1")
manifest := b.archive(t, "ca-trust", "modules/ca-trust", commit)
old := digestOf("collected last night")
oldRef := "artifact-store://ca-trust/files/blobs/" + old
rev := doneKey(t, b, "ca-trust", manifest, oldRef, old)
h := &heldVerb{gone: func(r string) bool { return r == oldRef }}
h.serve(t, b)
b.start(t, "n1", nil)
v := b.await(t, "ca-trust", func(v delivery.BuilderBuild) bool {
return v.State == delivery.BuildDone && len(v.Artifacts) == 1 && v.Artifacts[0].Digest == digestOf("files")
})
if !strings.Contains(v.Step, "built again: the artifact store no longer held "+oldRef) {
t.Errorf("the rebuilt key's step %q does not say why it was built again", v.Step)
}
if got := revisionOf(t, b, "ca-trust"); got <= rev {
t.Errorf("revision %d, was %d: want a new revision", got, rev)
}
if n := b.fake.count("mesh-build-go.build_go_build"); n != 1 {
t.Errorf("the archive was built %d times, want once", n)
}
if h.asks() == 0 {
t.Error("the builder never asked held")
}
}
// 1937/AC4 (integration): a done key whose blob held answers held at start stays done; when the controller says
// artifacts-gone for it, the builder asks again and builds it again.
func TestArtifactsGoneLeadsTheBuilderToBuildTheKeyAgain(t *testing.T) {
b := startBed(t)
eventsConsumer(t, b, "n1")
manifest := b.archive(t, "ca-trust", "modules/ca-trust", commit)
d := digestOf("files")
ref := "artifact-store://ca-trust/files/blobs/" + d
rev := doneKey(t, b, "ca-trust", manifest, ref, d)
var mu sync.Mutex
collected := false
h := &heldVerb{gone: func(r string) bool {
mu.Lock()
defer mu.Unlock()
if r == ref && collected {
collected = false // gone once: the rebuild pushes it again
return true
}
return false
}}
h.serve(t, b)
b.start(t, "n1", nil)
for deadline := time.Now().Add(20 * time.Second); h.asks() == 0 && time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
}
if h.asks() == 0 {
t.Fatal("the builder never asked held at its start")
}
time.Sleep(300 * time.Millisecond)
if got := revisionOf(t, b, "ca-trust"); got != rev || b.fake.count("mesh-build-go.build_go_build") != 0 {
t.Fatalf("a key whose blob is held was built again (revision %d, was %d)", got, rev)
}
mu.Lock()
collected = true
mu.Unlock()
key, _ := delivery.BuilderBuildKey("ca-trust", fp)
body, err := delivery.EncodeArtifactsGone(delivery.ArtifactsGone{Module: "ca-trust", Key: key, Fingerprint: fp,
Gone: []string{ref}, At: time.Now().UTC()})
if err != nil {
t.Fatal(err)
}
if _, err := b.js.Publish(b.ctx, delivery.ArtifactsGoneSubject, body); err != nil {
t.Fatal(err)
}
v := b.await(t, "ca-trust", func(v delivery.BuilderBuild) bool {
return v.State == delivery.BuildDone && strings.Contains(v.Step, "built again")
})
if got := revisionOf(t, b, "ca-trust"); got <= rev || b.fake.count("mesh-build-go.build_go_build") != 1 {
t.Errorf("build = %+v at revision %d (was %d), %d builds: want built again once on the event", v, got, rev,
b.fake.count("mesh-build-go.build_go_build"))
}
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go
+10 -37
View File
@@ -162,29 +162,11 @@ func (f *fakeBuildModules) serve(t *testing.T, module string, nodes ...string) {
}
}
// trunkHead is the trunk head of novox/mesh-catalog the bed's source revision names: newer than the module values' commit.
const trunkHead = "3333333333333333333333333333333333333333"
// trunk writes the gate's source.main naming head as novox/mesh-catalog's trunk head, and answers its revision.
func (b *bed) trunk(t *testing.T, head string) uint64 {
t.Helper()
data, err := delivery.EncodeSource(delivery.Source{V: delivery.Version, Group: "g", Heads: map[string]string{"novox/mesh-catalog": head}})
if err != nil {
t.Fatal(err)
}
rev, err := b.source.Put(b.ctx, delivery.SourceMainKey, data)
if err != nil {
t.Fatal(err)
}
return rev
}
type bed struct {
ctx context.Context
js jetstream.JetStream
modules jetstream.KeyValue
candidates jetstream.KeyValue
source jetstream.KeyValue
builds jetstream.KeyValue
tools map[string]jetstream.KeyValue
instances jetstream.KeyValue
@@ -197,11 +179,7 @@ type bed struct {
// startBed starts the bus, the buckets the controller makes from the builder's and the build modules' manifests, the
// build modules' streams and the builder's durable consumers on them, two nodes' toolchains, the indexer's manifests, and
// the fake build modules. It runs no builder.
func startBed(t *testing.T) *bed { return startBedWith(t, false) }
// startBedWith is startBed with the nodes' capacity 0 from the start when closed: a puller that reads its capacity before a
// test could close the nodes may be in a pull already and takes the first ask put (issue 1928).
func startBedWith(t *testing.T, closed bool) *bed {
func startBed(t *testing.T) *bed {
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "b")
if err != nil {
t.Fatal(err)
@@ -261,12 +239,10 @@ func startBedWith(t *testing.T, closed bool) *bed {
}
b.modules = mk(delivery.ModulesBucket, delivery.ModulesHistory)
b.candidates = mk(delivery.CandidatesBucket, delivery.CandidatesHistory)
b.source = mk(delivery.SourceBucket, delivery.SourceHistory)
mk(delivery.ManifestsBucket, delivery.ManifestsHistory)
b.builds = mk(delivery.BuilderBuildsBucket, delivery.BuilderBuildsHistory)
b.instances = mk(instancesBucket, 1)
b.fake = &fakeBuildModules{js: js, ctx: ctx, calls: map[string]int{}}
b.fake.closed.Store(closed)
b.fake.serve(t, "mesh-build-go", "n1", "n2")
b.fake.serve(t, "mesh-build-node", "n1", "n2")
b.fake.serve(t, "mesh-build-image", "n1") // the image build module runs on n1 only
@@ -532,9 +508,7 @@ func TestAKeyLeftBuildingForAModuleThatIsGoneIsClosedAtStart(t *testing.T) {
// TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk: the trunk's record of netcheck carries a fingerprint of its
// own; a candidate the indexer wrote gives netcheck the fingerprint fp at a pull request's head; the program writes
// builds.netcheck.<fp> done, its fetch given the head, the ref that carries it and the trunk head of the candidate's source
// revision, newer than the module value's commit (issue 1684; the operator 2026-10-25: a value names the commit its
// fingerprint first appeared at).
// builds.netcheck.<fp> done, its fetch given the head, the ref that carries it and the trunk commit (issue 1684).
func TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk(t *testing.T) { // N7/AC1
b := startBed(t)
other := "src2:" + strings.Repeat("e", 64)
@@ -551,7 +525,7 @@ func TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk(t *tes
pulls := []delivery.CandidatePull{{Head: delivery.Head{Repository: "novox/mesh-catalog", Number: 7, Head: head}, Modules: []string{"netcheck"},
Fps: map[string]delivery.CandidateModule{"netcheck": {FP2: fp, Manifest: digest}}}}
hash, _ := delivery.HeadsHash([]delivery.Head{pulls[0].Head})
cand, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", Base: b.trunk(t, trunkHead), HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
cand, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
if err != nil {
t.Fatal(err)
}
@@ -571,12 +545,12 @@ func TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk(t *tes
defer b.fake.mu.Unlock()
found := false
for _, f := range b.fake.fetches {
if f["commit"] == head && f["ref"] == "refs/pull/7/head" && f["base"] == trunkHead {
if f["commit"] == head && f["ref"] == "refs/pull/7/head" && f["base"] == commit {
found = true
}
}
if !found {
t.Errorf("fetches = %v, want one of the candidate's head with its ref and the trunk head it was previewed onto, not the module value's commit", b.fake.fetches)
t.Errorf("fetches = %v, want one of the candidate's head with its ref and the trunk commit", b.fake.fetches)
}
}
@@ -624,7 +598,7 @@ func TestAnImageIsBuiltOnOneNodeAndOnlyATrunkKeyPublishes(t *testing.T) { // 170
pulls := []delivery.CandidatePull{{Head: delivery.Head{Repository: "novox/mesh-catalog", Number: 7, Head: head}, Modules: []string{"imaged"},
Fps: map[string]delivery.CandidateModule{"imaged": {FP2: fp, Manifest: digest}}}}
hash, _ := delivery.HeadsHash([]delivery.Head{pulls[0].Head})
cand, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", Base: b.trunk(t, trunkHead), HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
cand, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
if err != nil {
t.Fatal(err)
}
@@ -807,7 +781,8 @@ func (b *bed) asksPut(t *testing.T, nc *nats.Conn) func() []delivery.StepAsk {
// 1887/AC3: a builder that started again while its asks waited in the streams (no node had a free slot) puts no second
// ask: it waits for the asks its building key names, and the key is done once a node took them.
func TestARestartedBuilderPutsNoSecondAsk(t *testing.T) {
b := startBedWith(t, true) // 1928/AC1: no node can take an ask from the start, so no pull in hand takes the first one
b := startBed(t)
b.fake.closed.Store(true)
b.module(t, "netcheck", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck")
sub, err := nats.Connect(b.srvURL, nats.UserInfo("ctl", "pw"))
if err != nil {
@@ -817,13 +792,11 @@ func TestARestartedBuilderPutsNoSecondAsk(t *testing.T) {
asked := b.asksPut(t, sub)
b.start(t, "n1", nil)
v := b.await(t, "netcheck", func(v delivery.BuilderBuild) bool { return v.State == delivery.BuildBuilding && len(v.Asks) == 2 })
for deadline := time.Now().Add(20 * time.Second); len(asked()) < 2; time.Sleep(20 * time.Millisecond) {
if time.Now().After(deadline) {
t.Fatalf("asks put = %+v within 20 s, want the two the key names", asked())
}
for deadline := time.Now().Add(5 * time.Second); len(asked()) < 2 && time.Now().Before(deadline); time.Sleep(20 * time.Millisecond) {
}
b.stop()
b.start(t, "n1", nil) // the same builder, started again, while both asks wait
time.Sleep(500 * time.Millisecond)
b.fake.open()
done := b.await(t, "netcheck", isState(delivery.BuildDone))
if got := asked(); len(got) != 2 {
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go
+1 -119
View File
@@ -16,12 +16,9 @@ import (
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"io"
"log"
"maps"
"os"
"os/signal"
"slices"
"strings"
"sync"
"syscall"
"time"
@@ -29,7 +26,6 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-builder/internal/build"
)
@@ -157,17 +153,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// Each build module's part of a build is an ask on its build seat's stream, pulled by a build node with a free slot
// (novox/hq ADR 0525): the builder chooses no node, so it reads neither presence nor have and holds no capacity.
asks := buildrun.JetStreamAsks{JS: js}
// The builder asks the controller, as its own account, whether the artifact store still holds a done key's blobs
// (the seat verb held, novox/hq issue 1937) before it leaves the key done.
account, err := process.ReadAccount(path)
if err != nil {
return err
}
calls, err := process.Connect(account, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
if err != nil {
return err
}
defer calls.Close()
b := &build.Builder{
Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
Asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: bound},
@@ -175,28 +160,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
Toolchains: tools,
Manifests: manifestReader{manifests},
Logf: logger.Printf,
Stored: storedVia(calls.Call),
}
// A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base
// revision (the gate's source state). Where the bucket is not there yet no candidate's fingerprint is built, and said.
// The source's revisions are watched with their history (a direct get of one revision is not what a read grants): a
// revision heard after the candidate that names it builds that candidate then.
trunks := &sourceHeads{}
var sw jetstream.KeyWatcher
switch kv, err := js.KeyValue(ctx, delivery.SourceBucket); {
case errors.Is(err, jetstream.ErrBucketNotFound):
logger.Printf("the bucket %s does not exist yet: no candidate's fingerprint is built until the builder starts again", delivery.SourceBucket)
case err != nil:
return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err)
default:
w, err := kv.Watch(ctx, delivery.SourceMainKey, jetstream.IncludeHistory())
if err != nil {
return fmt.Errorf("the bucket %s cannot be watched: %w", delivery.SourceBucket, err)
}
if sw, err = settled(ctx, w, func(e jetstream.KeyValueEntry) { trunks.put(e) }); err != nil {
return err
}
b.Trunk = trunks.head
}
// **Said once every build module answered a reference, taken away when stopped** (issue 1886, ADR 0522): the
// controller delivers from this builder's keys only while an instance says it writes references. A bucket not there
@@ -220,14 +183,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
// The toolchains first, so a build that starts from the first modules value finds the nodes.
watch := newToolchainWatch(ctx, js, tools, b.NodesChanged, logger)
// Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's
// assignments are settled first, so the toolchains settle through the gate already knowing them.
gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf)
watch.put = gate.Toolchain
assignErrs, err := watchAssignments(ctx, js, gate, b.NodesChanged, logger)
if err != nil {
return err
}
if err := watch.start(); err != nil {
return err
}
@@ -248,21 +203,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
defer cc.Stop()
}
// The controller says artifacts-gone when its take-in found a key's blob gone (issue 1937): the builder's durable
// consumer on EVENTS, made by the controller from the manifest's consumes. Where it is not there yet the builder says
// so; it still asks held at its start and at every module value.
switch cons, err := js.Consumer(ctx, eventsStream, cred.Node+"_"+cred.Module); {
case err != nil:
logger.Printf("the consumer %s_%s on %s does not exist yet, so %s is not heard until the builder starts again: %v",
cred.Node, cred.Module, eventsStream, delivery.ArtifactsGoneEvent, err)
default:
cc, err := cons.Consume(func(m jetstream.Msg) { heardGone(ctx, b, m, logger.Printf) },
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
if err != nil {
return err
}
defer cc.Stop()
}
each := func(e jetstream.KeyValueEntry) {
if opOf(e) {
b.Module(ctx, e.Key(), e.Value())
@@ -291,74 +231,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return err
}
logger.Printf("started on %s", cred.Node)
errs := make(chan error, 3)
errs := make(chan error, 2)
go func() { errs <- follow(ctx, w, each) }()
go func() { errs <- follow(ctx, cw, eachCandidate) }()
if sw != nil {
go func() {
errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) {
if trunks.put(e) {
b.TrunkMoved(ctx)
}
})
}()
}
select {
case <-ctx.Done():
b.Wait()
return nil
case err := <-errs:
return err
case err := <-assignErrs:
return err
case err := <-watch.errs:
return err
}
}
// sourceHeads are the trunk heads of each revision of the gate's source.main the watch delivered, the newest
// delivery.SourceHistory of them: what a candidate's base revision names.
type sourceHeads struct {
mu sync.Mutex
byRev map[uint64]map[string]string
}
// put takes one revision in and answers whether it is new.
func (s *sourceHeads) put(e jetstream.KeyValueEntry) bool {
if !opOf(e) {
return false
}
src, err := delivery.DecodeSource(e.Value())
if err != nil {
return false
}
s.mu.Lock()
defer s.mu.Unlock()
if s.byRev == nil {
s.byRev = map[uint64]map[string]string{}
}
if _, had := s.byRev[e.Revision()]; had {
return false
}
s.byRev[e.Revision()] = src.Heads
for len(s.byRev) > delivery.SourceHistory {
delete(s.byRev, slices.Min(slices.Collect(maps.Keys(s.byRev))))
}
return true
}
// head is the head of repository in revision rev; an error when the revision is not heard (yet, or any more).
func (s *sourceHeads) head(_ context.Context, rev uint64, repository string) (string, error) {
s.mu.Lock()
defer s.mu.Unlock()
heads, ok := s.byRev[rev]
if !ok {
return "", fmt.Errorf("source revision %d is not heard (it is built once it is)", rev)
}
for r, head := range heads {
if delivery.SameRepository(r, repository) {
return head, nil
}
}
return "", nil
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing.go
-209
View File
@@ -1,209 +0,0 @@
package main
// This file is the same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker (issue 1940): it moves
// to mesh-sdk go/delivery/buildrun once mesh-sdk can be delivered again, since a change of mesh-sdk waits on the very
// toolchain agreement this file repairs.
import (
"context"
"encoding/json"
"fmt"
"log"
"maps"
"slices"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provider"
)
// A build module writes <node>.<language> of its toolchains state on every node it runs on, and nothing deletes the key
// when the module is unassigned from a node: the controller writes no module's state, and the module is gone. A report of
// a node the build module no longer runs on is therefore not a builder (issue 1940: mesh-build-node unassigned from ace
// on 2026-10-23 left ace.typescript at v26.12.0, and every Node fingerprint said "builders disagree").
//
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
// key <node>.<module>) hold an open assignment of its build module on its node, and only once it was written since that
// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report
// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node
// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran
// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
// refusal is the guard of novox/hq ADR 0465, and it stays.
type toolchainGate struct {
mu sync.Mutex
modules map[string]bool // the build modules whose toolchains pass the gate
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
logf func(format string, a ...any)
known bool // the assignments are settled
standing map[string]time.Time // <node>.<module> with an open assignment -> since when it stands
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
left map[string]string // <module>/<key> -> why it was said left out, so each reason is said once
}
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{},
raw: map[string]map[string][]byte{}, left: map[string]string{}}
for _, m := range modules {
g.modules[m] = true
}
return g
}
// Toolchain takes one change of a build module's toolchains bucket in and answers whether what it passed on changed.
func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool) bool {
g.mu.Lock()
defer g.mu.Unlock()
if g.raw[module] == nil {
g.raw[module] = map[string][]byte{}
}
if deleted {
delete(g.raw[module], key)
} else {
g.raw[module][key] = value
}
return g.pass(module, key)
}
// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers
// whether what it passed on changed.
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool {
node, module, ok := strings.Cut(key, ".")
if !ok || !g.modules[module] {
return false
}
assigned, open := openAssignment(value)
open = open && !deleted
g.mu.Lock()
defer g.mu.Unlock()
was, stood := g.standing[key]
since := assigned
if since.IsZero() {
// A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it.
since = at
if stood {
since = was
}
}
if open == stood && (!open || since.Equal(was)) {
return false
}
if open {
g.standing[key] = since
} else {
delete(g.standing, key)
}
if !g.known {
return false
}
changed := false
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
if n, _, err := delivery.ParseToolchainKey(k); err == nil && n == node {
changed = g.pass(module, k) || changed
}
}
return changed
}
// Settled says the assignments are read: from now on only the reports of assigned nodes pass. It answers whether what it
// passed on changed.
func (g *toolchainGate) Settled() bool {
g.mu.Lock()
defer g.mu.Unlock()
g.known = true
changed := false
for _, module := range slices.Sorted(maps.Keys(g.raw)) {
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
changed = g.pass(module, k) || changed
}
}
return changed
}
// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not
// known), deleted otherwise. The caller holds mu.
func (g *toolchainGate) pass(module, key string) bool {
value, held := g.raw[module][key]
if !held {
return g.forward(module, key, nil, true)
}
node, _, err := delivery.ParseToolchainKey(key)
why := ""
if err == nil && g.known {
why = g.leftOut(module, node, value)
}
if why == "" {
delete(g.left, module+"/"+key)
return g.forward(module, key, value, false)
}
if g.left[module+"/"+key] != why {
g.left[module+"/"+key] = why
g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why)
}
return g.forward(module, key, nil, true)
}
// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module,
// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse.
// The caller holds mu.
func (g *toolchainGate) leftOut(module, node string, value []byte) string {
since, open := g.standing[node+"."+module]
if !open {
return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node)
}
tc, err := delivery.DecodeToolchain(value)
if err == nil && tc.At.Before(since) {
return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again",
tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node)
}
return ""
}
// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not
// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when
// it was assigned, zero where the value does not say.
func openAssignment(value []byte) (time.Time, bool) {
var a struct {
Assigned time.Time `json:"assigned"`
Ended time.Time `json:"ended"`
State string `json:"state"`
}
if json.Unmarshal(value, &a) != nil {
return time.Time{}, false
}
return a.Assigned, a.Ended.IsZero() && a.State == ""
}
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
// told whenever what the gate passes on changed. Where the bucket cannot be read (not there, or this module may not read
// it) it says so and the gate passes every report on, as before; an error of the watch after that ends on the channel.
func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate, changed func(context.Context), logger *log.Logger) (<-chan error, error) {
errs := make(chan error, 1)
kv, err := js.KeyValue(ctx, provider.AssignmentsBucket)
if err != nil {
// Not fatal: an indexer, builder or checker that stops for it stops every delivery (a grant comes with a delivery).
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
return errs, nil
}
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
w, err := settle(ctx, kv, each)
if err != nil {
return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err)
}
if gate.Settled() {
changed(ctx)
}
go func() {
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) {
changed(ctx)
}
})
}()
return errs, nil
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing_test.go
-165
View File
@@ -1,165 +0,0 @@
package main
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
import (
"fmt"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
func gateReport(t *testing.T, node, version string) (string, []byte) {
t.Helper()
key, err := delivery.ToolchainKey(node, "typescript")
if err != nil {
t.Fatal(err)
}
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: time.Unix(0, 0).UTC()})
if err != nil {
t.Fatal(err)
}
return key, value
}
func agreedOf(set *buildrun.ToolchainSet) (string, string) {
tc, none := delivery.AgreedToolchain(set.Entries("typescript"))
return tc.Version, none
}
// The live case of issue 1940: mesh-build-node was unassigned from ace, whose report stayed at v26.12.0 while the
// three nodes it runs on report v26.12.1. Without the assignments every report counts (and they disagree); with them
// ace's is left out and the three agree; assigned to ace again, its report counts again and the guard refuses again.
func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
set := &buildrun.ToolchainSet{}
var said []string
gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, f) })
for node, version := range map[string]string{"ace": "v26.12.0", "g14": "v26.12.1", "novox": "v26.12.1", "shanks": "v26.12.1"} {
key, value := gateReport(t, node, version)
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
}
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree") {
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
}
for _, node := range []string{"g14", "novox", "shanks"} {
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{})
}
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{})
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter
if !gate.Settled() {
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
}
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
t.Fatalf("the assigned nodes agree on v26.12.1; got %q, %q", v, none)
}
if len(said) != 1 {
t.Fatalf("ace's report is said left out once; said %v", said)
}
// ace reports again while unassigned (an old value seen again): still left out.
key, value := gateReport(t, "ace", "v26.12.0")
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
t.Fatal("an unassigned node's report changed what was passed on")
}
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) {
t.Fatal("assigning ace again passed nothing on")
}
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
t.Fatalf("builders that run different versions still disagree; got %q", none)
}
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) {
t.Fatal("a pending assignment is no assignment on the machine")
}
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
t.Fatalf("got %q, %q", v, none)
}
gate.Assignment("ace.mesh-build-node", nil, true, time.Time{})
if v, _ := agreedOf(set); v != "v26.12.1" {
t.Fatalf("got %q", v)
}
// A report deleted from the bucket goes whether its node is assigned or not.
key, _ = gateReport(t, "g14", "v26.12.1")
if !gate.Toolchain(buildrun.NodeBuildModule, key, nil, true) {
t.Fatal("deleting g14's report passed nothing on")
}
if n := len(set.Entries("typescript")); n != 2 {
t.Fatalf("novox and shanks are left; got %d entries", n)
}
}
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript
// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node
// writes it again. An assignment whose value names no time stands since the revision that opened it.
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again
report := func(t *testing.T, node, version string, at time.Time) (string, []byte) {
key, _ := gateReport(t, node, version)
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
if err != nil {
t.Fatal(err)
}
return key, value
}
for _, c := range []struct {
name string
assignment string // ace's assignment value
revision time.Time // when the bus wrote it
at time.Time // when ace wrote its report
counts bool
}{
{"no report yet, then one", `{}`, time.Time{}, time.Now(), true},
{"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false},
{"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false},
{"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true},
} {
t.Run(c.name, func(t *testing.T) {
set := &buildrun.ToolchainSet{}
var said []string
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
for _, node := range []string{"g14", "novox", "shanks"} {
gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{})
key, value := gateReport(t, node, "v26.12.1")
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
}
gate.Settled()
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) {
t.Fatal("an assignment with no report passed something on")
}
key, value := report(t, "ace", "v26.12.0", c.at)
if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts {
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
}
// A rewrite of the assignment that does not move it changes nothing, and nothing is said again.
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) {
t.Fatal("the same assignment written again changed what was passed on")
}
v, none := agreedOf(set)
if c.counts {
if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none)
}
return
}
if v != "v26.12.1" || none != "" {
t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none)
}
if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") {
t.Fatalf("said %q, want once why ace's report is left out", said)
}
// ace writes its report again since it was assigned: it counts.
key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute))
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
t.Fatal("ace's new report was not passed on")
}
if n := len(set.Entries("typescript")); n != 4 {
t.Fatalf("four builders report; got %d", n)
}
})
}
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stored.go
-61
View File
@@ -1,61 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"sort"
"strings"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-builder/internal/build"
)
// eventsStream is the stream every module's and role's events are on, where the controller makes the builder's durable
// consumer of artifacts-gone (mesh-controller internal/broker EventsStream).
const eventsStream = "EVENTS"
// toolCaller calls a tool or a seat verb as the builder's own account (mesh-sdk process.Conn.Call).
type toolCaller func(key string, body any) (json.RawMessage, error)
// storedVia is build.Builder's Stored: it asks the controller's verb held (mesh-sdk delivery.HeldVerb; novox/hq issue
// 1937) and answers the references the store no longer holds. One it could not ask about, with none gone, is an error:
// the key is then left done, and the controller's take-in still refuses a gone blob.
func storedVia(call toolCaller) func(ctx context.Context, references []string) ([]string, error) {
return func(_ context.Context, references []string) ([]string, error) {
raw, err := call(delivery.HeldVerb, map[string]any{"references": references})
if err != nil {
return nil, fmt.Errorf("asking %s: %w", delivery.HeldVerb, err)
}
a, err := delivery.DecodeHeldAnswer(raw, references)
if err != nil {
return nil, err
}
if len(a.Gone) == 0 && len(a.Unknown) > 0 {
var why []string
for r, w := range a.Unknown {
why = append(why, r+": "+w)
}
sort.Strings(why)
return nil, fmt.Errorf("the artifact store could not be asked about %s", strings.Join(why, "; "))
}
return a.Gone, nil
}
}
// heardGone takes one artifacts-gone event in: acknowledged, then handed to the builder, which asks the store and builds
// the key again. A body that is not one is said and acknowledged, so it is not handed over again.
func heardGone(ctx context.Context, b *build.Builder, m jetstream.Msg, logf func(string, ...any)) {
e, err := delivery.DecodeArtifactsGone(m.Data())
if ackErr := m.Ack(); ackErr != nil {
logf("acknowledging %s: %v", m.Subject(), ackErr)
}
if err != nil {
logf("%s holds what is not %s, left: %v", m.Subject(), delivery.ArtifactsGoneEvent, err)
return
}
b.ArtifactsGone(ctx, e)
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go
+6 -8
View File
@@ -23,11 +23,9 @@ var toolchainStreams = map[string]string{buildrun.GoBuildModule: "TOOLCHAINS_GO"
// §8): it watches the buckets that exist at start, and a build module says each time it wrote its toolchains key
// (toolchain.reported, handled by Reported); a bucket not watched yet is then watched.
type toolchainWatch struct {
ctx context.Context
js bucketOpener
set *buildrun.ToolchainSet
// put takes a change of a toolchains bucket in: set.Put, or the gate of the assigned build nodes before it (issue 1940).
put func(module, key string, value []byte, deleted bool) bool
ctx context.Context
js bucketOpener
set *buildrun.ToolchainSet
changed func(ctx context.Context) // a language's entries changed: the failed builds are tried again
logger *log.Logger
errs chan error
@@ -36,7 +34,7 @@ type toolchainWatch struct {
}
func newToolchainWatch(ctx context.Context, js bucketOpener, set *buildrun.ToolchainSet, changed func(context.Context), logger *log.Logger) *toolchainWatch {
return &toolchainWatch{ctx: ctx, js: js, set: set, put: set.Put, changed: changed, logger: logger, errs: make(chan error, len(buildrun.LanguageOf)), watched: map[string]bool{}}
return &toolchainWatch{ctx: ctx, js: js, set: set, changed: changed, logger: logger, errs: make(chan error, len(buildrun.LanguageOf)), watched: map[string]bool{}}
}
// start watches the buckets that exist now; a missing one is said and waits for its build module's event.
@@ -64,14 +62,14 @@ func (w *toolchainWatch) ensure(module string) error {
case err != nil:
return fmt.Errorf("the bucket %s: %w", name, err)
}
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.put(module, e.Key(), e.Value(), !opOf(e)) })
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.set.Put(module, e.Key(), e.Value(), !opOf(e)) })
if err != nil {
return err
}
w.watched[module] = true
go func() {
w.errs <- follow(w.ctx, watcher, func(e jetstream.KeyValueEntry) {
if w.put(module, e.Key(), e.Value(), !opOf(e)) {
if w.set.Put(module, e.Key(), e.Value(), !opOf(e)) {
w.changed(w.ctx)
}
})
modules/mesh-delivery-builder/go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-builder
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
modules/mesh-delivery-builder/go.sum
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1 h1:fzOkrz9CSp3mEjeV7NfZx8yoivI1sD1AOtIPtyyF8RI=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
modules/mesh-delivery-builder/internal/build/candidates.go
+1 -24
View File
@@ -73,17 +73,9 @@ func (b *Builder) considerCandidate(ctx context.Context, c delivery.Candidate) {
// The named paths' objects the preview gave it: the fetch names the paths (runLane, ADR 0499 clause 4).
m.Inputs2 = &delivery.KeptInputs2{Paths: cm.Paths}
}
// The preview is merged onto the trunk head the candidate was previewed onto: the head of the pull request's
// repository in the gate's source revision c.Base (ADR 0484 §3). Never the module's value's commit: that is the
// commit its fingerprint first appeared at, and may be many merges behind the trunk.
base, err := b.trunkHead(ctx, c.Base, p.Repository)
if err != nil {
b.logf("candidates.%s: the module %s's fingerprint %s is not built: %v", c.Group, name, cm.FP2, err)
continue
}
b.mu.Lock()
if _, onTrunk := b.trunkFP[key]; !onTrunk {
b.via[key] = via{Ref: fmt.Sprintf("refs/pull/%d/head", p.Number), Base: base}
b.via[key] = via{Ref: fmt.Sprintf("refs/pull/%d/head", p.Number), Base: trunk.Commit}
}
b.mu.Unlock()
b.consider(ctx, name, m)
@@ -114,18 +106,3 @@ func (b *Builder) agreedFor(ctx context.Context, manifest string) map[string]del
}
return out
}
// trunkHead is the head of repository in the gate's source revision rev (Builder.Trunk), or why it cannot be said.
func (b *Builder) trunkHead(ctx context.Context, rev uint64, repository string) (string, error) {
if b.Trunk == nil {
return "", fmt.Errorf("the gate's source cannot be read here, so the trunk head %s was previewed onto is not known", repository)
}
head, err := b.Trunk(ctx, rev, repository)
if err != nil {
return "", fmt.Errorf("the trunk head of %s at source revision %d cannot be read: %w", repository, rev, err)
}
if head == "" {
return "", fmt.Errorf("source revision %d names no trunk head of %s", rev, repository)
}
return head, nil
}
modules/mesh-delivery-builder/internal/build/candidates_test.go
+4 -71
View File
@@ -2,13 +2,9 @@ package build
import (
"context"
"encoding/json"
"fmt"
"os"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -44,74 +40,11 @@ func candidateValue(t *testing.T, group string, number int, head string) []byte
}
const (
// trunkHead is the trunk head the rig's source revision names: newer than the module value's commit, which stays at the
// commit its fingerprint first appeared at.
trunkHead = "3333333333333333333333333333333333333333"
headX = "1111111111111111111111111111111111111111"
headY = "2222222222222222222222222222222222222222"
headX = "1111111111111111111111111111111111111111"
headY = "2222222222222222222222222222222222222222"
)
// indexerCandidate is a candidate as the indexer writes it, with the gate's source revision it names (mesh-delivery-indexer
// internal/index testdata/candidate-written.json, kept equal to what the indexer writes by its own test).
func indexerCandidate(t *testing.T) (rev uint64, heads map[string]string, c delivery.Candidate, raw []byte) {
t.Helper()
data, err := os.ReadFile("../../../mesh-delivery-indexer/internal/index/testdata/candidate-written.json")
if err != nil {
t.Fatal(err)
}
var w struct {
Source struct {
Rev uint64 `json:"rev"`
Heads map[string]string `json:"heads"`
} `json:"source"`
Candidate json.RawMessage `json:"candidate"`
}
if err := json.Unmarshal(data, &w); err != nil {
t.Fatal(err)
}
if c, err = delivery.DecodeCandidate(w.Candidate); err != nil {
t.Fatal(err)
}
return w.Source.Rev, w.Source.Heads, c, w.Candidate
}
func TestACandidatesFingerprintIsBuiltOnceAsAKeyOfItsOwnFromTheHeadMergedOntoTheTrunk(t *testing.T) { // N7/AC1, N7/AC2 (issue 1684)
// The candidate the indexer writes is merged onto the trunk head of the source revision it names, never onto the module
// value's commit, which stays at the commit its fingerprint first appeared at (review of the indexer writing on change).
{
rev, heads, c, raw := indexerCandidate(t)
p := c.Pulls[0]
cm := p.Fps["avahi"]
ir := newRig(t, goBundle)
ir.mf.byDigest[cm.Manifest] = catalogue.Manifest{Module: "avahi", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}}
ir.b.Trunk = func(_ context.Context, at uint64, repository string) (string, error) {
if at != rev {
return "", fmt.Errorf("source revision %d is not heard", at)
}
return heads[repository], nil
}
const valueCommit = "7a74cee83f88e6585a5511c3cc53bdf0f5303d20"
value, err := delivery.EncodeModule(delivery.Module{Repository: p.Repository, Commit: valueCommit, FP2: "src2:" + hex64('e'),
Manifest: cm.Manifest, Path: "modules/avahi", Toolchains: agreedToolchains})
if err != nil {
t.Fatal(err)
}
ir.b.Module(context.Background(), "avahi", value)
ir.b.Wait()
ir.bm.mu.Lock()
ir.bm.calls = nil
ir.bm.mu.Unlock()
ir.b.Candidate(context.Background(), "feat%2Fx", raw)
ir.b.Wait()
fetches := ir.bm.callsOf("build_go_fetch")
if len(fetches) != 1 {
t.Fatalf("%d fetches of the indexer's candidate, want one", len(fetches))
}
f := fetches[0].body.(map[string]any)
if want := heads[p.Repository]; want == "" || f["base"] != want || f["base"] == valueCommit || f["commit"] != p.Head.Head {
t.Errorf("fetch args = %+v, want the head merged onto the trunk head %.8s of source revision %d", f, want, rev)
}
}
r := newRig(t, goBundle)
trunkModule(t, r)
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
@@ -125,8 +58,8 @@ func TestACandidatesFingerprintIsBuiltOnceAsAKeyOfItsOwnFromTheHeadMergedOntoThe
t.Fatalf("%d fetches, want one", len(fetches))
}
f := fetches[0].body.(map[string]any)
if f["commit"] != headX || f["ref"] != "refs/pull/7/head" || f["base"] != trunkHead || f["repository"] != "novox/mesh-catalog" {
t.Errorf("fetch args = %+v, want the head, its ref and the trunk head the candidate was previewed onto", f)
if f["commit"] != headX || f["ref"] != "refs/pull/7/head" || f["base"] != "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" || f["repository"] != "novox/mesh-catalog" {
t.Errorf("fetch args = %+v, want the head, its ref and the trunk commit to merge onto", f)
}
if b := r.bm.callsOf("build_go_build")[0].body.(map[string]any); b["module"] != "modules/m" {
t.Errorf("build args = %+v", b)
modules/mesh-delivery-builder/internal/build/claim.go
-54
View File
@@ -4,7 +4,6 @@ import (
"context"
"errors"
"fmt"
"slices"
"sort"
"strings"
@@ -38,11 +37,6 @@ type Claim struct {
Key string
Rev uint64
Build delivery.BuilderBuild
// Gone are the references of a done key the artifact store no longer holds, for which it was claimed again
// (ClaimOptions.Stored; issue 1937): said in the log and the key's step.
Gone []string
// Unasked is why the store could not be asked about a done key taken as it stands to publish: said in the log.
Unasked error
}
// ClaimOptions say what the builder knows about the moment of a claim.
@@ -61,30 +55,6 @@ type ClaimOptions struct {
// a package's reference. A key queued waiting for a version (BuilderBuild.Waits) is taken only once it is (ADR 0500
// clause 6). Nil means none is.
Published func(ref string) bool
// Stored answers which of the references given the artifact store no longer holds (the controller's verb held,
// mesh-sdk delivery.HeldVerb; novox/hq issue 1937). A done key of the trunk's fingerprint is asked before it is left
// done: the store's nightly collector deletes a blob nothing holds, and a key the controller never took in holds
// nothing, so a key whose blob is gone is claimed building as a new revision and built again. Nil asks nothing.
Stored func(ctx context.Context, references []string) (gone []string, err error)
}
// StoredReferences are the references of a done build's bundles and archives that name a blob in the artifact store
// (`artifact-store://<module>/<artifact>/blobs/<digest>`): what the store's collector can delete. An image or an upstream
// copy is a manifest, which the collector keeps, and a package lives in the package registry.
func StoredReferences(b delivery.BuilderBuild) []string {
var out []string
for _, a := range b.Artifacts {
if strings.HasPrefix(a.Reference, "artifact-store://") && strings.Contains(a.Reference, "/blobs/") &&
!slices.Contains(out, a.Reference) {
out = append(out, a.Reference)
}
}
return out
}
// GoneStep is the step a key claimed again for gone blobs says (issue 1937).
func GoneStep(gone []string) string {
return "claimed again: the artifact store no longer holds " + strings.Join(gone, ", ")
}
// Unpublished is whether a done build holds an image built without a push or a package built without a publish: it has
@@ -148,8 +118,6 @@ func Unreferenced(b delivery.BuilderBuild) bool {
// names it (ClaimOptions.Trunk): that is taken as it stands, with the done value, and its builder writes the new
// revision when the publishing build ends (ADR 0498 clause 6). From R4e one node builds each fingerprint (ADR 0478
// clause 4), so a key built on one node is never built again for a second;
// - a done key of the trunk's fingerprint whose bundle or archive blob the artifact store no longer holds
// (ClaimOptions.Stored, issue 1937) is claimed building as a new revision and built whole, its step saying which;
// - a failed key is taken again as a new revision of the same key (ADR 0429 clause 18), except one failed for a kind no
// builder builds, for an upstream pinned by no digest or a package version taken (delivery.FailedUntilChanged: ADR 0478
// clause 2, ADR 0498 clauses 4 and 5) or for digests that differ (buildrun.DigestsDifferPrefix, clause 4),
@@ -181,28 +149,6 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string
if Unreferenced(cur) {
return c.write(ctx, key, building, func(data []byte) (uint64, error) { return c.Store.Update(ctx, key, data, rev) })
}
if opt.Trunk && opt.Stored != nil {
if refs := StoredReferences(cur); len(refs) > 0 {
gone, err := opt.Stored(ctx, refs)
if err != nil {
err = fmt.Errorf("%s: whether the artifact store still holds %s cannot be asked, so the key is "+
"left done (the controller's take-in refuses it if one is gone): %w", key, strings.Join(refs, ", "), err)
if len(publishable(cur, opt.ImageBuilder)) > 0 {
return &Claim{Key: key, Rev: rev, Build: cur, Unasked: err}, nil
}
return nil, err
}
if len(gone) > 0 {
again := building
again.Step = GoneStep(gone)
cl, err := c.write(ctx, key, again, func(data []byte) (uint64, error) { return c.Store.Update(ctx, key, data, rev) })
if cl != nil {
cl.Gone = gone
}
return cl, err
}
}
}
if opt.Trunk && len(publishable(cur, opt.ImageBuilder)) > 0 {
return &Claim{Key: key, Rev: rev, Build: cur}, nil
}
modules/mesh-delivery-builder/internal/build/gone_test.go
-204
View File
@@ -1,204 +0,0 @@
package build
import (
"context"
"errors"
"fmt"
"slices"
"strings"
"sync"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// A done key whose blob the artifact store no longer holds is built again (novox/hq issue 1937): on 2026-10-24 the
// store's 03:30 collection deleted the bare blobs of node-tools', mesh-host's and the forge bridge's done keys, which the
// controller had never taken in, and the builder kept them done.
// fakeHeld answers the controller's verb held: the references in gone are gone.
type fakeHeld struct {
mu sync.Mutex
gone map[string]bool
err error
asked [][]string
}
func (f *fakeHeld) stored(_ context.Context, refs []string) ([]string, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.asked = append(f.asked, slices.Clone(refs))
if f.err != nil {
return nil, f.err
}
var out []string
for _, r := range refs {
if f.gone[r] {
out = append(out, r)
}
}
return out, nil
}
func (f *fakeHeld) asks() int {
f.mu.Lock()
defer f.mu.Unlock()
return len(f.asked)
}
// logged keeps what the builder says.
type logged struct {
mu sync.Mutex
lines []string
}
func (l *logged) logf(format string, args ...any) {
l.mu.Lock()
defer l.mu.Unlock()
l.lines = append(l.lines, fmt.Sprintf(format, args...))
}
func (l *logged) has(words ...string) bool {
l.mu.Lock()
defer l.mu.Unlock()
for _, line := range l.lines {
all := true
for _, w := range words {
all = all && strings.Contains(line, w)
}
if all {
return true
}
}
return false
}
const goneRef = "artifact-store://m/tool/blobs/" + digest
func doneKeyNaming(ref string) delivery.BuilderBuild {
return delivery.BuilderBuild{State: delivery.BuildDone, By: "shanks.builder", Manifest: manifestD, Step: "built mesh-build-go on shanks",
Artifacts: []delivery.BuiltArtifact{{Name: "tool", Kind: "bundle", Digest: digest, Reference: ref}}}
}
// 1937/AC3 (unit, the claim): a done key of the trunk's fingerprint whose blob is gone is claimed building as a new
// revision, its step naming the gone reference; a candidate's done key is not asked about.
func TestADoneTrunkKeyWhoseBlobIsGoneIsClaimedAgain(t *testing.T) {
st := newMem()
c := Claimer{Store: st, By: "novox.builder"}
key, _ := delivery.BuilderBuildKey("m", fp1)
rev := putBuild(t, st, key, doneKeyNaming(goneRef))
held := &fakeHeld{gone: map[string]bool{goneRef: true}}
cl, err := c.Claim(t.Context(), "m", fp1, manifestD, ClaimOptions{Stored: held.stored})
if err != nil || cl != nil || held.asks() != 0 {
t.Fatalf("a candidate's done key: claim %+v, %v, %d asks; want nothing and no ask", cl, err, held.asks())
}
cl, err = c.Claim(t.Context(), "m", fp1, manifestD, ClaimOptions{Trunk: true, Stored: held.stored})
if err != nil || cl == nil {
t.Fatalf("claim = %+v, %v; want the key claimed again", cl, err)
}
if cl.Rev <= rev || cl.Build.State != delivery.BuildBuilding || !slices.Equal(cl.Gone, []string{goneRef}) {
t.Errorf("claim = %+v (revision was %d); want building as a new revision naming the gone reference", cl, rev)
}
v := valueOf(t, st, key)
if v.State != delivery.BuildBuilding || !strings.Contains(v.Step, "no longer holds") || !strings.Contains(v.Step, goneRef) {
t.Errorf("the key holds %+v; want building, its step naming the gone reference", v)
}
}
// 1937/AC5 (unit, the claim): a done trunk key whose blobs are all held stays done; a store that cannot be asked leaves
// it done and says why; images and packages are not asked about.
func TestADoneTrunkKeyStaysDoneWhileTheStoreHoldsItsBlobs(t *testing.T) {
st := newMem()
c := Claimer{Store: st, By: "novox.builder"}
key, _ := delivery.BuilderBuildKey("m", fp1)
b := doneKeyNaming(goneRef)
b.Artifacts = append(b.Artifacts, delivery.BuiltArtifact{Name: "img", Kind: "image", Digest: digest, Reference: "registry.example/m/img@" + digest})
rev := putBuild(t, st, key, b)
held := &fakeHeld{gone: map[string]bool{}}
if cl, err := c.Claim(t.Context(), "m", fp1, manifestD, ClaimOptions{Trunk: true, Stored: held.stored}); err != nil || cl != nil {
t.Fatalf("claim = %+v, %v; want nothing: the store holds the blob", cl, err)
}
if len(held.asked) != 1 || !slices.Equal(held.asked[0], []string{goneRef}) {
t.Errorf("asked %v; want the bundle's reference alone", held.asked)
}
held.err = errors.New("the controller did not answer")
cl, err := c.Claim(t.Context(), "m", fp1, manifestD, ClaimOptions{Trunk: true, Stored: held.stored})
if cl != nil || err == nil || !strings.Contains(err.Error(), "cannot be asked") || !strings.Contains(err.Error(), "left done") {
t.Errorf("claim = %+v, %v; want the key left done and said", cl, err)
}
if st.revs[key] != rev {
t.Errorf("the key moved to revision %d (was %d)", st.revs[key], rev)
}
}
// 1937/AC3 (unit, the builder): the builder's look at a module whose done key's blob is gone builds it again, says so
// in a log line and in the done key's step, and asks the store once per process while it answers held.
func TestTheBuilderBuildsAgainADoneKeyWhoseBlobIsGone(t *testing.T) {
r := newRig(t, goBundle)
log := &logged{}
r.b.Logf = log.logf
held := &fakeHeld{gone: map[string]bool{goneRef: true}}
r.b.Stored = held.stored
rev := putBuild(t, r.st, r.key, doneKeyNaming(goneRef))
v := r.consider(t, "modules/m")
if v.State != delivery.BuildDone || r.st.revs[r.key] <= rev || len(r.bm.callsOf("build_go_build")) == 0 {
t.Fatalf("build = %+v at revision %d (was %d); want built again", v, r.st.revs[r.key], rev)
}
if !strings.Contains(v.Step, "built again: the artifact store no longer held "+goneRef) {
t.Errorf("the done key's step %q does not say it was built again for the gone blob", v.Step)
}
if !log.has(r.key, "built again", goneRef) {
t.Errorf("no log line says the key was built again: %v", log.lines)
}
// The new revision's blob is held: asked once, then not again in this process.
held.mu.Lock()
held.gone = map[string]bool{}
held.mu.Unlock()
rev = r.st.revs[r.key]
asks := held.asks()
r.consider(t, "modules/m")
r.consider(t, "modules/m")
if r.st.revs[r.key] != rev {
t.Errorf("a key whose blob is held was built again")
}
if held.asks() != asks+1 {
t.Errorf("asked the store %d times for two looks at a held key; want once", held.asks()-asks)
}
}
// 1937/AC4 (unit, the builder): artifacts-gone for the module's fingerprint makes the builder ask again, even for a key
// it found held before, and build it again; one for another fingerprint builds nothing.
func TestArtifactsGoneMakesTheBuilderAskAndBuildAgain(t *testing.T) {
r := newRig(t, goBundle)
log := &logged{}
r.b.Logf = log.logf
held := &fakeHeld{gone: map[string]bool{}}
r.b.Stored = held.stored
putBuild(t, r.st, r.key, doneKeyNaming(goneRef))
if v := r.consider(t, "modules/m"); v.State != delivery.BuildDone || len(r.bm.callsOf("build_go_build")) != 0 {
t.Fatalf("a key whose blob is held was built: %+v", v)
}
rev := r.st.revs[r.key]
held.mu.Lock()
held.gone[goneRef] = true // collected after the builder's look; the controller's take-in found it
held.mu.Unlock()
r.b.ArtifactsGone(context.Background(), delivery.ArtifactsGone{Event: delivery.ArtifactsGoneEvent, Module: "m",
Fingerprint: "src2:" + strings.Repeat("9", 64), Gone: []string{goneRef}})
r.b.Wait()
if r.st.revs[r.key] != rev || !log.has("a fingerprint the module does not have now") {
t.Fatalf("an event for another fingerprint moved the key (revision %d, was %d): %v", r.st.revs[r.key], rev, log.lines)
}
r.b.ArtifactsGone(context.Background(), delivery.ArtifactsGone{Event: delivery.ArtifactsGoneEvent, Module: "m",
Key: r.key, Fingerprint: fp1, Gone: []string{goneRef}})
r.b.Wait()
v := valueOf(t, r.st, r.key)
if v.State != delivery.BuildDone || r.st.revs[r.key] <= rev || len(r.bm.callsOf("build_go_build")) == 0 {
t.Fatalf("build = %+v at revision %d (was %d); want built again on the event", v, r.st.revs[r.key], rev)
}
if !log.has("heard artifacts-gone") || !log.has(r.key, "built again", goneRef) {
t.Errorf("the builder did not say what it heard and did: %v", log.lines)
}
}
modules/mesh-delivery-builder/internal/build/r4e_test.go
+2 -3
View File
@@ -379,9 +379,8 @@ func TestACandidatesFetchNamesTheNamedPathsTheIndexerGaveIt(t *testing.T) { // A
t.Fatal("no fetch")
}
req, err := buildstep.ParseFetchRequest(f[0].body.(map[string]any))
// The base is the trunk head the candidate was previewed onto, never the module value's commit (an older one).
if err != nil || req.Dir != "modules/m" || strings.Join(req.Paths, ",") != "package.json,src" || req.Base != trunkHead {
t.Errorf("fetch request = %+v, %v; want the trunk head %.8s as the base, the module's directory and its named paths", req, err, trunkHead)
if err != nil || req.Dir != "modules/m" || strings.Join(req.Paths, ",") != "package.json,src" || req.Base == "" {
t.Errorf("fetch request = %+v, %v; want the candidate's base, the module's directory and its named paths", req, err)
}
}
modules/mesh-delivery-builder/internal/build/run.go
+2 -78
View File
@@ -50,12 +50,6 @@ type Builder struct {
// Answered is told the build module whose step answered a reference (issue 1886): what the builder says it writes
// rests on it. Nil tells nobody.
Answered func(buildModule string)
// Stored answers which references the artifact store no longer holds (the controller's verb held; issue 1937): a done
// key of the trunk's fingerprint is asked before it is left done (ClaimOptions.Stored). Nil asks nothing.
Stored func(ctx context.Context, references []string) (gone []string, err error)
// Trunk answers the head of a repository in the gate's source revision rev (source.main at that revision): the trunk a
// candidate's preview is merged onto (Candidate.Base). Nil builds no candidate's fingerprint, and says why.
Trunk func(ctx context.Context, rev uint64, repository string) (string, error)
once sync.Once
sem chan struct{}
@@ -72,9 +66,6 @@ type Builder struct {
// published are the package versions `<name>@<version>` a done key of this bucket names as a package's reference: a key
// waiting for one is built once it is here (ADR 0500 clause 6).
published map[string]bool
// held are the done keys' references the store answered held, by `<key> <references>`, asked once per process: a
// blob collected later is found by the take-in, whose artifacts-gone forgets the key here (issue 1937).
held map[string]bool
}
type planned struct {
@@ -91,7 +82,7 @@ func (b *Builder) init() {
b.sem = make(chan struct{}, limit)
b.cands, b.via, b.trunkFP = map[string]delivery.Candidate{}, map[string]via{}, map[string]bool{}
b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{}
b.published, b.held = map[string]bool{}, map[string]bool{}
b.published = map[string]bool{}
})
}
@@ -211,55 +202,6 @@ func (b *Builder) Forget(key string) {
b.mu.Unlock()
}
// storedOnce is Stored for one key, asked once per process while the store answers every reference held.
func (b *Builder) storedOnce(key string) func(ctx context.Context, references []string) ([]string, error) {
if b.Stored == nil {
return nil
}
return func(ctx context.Context, references []string) ([]string, error) {
id := key + " " + strings.Join(references, " ")
b.mu.Lock()
known := b.held[id]
b.mu.Unlock()
if known {
return nil, nil
}
gone, err := b.Stored(ctx, references)
if err == nil && len(gone) == 0 {
b.mu.Lock()
b.held[id] = true
b.mu.Unlock()
}
return gone, err
}
}
// ArtifactsGone takes the controller's event artifacts-gone in (issue 1937): a take-in found that the store does not
// hold what the module's key names. The module is looked at again, which asks the store (ClaimOptions.Stored) and
// builds the key again; an event for a fingerprint the module no longer has is said and left. The event only wakes the
// builder: what is built rests on the store's answer, never on the event alone.
func (b *Builder) ArtifactsGone(ctx context.Context, e delivery.ArtifactsGone) {
b.init()
b.mu.Lock()
m, ok := b.known[e.Module]
if key, err := delivery.BuilderBuildKey(e.Module, e.Fingerprint); err == nil {
for id := range b.held {
if strings.HasPrefix(id, key+" ") {
delete(b.held, id)
}
}
}
b.mu.Unlock()
if !ok || m.FP2 != e.Fingerprint {
b.logf("%s: heard %s for %s, a fingerprint the module does not have now: nothing to build", e.Module,
delivery.ArtifactsGoneEvent, e.Fingerprint)
return
}
b.logf("%s: heard %s (%s): asking the artifact store and building the key again", e.Module,
delivery.ArtifactsGoneEvent, strings.Join(e.Gone, ", "))
b.consider(ctx, e.Module, m)
}
// Reconsider looks at every module again: the toolchains changed, so a build that failed for them may build now.
func (b *Builder) Reconsider(ctx context.Context) {
b.init()
@@ -272,13 +214,6 @@ func (b *Builder) Reconsider(ctx context.Context) {
for _, k := range sortedKeys(known) {
b.consider(ctx, k, known[k])
}
b.TrunkMoved(ctx)
}
// TrunkMoved considers every candidate again once a revision of the gate's source is heard: a candidate heard before the
// revision it was previewed onto had no trunk head to be merged onto.
func (b *Builder) TrunkMoved(ctx context.Context) {
b.init()
b.mu.Lock()
cands := make([]delivery.Candidate, 0, len(b.cands))
for _, c := range b.cands {
@@ -377,8 +312,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
first := !b.tried[key]
b.tried[key] = true
b.mu.Unlock()
cl, err := b.Claimer.Claim(ctx, module, m.FP2, m.Manifest, ClaimOptions{TakeOver: first, ImageBuilder: imageSeat, Trunk: trunk, Published: b.isPublished,
Stored: b.storedOnce(key)})
cl, err := b.Claimer.Claim(ctx, module, m.FP2, m.Manifest, ClaimOptions{TakeOver: first, ImageBuilder: imageSeat, Trunk: trunk, Published: b.isPublished})
if err != nil {
b.logf("%s: %v", module, err)
return
@@ -386,13 +320,6 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
if cl == nil {
return
}
if cl.Unasked != nil {
b.logf("%v", cl.Unasked)
}
if len(cl.Gone) > 0 {
// Said aloud (issue 1937): the key was done, and the store's collector deleted what it names.
b.logf("%s: built again: the artifact store no longer holds %s", cl.Key, strings.Join(cl.Gone, ", "))
}
// A done key taken again builds only what is missing: the image or the package built unpublished as a candidate's; the
// other artifacts stay as the done key has them (ADR 0487 clauses 5 and 8, ADR 0498 clause 6).
var prior *delivery.BuilderBuild
@@ -423,9 +350,6 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
return
}
result.By, result.Manifest = b.Claimer.By, m.Manifest
if len(cl.Gone) > 0 && result.State == delivery.BuildDone {
result.Step += " (built again: the artifact store no longer held " + strings.Join(cl.Gone, ", ") + ")"
}
if publishing && result.State == delivery.BuildFailed {
// The trunk's image or package was built only as a candidate's and its push or publish failed: the key says so, so the builds view, the
// checker and the failed-key retake see it, and a look does not build the image again for nothing.
modules/mesh-delivery-builder/internal/build/run_test.go
-1
View File
@@ -176,7 +176,6 @@ func newRig(t *testing.T, artifacts ...catalogue.Artifact) *rig {
r.b = &Builder{
Claimer: Claimer{Store: r.st, By: "novox.builder"}, Asks: buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 2 * time.Second},
Toolchains: r.tools, Manifests: r.mf, Logf: t.Logf,
Trunk: func(context.Context, uint64, string) (string, error) { return trunkHead, nil },
}
var err error
if r.key, err = delivery.BuilderBuildKey("m", fp1); err != nil {
modules/mesh-delivery-builder/manifest_test.go
+7 -10
View File
@@ -89,21 +89,18 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
}
// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and
// holds no capacity; it puts asks on the three build seats and reads their steps (ADR 0525 §1, §4, §5).
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-controller.assignments",
"mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules"}; !reflect.DeepEqual(sorted(m.Reads), want) {
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains",
"mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules"}; !reflect.DeepEqual(sorted(m.Reads), want) {
t.Errorf("reads = %v, want %v", m.Reads, want)
}
if want := []string{"node-build-go", "node-build-image", "node-build-node"}; !reflect.DeepEqual(sorted(m.Uses), want) {
t.Errorf("uses = %v, want the three build seats", m.Uses)
}
// 1937/AC3: the builder asks the controller whether the artifact store still holds a done key's blobs (held), and no
// other tool; it holds no capacity.
if want := []string{"seat:mesh-controller.held"}; !reflect.DeepEqual(m.Invokes, want) || len(m.Config) != 0 {
t.Errorf("invokes = %v, config = %v, want %v and no config", m.Invokes, m.Config, want)
if len(m.Invokes) != 0 || len(m.Config) != 0 {
t.Errorf("invokes = %v, config = %v, want none: no tool call from the builder and no capacity", m.Invokes, m.Config)
}
// The builder is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it
// starts; and by the controller's artifacts-gone, a take-in refused for a gone blob (1937/AC4).
if want := []string{"mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-controller.artifacts-gone"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
// The builder is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it starts.
if want := []string{"mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
t.Errorf("consumes = %v, want %v", m.Consumes, want)
}
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
@@ -112,7 +109,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
if m.SecretsOwner != "mesh-delivery-builder" {
t.Errorf("secrets-owner = %q", m.SecretsOwner)
}
for _, k := range []string{"streams", "logs", "emits", "claims", "provides", "requires", "defines-seats", "tools", "seat", "seats", "bus", "config"} {
for _, k := range []string{"streams", "logs", "emits", "claims", "provides", "requires", "defines-seats", "tools", "seat", "seats", "bus", "invokes", "config"} {
if _, ok := top[k]; ok {
t.Errorf("the manifest declares %q: the builder has none", k)
}
modules/mesh-delivery-builder/module.json
+1 -7
View File
@@ -4,11 +4,7 @@
"slug": "builder",
"consumes": [
"mesh-build-go.toolchain.reported",
"mesh-build-node.toolchain.reported",
"mesh-controller.artifacts-gone"
],
"invokes": [
"seat:mesh-controller.held"
"mesh-build-node.toolchain.reported"
],
"state": [
{ "name": "builds", "history": 5 },
@@ -20,8 +16,6 @@
"mesh-build-image.steps",
"mesh-build-node.steps",
"mesh-build-node.toolchains",
"mesh-controller.assignments",
"mesh-delivery-gate.source",
"mesh-delivery-indexer.candidates",
"mesh-delivery-indexer.manifests",
"mesh-delivery-indexer.modules"
modules/mesh-delivery-builder/reads-outside
-1
View File
@@ -1 +0,0 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/bus.go
+1 -8
View File
@@ -121,12 +121,5 @@ func (m manifestReader) Manifest(ctx context.Context, digest string) (catalogue.
if err != nil {
return catalogue.Manifest{}, err
}
return parseManifest(kept.JSON)
}
// parseManifest reads a manifest the indexer kept as the indexer and the assigner read it (issues 1751, 1906): a config value that
// falls back to a predefined key is judged by the merge gate in the catalogue, against the predefined.json at its root that this
// process never holds, so it is not unreadable here (novox/hq ADR 0348 §7).
func parseManifest(raw string) (catalogue.Manifest, error) {
return catalogue.ParseManifestForInputs([]byte(raw))
return catalogue.ParseManifest([]byte(kept.JSON))
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/forge.go
-25
View File
@@ -20,8 +20,6 @@ const (
getFileTool = "gitea.gitea_get_file"
compareTool = "gitea.gitea_compare"
commitsTool = "gitea.gitea_commits"
// gitea_list_branches answers {branches: [{name, commit}]}, every branch with the commit it points at.
branchesTool = "gitea.gitea_list_branches"
)
// notFoundStatus is the 404 of the forge's error, `Gitea API <path>: <status> <body>`.
@@ -104,26 +102,3 @@ func (f forgeReads) FullSHA(_ context.Context, repository, commit string) (strin
}
return a.Commits[0].SHA, nil
}
// BranchHeads is the commit each branch of the repository points at.
func (f forgeReads) BranchHeads(_ context.Context, repository string) (map[string]string, error) {
owner, repo := ownerRepo(repository)
raw, err := f.ask(branchesTool, map[string]string{"owner": owner, "repo": repo})
if err != nil {
return nil, err
}
var a struct {
Branches *[]struct {
Name string `json:"name"`
Commit string `json:"commit"`
} `json:"branches"`
}
if err := json.Unmarshal(raw, &a); err != nil || a.Branches == nil {
return nil, fmt.Errorf("the answer of %s is not {branches: [{name, commit}]}: %v", branchesTool, err)
}
heads := map[string]string{}
for _, b := range *a.Branches {
heads[b.Name] = b.Commit
}
return heads, nil
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go
+9 -40
View File
@@ -10,7 +10,6 @@ import (
"io"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"sync/atomic"
@@ -82,9 +81,6 @@ type fakeBuildModules struct {
mu sync.Mutex
calls map[string]int // by `<module>.<tool>`
testTool *testing.T
// beside is, by workspace, the siblings the fetch placed in it as `<owner/repo> <commit>`, sorted: a digest echoes them
// (1929/AC2; mesh-sdk buildstep.besideOf), or the level runs whole and keeps nothing.
beside map[string][]string
}
func (f *fakeBuildModules) count(tool string) int {
@@ -104,30 +100,10 @@ func (f *fakeBuildModules) result(t *testing.T, tool string, args map[string]any
}
fetched := readCaptured(t, name)
fetched["commit"], fetched["base"] = args["commit"], args["base"]
var refs []struct {
Repository string `json:"repository"`
Commit string `json:"commit"`
}
raw, _ := json.Marshal(args["beside"])
_ = json.Unmarshal(raw, &refs)
var lines []string
for _, r := range refs {
lines = append(lines, r.Repository+" "+r.Commit)
}
sort.Strings(lines)
f.mu.Lock()
if f.beside == nil {
f.beside = map[string][]string{}
}
f.beside[fmt.Sprint(fetched["workspace"])] = lines
f.mu.Unlock()
v = fetched
case strings.HasSuffix(tool, "_digest"):
d := map[string]string{"unit": digU, "integration": digI}[level]
f.mu.Lock()
beside := f.beside[fmt.Sprint(args["workspace"])]
f.mu.Unlock()
v = map[string]any{"level": level, "packages": map[string]string{"fixture/slow": d}, "beside": beside}
v = map[string]any{"level": level, "packages": map[string]string{"fixture/slow": d}}
case strings.HasSuffix(tool, "_test"):
r := readCaptured(t, "go-test-slow")
r["level"] = level
@@ -312,15 +288,6 @@ func startBed(t *testing.T) *bed {
}
b.buckets[name] = kv
}
// 1929/AC1: the indexer's value of the repository's module, with no TypeScript manifest, so the Go build module alone
// judges the repository (an unknown repository is judged by both, issue 1887 A12).
known, err := delivery.EncodeModule(delivery.Module{Repository: repo, Commit: trunk, Path: "modules/m", None2: "the fingerprint is computed with its chain"})
if err != nil {
t.Fatal(err)
}
if _, err := b.buckets[delivery.ModulesBucket].Put(ctx, "m", known); err != nil {
t.Fatal(err)
}
b.setOnline(t, "n1", true)
for _, module := range []string{"mesh-build-go", "mesh-build-node"} {
b.setHealth(t, "n1", module)
@@ -469,7 +436,7 @@ func (b *bed) owed(t *testing.T, number int, head, state string) delivery.Owed {
// TestCheckerJudgesACandidateAndOwesItsStatus: a candidate the indexer wrote, one pull request of mesh-delivery, is fetched
// merged onto the trunk the gate names, digested and tested at both levels on the node, kept, previewed; the verdict passes
// with the counts of both levels and the composition not run, and the status mesh/check is owed on the head as success.
func TestCheckerJudgesACandidateAndOwesItsStatus(t *testing.T) { // N6/AC5, 1300/AC5, 1445/AC1, 1929/AC1
func TestCheckerJudgesACandidateAndOwesItsStatus(t *testing.T) { // N6/AC5, 1300/AC5, 1445/AC1
b := startBed(t)
key := b.candidate(t, "feat/x", pull(repo, 7, head1))
b.start(t, nil)
@@ -509,7 +476,7 @@ func TestCheckerJudgesACandidateAndOwesItsStatus(t *testing.T) { // N6/AC5, 1300
// TestACheckOfEqualInputsRunsNothing: a second candidate (another head, the same digests) is checked with every package
// taken from the first's kept verdicts: no test step runs and no second kept key appears.
func TestACheckOfEqualInputsRunsNothing(t *testing.T) { // 1287/AC5, 1288/AC6, 1929/AC2
func TestACheckOfEqualInputsRunsNothing(t *testing.T) { // 1287/AC5, 1288/AC6
b := startBed(t)
first := b.candidate(t, "feat/x", pull(repo, 7, head1))
b.start(t, nil)
@@ -607,7 +574,7 @@ func TestACheckLeftRunningIsTakenOverWhenTheCheckerStarts(t *testing.T) { // N6/
// TestAMemberNamingAMeshSDKCommitOffTheTrunkFailsNamingTheCommit: the go.mod the forge answers names a commit that main does not
// hold; the verdict fails naming the member and the commit and no test runs (ADR 0484 clause 9).
func TestAMemberNamingAMeshSDKCommitOffTheTrunkFailsNamingTheCommit(t *testing.T) { // 1624/AC2, 1929/AC3
func TestAMemberNamingAMeshSDKCommitOffTheTrunkFailsNamingTheCommit(t *testing.T) { // 1624/AC2
b := startBed(t)
b.stalePin.Store(true)
key := b.candidate(t, "feat/x", pull(repo, 7, head1))
@@ -616,7 +583,9 @@ func TestAMemberNamingAMeshSDKCommitOffTheTrunkFailsNamingTheCommit(t *testing.T
if v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "9aaa5c80ac5c") || !strings.Contains(v.Failed.Words, repo+"#7") {
t.Fatalf("verdict = %+v", v)
}
// 1929/AC3: no assertion that no test ran: one ask runs fetch, digest and tests (ADR 0525 §3).
if n := b.fake.count("mesh-build-go.build_go_test"); n != 0 {
t.Errorf("%d test steps ran after a stale pin", n)
}
}
// triesStore is the tries bucket as the tools write it.
@@ -702,7 +671,7 @@ func TestTheCheckerSaysOwedReadableOnTheBus(t *testing.T) { // R2-1
break
}
if said.Module != "mesh-delivery-checker" || said.Node != "n1" || !said.Keeps("owed") || !said.Readable("owed") ||
!said.Readable("previews") || !said.Readable("verdicts") || !said.Readable("kept") || said.Readable("tries") || len(said.States) != 5 {
t.Fatalf("the checker said %+v, want its five states with kept, owed, previews and verdicts readable (1929/AC4; 1887 A5b)", said)
!said.Readable("previews") || !said.Readable("verdicts") || said.Readable("kept") || said.Readable("tries") || len(said.States) != 5 {
t.Fatalf("the checker said %+v, want its five states with owed, previews and verdicts readable", said)
}
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go
+3 -3
View File
@@ -6,6 +6,7 @@ import (
"strings"
"sync"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
@@ -92,7 +93,7 @@ func (r *repoModules) Put(ctx context.Context, key string, value []byte, deleted
if mk, err := delivery.ManifestsKey(m.Manifest); err == nil {
if raw, _, found, err := r.manifests.Get(ctx, mk); err == nil && found {
if kept, err := delivery.DecodeManifest(raw); err == nil {
if man, err := parseManifest(kept.JSON); err == nil && man.Build != nil {
if man, err := catalogue.ParseManifest([]byte(kept.JSON)); err == nil && man.Build != nil {
for _, a := range man.Build.Artifacts {
if a.Language != "" {
set[a.Language] = true
@@ -182,8 +183,7 @@ func (r buildReader) Build(ctx context.Context, module, fp2 string) (delivery.Bu
return b, err == nil, err
}
// CommitOf is the commit the indexer's record of the module names: the commit its fingerprint first appeared at, so the source
// of the build the mesh runs (a later merge that leaves the module alone writes no record, the operator 2026-10-25).
// CommitOf is the commit the indexer's record of the module was computed at: the commit of its repository the mesh runs.
func (r *repoModules) CommitOf(module string) string {
r.mu.Lock()
defer r.mu.Unlock()
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go
-29
View File
@@ -3,7 +3,6 @@ package main
import (
"context"
"encoding/json"
"fmt"
"os"
"strings"
"testing"
@@ -113,31 +112,3 @@ func TestTheTypeScriptPackagesPathsComeFromTheIndexersModulesNotTheManifest(t *t
t.Errorf("another repository: %v", got)
}
}
// A manifest whose config value falls back to a predefined key (ADR 0348 §7) is read, not logged as unreadable for the
// catalogue's predefined.json this process never holds: the indexer and the assigner read it with
// catalogue.ParseManifestForInputs (issues 1751, 1906). xterm's module.json from mesh-catalog at 835bf22b.
func TestAManifestFallingBackToAPredefinedKeyIsRead(t *testing.T) {
raw, err := os.ReadFile("testdata/catalogue-835bf22b-xterm.module.json")
if err != nil {
t.Fatal(err)
}
manifest := string(raw)
data, err := delivery.EncodeManifest(delivery.Manifest{Module: "xterm", JSON: manifest})
if err != nil {
t.Fatal(err)
}
store := memManifests{delivery.ManifestDigest([]byte(manifest)): data}
var said []string
r := &repoModules{manifests: store, logf: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }}
putModule(t, r, "xterm", "novox/mesh-catalog", manifest)
r.BuildModules("novox/mesh-catalog")
for _, s := range said {
if strings.Contains(s, "cannot be read") {
t.Errorf("the manifest was not read: %s", s)
}
}
if _, err := parseManifest(manifest); err != nil {
t.Errorf("manifestReader: %v", err)
}
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go
-10
View File
@@ -257,14 +257,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// The toolchains first, so a check that starts from the first candidate finds the nodes.
watch := newToolchainWatch(ctx, js, tools, ch.NodesChanged, logger)
// Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's
// assignments are settled first, so the toolchains settle through the gate already knowing them.
gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf)
watch.put = gate.Toolchain
assignErrs, err := watchAssignments(ctx, js, gate, ch.NodesChanged, logger)
if err != nil {
return err
}
if err := watch.start(); err != nil {
return err
}
@@ -377,8 +369,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return nil
case err := <-errs:
return err
case err := <-assignErrs:
return err
case err := <-watch.errs:
return err
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing.go
-209
View File
@@ -1,209 +0,0 @@
package main
// This file is the same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker (issue 1940): it moves
// to mesh-sdk go/delivery/buildrun once mesh-sdk can be delivered again, since a change of mesh-sdk waits on the very
// toolchain agreement this file repairs.
import (
"context"
"encoding/json"
"fmt"
"log"
"maps"
"slices"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provider"
)
// A build module writes <node>.<language> of its toolchains state on every node it runs on, and nothing deletes the key
// when the module is unassigned from a node: the controller writes no module's state, and the module is gone. A report of
// a node the build module no longer runs on is therefore not a builder (issue 1940: mesh-build-node unassigned from ace
// on 2026-10-23 left ace.typescript at v26.12.0, and every Node fingerprint said "builders disagree").
//
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
// key <node>.<module>) hold an open assignment of its build module on its node, and only once it was written since that
// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report
// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node
// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran
// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
// refusal is the guard of novox/hq ADR 0465, and it stays.
type toolchainGate struct {
mu sync.Mutex
modules map[string]bool // the build modules whose toolchains pass the gate
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
logf func(format string, a ...any)
known bool // the assignments are settled
standing map[string]time.Time // <node>.<module> with an open assignment -> since when it stands
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
left map[string]string // <module>/<key> -> why it was said left out, so each reason is said once
}
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{},
raw: map[string]map[string][]byte{}, left: map[string]string{}}
for _, m := range modules {
g.modules[m] = true
}
return g
}
// Toolchain takes one change of a build module's toolchains bucket in and answers whether what it passed on changed.
func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool) bool {
g.mu.Lock()
defer g.mu.Unlock()
if g.raw[module] == nil {
g.raw[module] = map[string][]byte{}
}
if deleted {
delete(g.raw[module], key)
} else {
g.raw[module][key] = value
}
return g.pass(module, key)
}
// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers
// whether what it passed on changed.
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool {
node, module, ok := strings.Cut(key, ".")
if !ok || !g.modules[module] {
return false
}
assigned, open := openAssignment(value)
open = open && !deleted
g.mu.Lock()
defer g.mu.Unlock()
was, stood := g.standing[key]
since := assigned
if since.IsZero() {
// A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it.
since = at
if stood {
since = was
}
}
if open == stood && (!open || since.Equal(was)) {
return false
}
if open {
g.standing[key] = since
} else {
delete(g.standing, key)
}
if !g.known {
return false
}
changed := false
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
if n, _, err := delivery.ParseToolchainKey(k); err == nil && n == node {
changed = g.pass(module, k) || changed
}
}
return changed
}
// Settled says the assignments are read: from now on only the reports of assigned nodes pass. It answers whether what it
// passed on changed.
func (g *toolchainGate) Settled() bool {
g.mu.Lock()
defer g.mu.Unlock()
g.known = true
changed := false
for _, module := range slices.Sorted(maps.Keys(g.raw)) {
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
changed = g.pass(module, k) || changed
}
}
return changed
}
// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not
// known), deleted otherwise. The caller holds mu.
func (g *toolchainGate) pass(module, key string) bool {
value, held := g.raw[module][key]
if !held {
return g.forward(module, key, nil, true)
}
node, _, err := delivery.ParseToolchainKey(key)
why := ""
if err == nil && g.known {
why = g.leftOut(module, node, value)
}
if why == "" {
delete(g.left, module+"/"+key)
return g.forward(module, key, value, false)
}
if g.left[module+"/"+key] != why {
g.left[module+"/"+key] = why
g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why)
}
return g.forward(module, key, nil, true)
}
// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module,
// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse.
// The caller holds mu.
func (g *toolchainGate) leftOut(module, node string, value []byte) string {
since, open := g.standing[node+"."+module]
if !open {
return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node)
}
tc, err := delivery.DecodeToolchain(value)
if err == nil && tc.At.Before(since) {
return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again",
tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node)
}
return ""
}
// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not
// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when
// it was assigned, zero where the value does not say.
func openAssignment(value []byte) (time.Time, bool) {
var a struct {
Assigned time.Time `json:"assigned"`
Ended time.Time `json:"ended"`
State string `json:"state"`
}
if json.Unmarshal(value, &a) != nil {
return time.Time{}, false
}
return a.Assigned, a.Ended.IsZero() && a.State == ""
}
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
// told whenever what the gate passes on changed. Where the bucket cannot be read (not there, or this module may not read
// it) it says so and the gate passes every report on, as before; an error of the watch after that ends on the channel.
func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate, changed func(context.Context), logger *log.Logger) (<-chan error, error) {
errs := make(chan error, 1)
kv, err := js.KeyValue(ctx, provider.AssignmentsBucket)
if err != nil {
// Not fatal: an indexer, builder or checker that stops for it stops every delivery (a grant comes with a delivery).
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
return errs, nil
}
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
w, err := settle(ctx, kv, each)
if err != nil {
return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err)
}
if gate.Settled() {
changed(ctx)
}
go func() {
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) {
changed(ctx)
}
})
}()
return errs, nil
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing_test.go
-165
View File
@@ -1,165 +0,0 @@
package main
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
import (
"fmt"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
func gateReport(t *testing.T, node, version string) (string, []byte) {
t.Helper()
key, err := delivery.ToolchainKey(node, "typescript")
if err != nil {
t.Fatal(err)
}
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: time.Unix(0, 0).UTC()})
if err != nil {
t.Fatal(err)
}
return key, value
}
func agreedOf(set *buildrun.ToolchainSet) (string, string) {
tc, none := delivery.AgreedToolchain(set.Entries("typescript"))
return tc.Version, none
}
// The live case of issue 1940: mesh-build-node was unassigned from ace, whose report stayed at v26.12.0 while the
// three nodes it runs on report v26.12.1. Without the assignments every report counts (and they disagree); with them
// ace's is left out and the three agree; assigned to ace again, its report counts again and the guard refuses again.
func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
set := &buildrun.ToolchainSet{}
var said []string
gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, f) })
for node, version := range map[string]string{"ace": "v26.12.0", "g14": "v26.12.1", "novox": "v26.12.1", "shanks": "v26.12.1"} {
key, value := gateReport(t, node, version)
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
}
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree") {
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
}
for _, node := range []string{"g14", "novox", "shanks"} {
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{})
}
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{})
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter
if !gate.Settled() {
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
}
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
t.Fatalf("the assigned nodes agree on v26.12.1; got %q, %q", v, none)
}
if len(said) != 1 {
t.Fatalf("ace's report is said left out once; said %v", said)
}
// ace reports again while unassigned (an old value seen again): still left out.
key, value := gateReport(t, "ace", "v26.12.0")
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
t.Fatal("an unassigned node's report changed what was passed on")
}
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) {
t.Fatal("assigning ace again passed nothing on")
}
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
t.Fatalf("builders that run different versions still disagree; got %q", none)
}
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) {
t.Fatal("a pending assignment is no assignment on the machine")
}
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
t.Fatalf("got %q, %q", v, none)
}
gate.Assignment("ace.mesh-build-node", nil, true, time.Time{})
if v, _ := agreedOf(set); v != "v26.12.1" {
t.Fatalf("got %q", v)
}
// A report deleted from the bucket goes whether its node is assigned or not.
key, _ = gateReport(t, "g14", "v26.12.1")
if !gate.Toolchain(buildrun.NodeBuildModule, key, nil, true) {
t.Fatal("deleting g14's report passed nothing on")
}
if n := len(set.Entries("typescript")); n != 2 {
t.Fatalf("novox and shanks are left; got %d entries", n)
}
}
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript
// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node
// writes it again. An assignment whose value names no time stands since the revision that opened it.
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again
report := func(t *testing.T, node, version string, at time.Time) (string, []byte) {
key, _ := gateReport(t, node, version)
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
if err != nil {
t.Fatal(err)
}
return key, value
}
for _, c := range []struct {
name string
assignment string // ace's assignment value
revision time.Time // when the bus wrote it
at time.Time // when ace wrote its report
counts bool
}{
{"no report yet, then one", `{}`, time.Time{}, time.Now(), true},
{"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false},
{"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false},
{"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true},
} {
t.Run(c.name, func(t *testing.T) {
set := &buildrun.ToolchainSet{}
var said []string
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
for _, node := range []string{"g14", "novox", "shanks"} {
gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{})
key, value := gateReport(t, node, "v26.12.1")
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
}
gate.Settled()
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) {
t.Fatal("an assignment with no report passed something on")
}
key, value := report(t, "ace", "v26.12.0", c.at)
if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts {
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
}
// A rewrite of the assignment that does not move it changes nothing, and nothing is said again.
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) {
t.Fatal("the same assignment written again changed what was passed on")
}
v, none := agreedOf(set)
if c.counts {
if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none)
}
return
}
if v != "v26.12.1" || none != "" {
t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none)
}
if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") {
t.Fatalf("said %q, want once why ace's report is left out", said)
}
// ace writes its report again since it was assigned: it counts.
key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute))
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
t.Fatal("ace's new report was not passed on")
}
if n := len(set.Entries("typescript")); n != 4 {
t.Fatalf("four builders report; got %d", n)
}
})
}
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/testdata/catalogue-835bf22b-xterm.module.json
Vendored
-236
View File
@@ -1,236 +0,0 @@
{
"module": "xterm",
"version": "1",
"moved": true,
"capabilities": [
"package-manager"
],
"requires": [
"x11-display"
],
"claims": [
{
"name": "node-terminal-emulator",
"scope": "node",
"serves": [
"open"
]
}
],
"tools": [
"xterm_font",
"xterm_colours"
],
"config": {
"face": {
"type": "string",
"description": "The font family xterm draws with, as fc-list names it: JetBrainsMono Nerd Font, because research 026/04 chose it for its glyphs the prompt uses. A family the machine does not have falls back to the nearest installed face (xterm_font says which).",
"pattern": "[A-Za-z0-9 ._-]{1,64}",
"default": "JetBrainsMono Nerd Font"
},
"size": {
"type": "number",
"description": "The size of xterm's text, in points: 12 reads well on a screen of about 100 DPI, and a denser screen needs a larger size.",
"minimum": 6,
"maximum": 72,
"predefined": "font-size",
"default": 12
},
"save-lines": {
"type": "number",
"description": "How many lines of scrollback xterm keeps per terminal: 10000 holds a long build log, and each line costs memory in every open terminal.",
"minimum": 0,
"maximum": 1000000,
"default": 10000
},
"term-name": {
"type": "string",
"description": "The TERM value xterm sets for the programs it runs: xterm-256color, because the palette has 256 colours and every program knows the name.",
"pattern": "[A-Za-z0-9._-]{1,64}",
"default": "xterm-256color"
},
"render-font": {
"type": "boolean",
"description": "Whether xterm draws with fontconfig faces (true) or the old X core fonts (false): true, because the face is a fontconfig family and core fonts cannot show it.",
"default": true
},
"bold-mode": {
"type": "boolean",
"description": "Whether xterm draws bold text by overstriking the normal face (true): false, because the face has a real bold style and overstriking blurs it.",
"default": false
},
"always-bold-mode": {
"type": "boolean",
"description": "Whether xterm overstrikes bold even where the face has a bold style (true): false, for the same reason as bold-mode.",
"default": false
},
"allow-bold-fonts": {
"type": "boolean",
"description": "Whether xterm may use the face's bold style for bold text (true): true, so bold text is drawn with the real bold weight.",
"default": true
},
"background": {
"type": "string",
"description": "The terminal's background colour, as #rrggbb: black, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#000000"
},
"foreground": {
"type": "string",
"description": "The terminal's text colour, as #rrggbb: a light grey, easier on the eyes than white on black.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#a8a8a8"
},
"color0": {
"type": "string",
"description": "Palette entry 0, the normal black, as #rrggbb: #000000, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#000000"
},
"color1": {
"type": "string",
"description": "Palette entry 1, the normal red, as #rrggbb: #9e1828, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#9e1828"
},
"color2": {
"type": "string",
"description": "Palette entry 2, the normal green, as #rrggbb: #008800, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#008800"
},
"color3": {
"type": "string",
"description": "Palette entry 3, the normal yellow, as #rrggbb: #d2bb4b, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#d2bb4b"
},
"color4": {
"type": "string",
"description": "Palette entry 4, the normal blue, as #rrggbb: #414171, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#414171"
},
"color5": {
"type": "string",
"description": "Palette entry 5, the normal magenta, as #rrggbb: #963c59, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#963c59"
},
"color6": {
"type": "string",
"description": "Palette entry 6, the normal cyan, as #rrggbb: #418179, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#418179"
},
"color7": {
"type": "string",
"description": "Palette entry 7, the normal white, as #rrggbb: #bebebe, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#bebebe"
},
"color8": {
"type": "string",
"description": "Palette entry 8, the bright black, as #rrggbb: #666666, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#666666"
},
"color9": {
"type": "string",
"description": "Palette entry 9, the bright red, as #rrggbb: #bc5766, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#bc5766"
},
"color10": {
"type": "string",
"description": "Palette entry 10, the bright green, as #rrggbb: #61a171, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#61a171"
},
"color11": {
"type": "string",
"description": "Palette entry 11, the bright yellow, as #rrggbb: #e7db52, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#e7db52"
},
"color12": {
"type": "string",
"description": "Palette entry 12, the bright blue, as #rrggbb: #5085af, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#5085af"
},
"color13": {
"type": "string",
"description": "Palette entry 13, the bright magenta, as #rrggbb: #a97a99, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#a97a99"
},
"color14": {
"type": "string",
"description": "Palette entry 14, the bright cyan, as #rrggbb: #6ba4a4, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#6ba4a4"
},
"color15": {
"type": "string",
"description": "Palette entry 15, the bright white, as #rrggbb: #ffffff, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#ffffff"
},
"copy-binding": {
"type": "string",
"description": "The key combination that copies the selection to the clipboard, in X translation syntax: Ctrl Shift ~Alt <Key> C, so a plain Ctrl+C still interrupts the program. It holds no colon, backslash or line break, which would end the resource.",
"pattern": "[^:\\\\\\n]+",
"default": "Ctrl Shift ~Alt <Key> C"
},
"paste-binding": {
"type": "string",
"description": "The key combination that pastes the clipboard, in X translation syntax: Ctrl Shift ~Alt <Key> V, the pair of copy-binding. It holds no colon, backslash or line break, which would end the resource.",
"pattern": "[^:\\\\\\n]+",
"default": "Ctrl Shift ~Alt <Key> V"
}
},
"items": [
{
"collection": "environment",
"name": "terminal",
"attributes": {
"kind": "variable",
"name": "TERMINAL"
},
"template": "files/env-terminal"
},
{
"collection": "shell:xresources:normal",
"name": "xresources-normal",
"template": "files/xresources-normal"
}
],
"resources": [
{
"id": "package",
"type": "package",
"package": "xterm"
}
],
"build": {
"artifacts": [
{
"name": "tools",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/xterm-tools",
"binary": "xterm-tools",
"loads": [
"xterm-tools"
]
},
{
"name": "files",
"kind": "archive",
"from": "files"
}
]
}
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/toolchains.go
+6 -8
View File
@@ -24,11 +24,9 @@ var toolchainStreams = map[string]string{buildrun.GoBuildModule: "TOOLCHAINS_GO"
// §8): it watches the buckets that exist at start, and a build module says each time it wrote its toolchains key
// (toolchain.reported, handled by Reported); a bucket not watched yet is then watched.
type toolchainWatch struct {
ctx context.Context
js bucketOpener
set *buildrun.ToolchainSet
// put takes a change of a toolchains bucket in: set.Put, or the gate of the assigned build nodes before it (issue 1940).
put func(module, key string, value []byte, deleted bool) bool
ctx context.Context
js bucketOpener
set *buildrun.ToolchainSet
changed func(ctx context.Context) // a language's entries changed: the failed builds are tried again
logger *log.Logger
errs chan error
@@ -37,7 +35,7 @@ type toolchainWatch struct {
}
func newToolchainWatch(ctx context.Context, js bucketOpener, set *buildrun.ToolchainSet, changed func(context.Context), logger *log.Logger) *toolchainWatch {
return &toolchainWatch{ctx: ctx, js: js, set: set, put: set.Put, changed: changed, logger: logger, errs: make(chan error, len(buildrun.LanguageOf)), watched: map[string]bool{}}
return &toolchainWatch{ctx: ctx, js: js, set: set, changed: changed, logger: logger, errs: make(chan error, len(buildrun.LanguageOf)), watched: map[string]bool{}}
}
// start watches the buckets that exist now; a missing one is said and waits for its build module's event.
@@ -65,14 +63,14 @@ func (w *toolchainWatch) ensure(module string) error {
case err != nil:
return fmt.Errorf("the bucket %s: %w", name, err)
}
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.put(module, e.Key(), e.Value(), !opOf(e)) })
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.set.Put(module, e.Key(), e.Value(), !opOf(e)) })
if err != nil {
return err
}
w.watched[module] = true
go func() {
w.errs <- follow(w.ctx, watcher, func(e jetstream.KeyValueEntry) {
if w.put(module, e.Key(), e.Value(), !opOf(e)) {
if w.set.Put(module, e.Key(), e.Value(), !opOf(e)) {
w.changed(w.ctx)
}
})
modules/mesh-delivery-checker/go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-checker
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
modules/mesh-delivery-checker/go.sum
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93 h1:FAiFY2KBKEoZSH49OZTt7/zRBIQxyl5g/KhYrpggbN4=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
modules/mesh-delivery-checker/internal/check/beside.go
+1 -5
View File
@@ -26,7 +26,7 @@ const (
// checks/deps). Which commit of each is besideCommit's rule.
var besideRules = map[string][]string{
controllerRepo: {catalogueRepository, nodeEngineRepo, SDKRepository},
nodeEngineRepo: {catalogueRepository, SDKRepository, controllerRepo},
nodeEngineRepo: {catalogueRepository, SDKRepository},
SDKRepository: {catalogueRepository, nodeEngineRepo},
catalogueRepository: {SDKRepository, controllerRepo},
"novox/mesh-delivery": {SDKRepository, catalogueRepository},
@@ -83,10 +83,6 @@ func (j *Examiner) besideFor(ctx context.Context, c *Case, p Pull) ([]BesideRef,
out = append(out, BesideRef{Repository: repo, Commit: member.Head, Ref: refOf(member.Number)})
} else if head := heads[repo]; head != "" {
out = append(out, BesideRef{Repository: repo, Commit: head})
} else if repo == controllerRepo && p.Repository == nodeEngineRepo {
// The node-engine's step-nine test judges the controller's own manifest and fails without it (issue 1295): a
// checker that cannot place it says so, where placing none left that test to skip.
return nil, fmt.Errorf("the trunk head of %s is not known, so the controller the tests of %s#%d read cannot be placed beside them", repo, p.Repository, p.Number)
}
}
}
modules/mesh-delivery-checker/internal/check/case.go
-20
View File
@@ -3,7 +3,6 @@ package check
import (
"fmt"
"maps"
"slices"
"sort"
"strings"
"sync"
@@ -41,9 +40,6 @@ type Case struct {
Queued time.Time
// Try narrows a try's check (its level, its tests); nil for a pull request's.
Try *TryOptions
notesMu sync.Mutex
notes []string
// Progress says what the check is doing, so a check that ran past its bound names the step it was stuck at.
Progress *Progress
// Kind is what asked the check (KindNew, KindNodes, KindTrunk): a re-judge after a trunk move is asked on the build
@@ -152,19 +148,3 @@ func (t *Trunks) Heads() map[string]string {
defer t.mu.Unlock()
return maps.Clone(t.heads)
}
// note puts a word of the check's way on the case, for a try's line (issue 1921).
func (c *Case) note(s string) {
c.notesMu.Lock()
defer c.notesMu.Unlock()
if !slices.Contains(c.notes, s) {
c.notes = append(c.notes, s)
}
}
// Notes are the words noted on the case, in the order they were noted.
func (c *Case) Notes() []string {
c.notesMu.Lock()
defer c.notesMu.Unlock()
return slices.Clone(c.notes)
}
modules/mesh-delivery-checker/internal/check/digestform_test.go
-83
View File
@@ -1,83 +0,0 @@
package check
import (
"encoding/json"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// 1930/AC3: given a check ask whose stored end keeps the digest's packages as a list of {package, digest} (issue 1930),
// or one stored before it with the packages a map keyed by package path, when the same heads are checked a second time,
// then the kept verdicts the first check wrote are read from it: every package it passed is kept, none run again.
func TestKeptVerdictsAreReadFromTheDigestsListForm(t *testing.T) {
for _, tc := range []struct {
name string
old bool
}{{"the list form", false}, {"a map stored before issue 1930", true}} {
t.Run(tc.name, func(t *testing.T) {
b := newBench(t)
var stored []string
b.asks.Answer = func(a delivery.StepAsk) delivery.Step {
s := b.answer(a)
if tc.old && s.State == delivery.StepPassed {
s.Result = oldDigestForm(t, s.Result)
}
stored = append(stored, string(s.Result))
return s
}
first := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
u := first.Levels[delivery.LevelUnit]
if first.State != delivery.VerdictPass || u.Run == 0 || u.Kept != 0 {
t.Fatalf("the first check = %+v", first)
}
if len(stored) == 0 {
t.Fatal("no ask's end was stored")
}
list := strings.Contains(stored[0], `"packages":[{"package":"fixture/slow"`)
if list == tc.old || strings.Contains(stored[0], `"fixture/slow":"`) != tc.old {
t.Fatalf("the stored end is not in the form under test: %s", stored[0])
}
b.steps.calls = nil
second := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
if s := second.Levels[delivery.LevelUnit]; second.State != delivery.VerdictPass || s.Kept != u.Run || s.Run != 0 {
t.Fatalf("the second check's unit level = %+v, want %d kept and none run", s, u.Run)
}
if n := len(b.steps.tool("test")); n != 0 {
t.Errorf("%d test steps ran on the second check", n)
}
})
}
}
// oldDigestForm rewrites a check answer's level digests to the form stored before issue 1930: packages as a map keyed by
// package path.
func oldDigestForm(t *testing.T, result json.RawMessage) json.RawMessage {
t.Helper()
var a struct {
Fetch json.RawMessage `json:"fetch"`
Levels []map[string]any `json:"levels"`
}
if err := json.Unmarshal(result, &a); err != nil {
t.Fatal(err)
}
for _, l := range a.Levels {
d, ok := l["digest"].(map[string]any)
if !ok {
continue
}
m := map[string]any{}
list, _ := d["packages"].([]any)
for _, p := range list {
pd := p.(map[string]any)
m[pd["package"].(string)] = pd["digest"]
}
d["packages"] = m
}
raw, err := json.Marshal(a)
if err != nil {
t.Fatal(err)
}
return raw
}
modules/mesh-delivery-checker/internal/check/failure_words_test.go
-17
View File
@@ -1,7 +1,6 @@
package check
import (
"encoding/json"
"strings"
"testing"
@@ -55,19 +54,3 @@ func TestWordsLongerThanTheBoundKeepTheirLastLines(t *testing.T) { // 1901/AC1
}
_ = delivery.VerdictFail
}
// An older build module sends no "cause": its answer is still accepted and the verdict's words are its output, as before.
func TestAnOlderBuildModulesFailureIsAcceptedWithItsWholeOutput(t *testing.T) { // 1901/AC3
old := `{"package":"p","test":"TestX","output":"x_test.go:1: logged\nx_test.go:2: the old kept lines"}`
var f Failure
if err := json.Unmarshal([]byte(old), &f); err != nil || f.Cause != "" {
t.Fatalf("1901/AC3: err = %v, cause = %q", err, f.Cause)
}
v := levelFailure(&target{}, "unit", Tested{Failed: []Failure{f}})
if v.Failed.Words != "x_test.go:1: logged\nx_test.go:2: the old kept lines" {
t.Errorf("1901/AC3: words = %q", v.Failed.Words)
}
if got := firstFailed(*v); !strings.Contains(got, "logged") || strings.Contains(got, "\n") {
t.Errorf("1901/AC3: status = %q", got)
}
}
modules/mesh-delivery-checker/internal/check/judge.go
+16 -89
View File
@@ -447,11 +447,13 @@ func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer)
return fail(t.node, fmt.Sprintf("the check of %s#%d answered the level %q, which is no level", t.pull.Repository, t.pull.Number, level)), 0
}
var digests map[string]string
if t.module == buildrun.GoBuildModule && la.Digest != nil {
// The packages come as a list, never keyed by a package path (issue 1930); an answer stored before reads too.
digests = la.Digest.Map()
if t.runner != "" {
t.judgedBy(la.Digest.JudgedBy) // the installed module judged it: the runner is not used again
if t.module == buildrun.GoBuildModule && len(la.Digest) > 0 {
var d Digested
if err := json.Unmarshal(la.Digest, &d); err == nil {
digests = d.Packages
if t.runner != "" {
t.judgedBy(d.JudgedBy) // the installed module judged it: the runner is not used again
}
}
}
if t.runner != "" && la.Judged != "" {
@@ -531,98 +533,23 @@ const MaxWords = 4096
// failureWords is a failure's words: its output's last lines up to MaxWords, with the line the build module told apart as
// the test's error first when the output does not start with it, so the first line of the words is the cause and the
// status line, which is cut short, names it. **A failure's words are never "PASS" or "ok"** (issue 1901, its third
// meeting: a package that failed after its tests passed was worded "PASS"): go test's summary lines are dropped, and with
// no cause told apart (an older build module) the cause is the output's first "panic:", "fatal error:" or "signal:"
// line, read before the output is cut, or the words say the tests passed and the program failed after them.
// status line, which is cut short, names it.
func failureWords(f Failure) string {
out, passed := dropSummary(strings.TrimSpace(f.Output))
cause := strings.TrimSpace(f.Cause)
if cause == "" {
cause = causeIn(out)
}
switch {
case cause == "" && out == "" && passed:
cause = "the tests passed, then the test program exited with a failure: an os.Exit or a panic in TestMain or after the tests"
case cause == "" && out == "":
// The build module on main sends a package that stopped (a timeout, an exit, a kill) as its "FAIL" line alone.
cause = "the package failed with no test to name and no panic, signal or build error in its output (an older build module sent no cause)"
}
if len(cause) > MaxCause {
cause = strings.ToValidUTF8(cause[:MaxCause], "") + "..."
}
// The words keep MaxWords exactly: the cause and its newline, the marker and the kept lines (review round 1).
budget := MaxWords
if cause != "" {
budget -= len(cause) + 1
}
if len(out) > budget {
const marker = "... "
tail := out[len(out)-(budget-len(marker)):]
out := strings.TrimSpace(f.Output)
if len(out) > MaxWords {
out = out[len(out)-MaxWords:]
// Start on a whole line, and a whole character.
if i := strings.IndexByte(tail, '\n'); i >= 0 {
tail = strings.TrimLeft(tail[i:], "\n")
if i := strings.IndexByte(out, '\n'); i >= 0 {
out = "... " + strings.TrimLeft(out[i:], "\n")
}
out = marker + strings.ToValidUTF8(tail, "")
out = strings.ToValidUTF8(out, "")
}
if cause != "" && !strings.HasPrefix(firstLine(out), cause) {
out = strings.TrimRight(cause+"\n"+out, "\n")
if c := strings.TrimSpace(f.Cause); c != "" && !strings.HasPrefix(firstLine(out), c) {
out = c + "\n" + out
}
return out
}
// MaxCause is the most a cause in a failure's words holds: one long line (the 1901 review: a cause of a kilobyte could
// push the words past MaxWords).
const MaxCause = 1024
// dropSummary drops go test's summary lines from a failure's output ("PASS", "FAIL", "ok <pkg>", "FAIL <pkg>"), and says
// whether a "PASS" was among them: the package's tests passed before it failed.
func dropSummary(out string) (string, bool) {
passed := false
lines := strings.Split(out, "\n")
kept := lines[:0]
for _, l := range lines {
// go's exact forms only (review round 1): "PASS" or "FAIL" alone, "ok \t<pkg>\t<time>", "FAIL\t<pkg>\t<time>".
switch t := strings.TrimRight(l, "\r"); {
case t == "PASS":
passed = true
case t == "FAIL", strings.HasPrefix(t, "ok \t"), strings.HasPrefix(t, "FAIL\t"):
default:
kept = append(kept, l)
}
}
return strings.TrimSpace(strings.Join(kept, "\n")), passed
}
// causeIn is a race the race detector reported, as its heading, its access and the frame that made it ("WARNING: DATA
// RACE: Write at 0x... by goroutine 9 in pkg.F.func1()"); else the first "panic:", "fatal error:" or "signal:" line of a
// failure's output, in that order; or "". **Only lines at column 0**, where go prints the runtime's: a test's indented log
// line holding the same words is not the cause (review round 1).
func causeIn(out string) string {
lines := strings.Split(out, "\n")
for i, l := range lines {
if strings.TrimRight(l, "\r") != "WARNING: DATA RACE" {
continue
}
words := "WARNING: DATA RACE"
if i+1 < len(lines) && strings.TrimSpace(lines[i+1]) != "" {
words += ": " + strings.TrimSuffix(strings.TrimSpace(lines[i+1]), ":")
if i+2 < len(lines) && strings.HasPrefix(lines[i+2], " ") {
words += " in " + strings.TrimSpace(lines[i+2])
}
}
return words
}
for _, prefix := range []string{"panic: ", "fatal error: ", "signal: "} {
for _, l := range lines {
if l = strings.TrimRight(l, "\r"); strings.HasPrefix(l, prefix) {
return l
}
}
}
return ""
}
// firstLine is the first non-empty line of a failure's words.
func firstLine(s string) string {
for _, l := range strings.Split(s, "\n") {
modules/mesh-delivery-checker/internal/check/package_cause_test.go
-92
View File
@@ -1,92 +0,0 @@
package check
import (
"strings"
"testing"
)
// What the build module before this fix sent for two packages that failed after their tests passed (issue 1901, its third
// meeting: a try of mesh-controller kept "PASS" as its failure's words). Each is that build module's Output for the real
// `go test -json` of a throwaway package (mesh-catalog modules/mesh-build-go testdata/captured/go-test-json-pkg-*.jsonl):
// TestMain panicking in a goroutine after m.Run, TestMain exiting 2 after m.Run, and TestMain racing after m.Run.
const (
oldPassThenPanic = "PASS\npanic: the bus closed after the tests: nats: connection closed\ngoroutine 20 [running]:\n" +
"example.com/passthenfail.TestMain.func1()\n\t/tmp/fix-1901b/passthenfail/p_test.go:13 +0x25\n" +
"created by example.com/passthenfail.TestMain in goroutine 1\n\t/tmp/fix-1901b/passthenfail/p_test.go:13 +0x25\n" +
"FAIL\texample.com/passthenfail\t0.005s"
oldPassThenExit = "PASS\nFAIL\texample.com/mainexit\t0.004s"
// TestMain racing after m.Run passed, under go test -race (go-test-json-pkg-racemain.jsonl).
oldPassThenRace = "PASS\n==================\nWARNING: DATA RACE\nWrite at 0x000000836428 by goroutine 9:\n example.com/racemain.TestMain.func1()\n /tmp/fix-1901b/racemain/p_test.go:15 +0x24\nPrevious write at 0x000000836428 by main goroutine:\n example.com/racemain.TestMain()\n /tmp/fix-1901b/racemain/p_test.go:16 +0x44\n main.main()\n _testmain.go:48 +0x171\nGoroutine 9 (running) created at:\n example.com/racemain.TestMain()\n /tmp/fix-1901b/racemain/p_test.go:15 +0x37\n main.main()\n _testmain.go:48 +0x171\n==================\nFound 1 data race(s)\nFAIL\texample.com/racemain\t1.031s"
)
// A failure with no cause told apart never has "PASS" or "ok" as its words: the checker takes the panic line from the
// output, or says that the tests passed and the program failed after them (1901/AC5).
func TestAPackageFailureIsNeverWordedPass(t *testing.T) { // 1901/AC5
cases := []struct {
name, output, want string
}{
{"panic after the tests", oldPassThenPanic, "panic: the bus closed after the tests: nats: connection closed"},
{"exit after the tests", oldPassThenExit, "the tests passed, then the test program exited with a failure"},
{"race after the tests", oldPassThenRace, "WARNING: DATA RACE: Write at 0x000000836428 by goroutine 9 in example.com/racemain.TestMain.func1()"},
{"a panic deep in a long output", "PASS\npanic: the seat row vanished\n" + strings.Repeat("goroutine 7 [chan receive]:\n\tsome/frame.go:1 +0x1\n", 300) + "FAIL\tp\t2.0s", "panic: the seat row vanished"},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
v := levelFailure(&target{}, "integration", Tested{Failed: []Failure{{Package: "github.com/novox/mesh-controller/cmd/mesh-controller", Output: c.output}}})
for _, l := range strings.Split(v.Failed.Words, "\n") {
if l = strings.TrimSpace(l); l == "PASS" || strings.HasPrefix(l, "ok ") || strings.HasPrefix(l, "ok\t") {
t.Errorf("1901/AC5: the words keep %q: %q", l, v.Failed.Words)
}
}
if !strings.HasPrefix(v.Failed.Words, c.want) {
t.Errorf("1901/AC5: words start %q, want %q", firstLine(v.Failed.Words), c.want)
}
if got := firstFailed(*v); !strings.Contains(got, c.want) || strings.HasSuffix(got, ": PASS") {
t.Errorf("1901/AC5: status = %q", got)
}
})
}
}
// A cause the build module told apart is cut to a line's bound, so the words keep their bound (the 1901 review's note).
func TestALongCauseIsCutToALinesBound(t *testing.T) { // 1901/AC5
f := Failure{Package: "p", Test: "TestX", Cause: strings.Repeat("c", 5000), Output: "x_test.go:1: something"}
v := levelFailure(&target{}, "unit", Tested{Failed: []Failure{f}})
if len(v.Failed.Words) > MaxWords+200 {
t.Errorf("1901/AC5: words are %d bytes, bound %d", len(v.Failed.Words), MaxWords)
}
}
// The build module on main sends a package that stopped (a timeout, an exit, a kill) as only go test's last line, with no
// cause: the words then say so, never nothing (review round 1). Each output is what that build module sends for the
// capture of the same name (go-test-json-pkg-timeout, -exiting, -killed): the package's own output, its "FAIL" line.
func TestAPackageStoppedWithOnlyItsFailLineIsStillWorded(t *testing.T) { // 1901/AC5
for _, out := range []string{"FAIL\texample.com/timeout\t2.017s", "FAIL\texample.com/exiting\t0.006s", "FAIL\texample.com/killed\t0.006s"} {
v := levelFailure(&target{}, "integration", Tested{Failed: []Failure{{Package: "example.com/x", Output: out}}})
if !strings.HasPrefix(v.Failed.Words, "the package failed with no test to name and no panic, signal or build error in its output") {
t.Errorf("1901/AC5: %q is worded %q", out, v.Failed.Words)
}
}
}
// A line of the test's own that starts as go's summary does, or that holds a runtime's words indented, is not taken for
// either (review round 1): go prints its summary in exact forms and the runtime's lines at column 0.
func TestOnlyGosExactFormsAreSummaryOrCause(t *testing.T) { // 1901/AC5
out := " x_test.go:3: ok so far\nok then: the seat is free\n panic: child retried and recovered\npanic: the real one\nFAIL\tp\t1.0s"
v := levelFailure(&target{}, "unit", Tested{Failed: []Failure{{Package: "p", Output: out}}})
if !strings.HasPrefix(v.Failed.Words, "panic: the real one") || !strings.Contains(v.Failed.Words, "ok then: the seat is free") {
t.Errorf("1901/AC5: words = %q", v.Failed.Words)
}
}
// The words keep MaxWords exactly, counting the cause, the marker and the newlines (review round 1).
func TestTheWordsKeepTheirBoundExactly(t *testing.T) { // 1901/AC5
for _, cause := range []string{"", "a cause", strings.Repeat("c", 3000)} {
for _, n := range []int{100, 170, 171, 172, 500} {
f := Failure{Package: "p", Test: "TestX", Cause: cause, Output: strings.Repeat("a log line of twenty\n", n) + "the end"}
if w := failureWords(f); len(w) > MaxWords {
t.Errorf("1901/AC5: cause %d bytes, %d lines: words are %d bytes, bound %d", len(cause), n, len(w), MaxWords)
}
}
}
}
modules/mesh-delivery-checker/internal/check/pins.go
-44
View File
@@ -6,7 +6,6 @@ import (
"fmt"
"path"
"regexp"
"sort"
"strings"
)
@@ -37,8 +36,6 @@ type Forge interface {
OnTrunk(ctx context.Context, repository, commit string) (bool, error)
// FullSHA is the 40-digit sha of a commit named by its first 12 digits.
FullSHA(ctx context.Context, repository, commit string) (string, error)
// BranchHeads is the commit every branch of a repository points at, by branch name (issue 1921).
BranchHeads(ctx context.Context, repository string) (map[string]string, error)
}
var pseudoVersion = regexp.MustCompile(`-([0-9a-f]{12})$`)
@@ -96,18 +93,6 @@ func (st *state) checkPinsOf(ctx context.Context, t *target) verdictPtr {
if onTrunk {
continue
}
if st.c.Try != nil {
// A try only (issue 1921): a repository tried on a branch that mesh-sdk shares by name (a milestone's branch across
// repositories) is built with that branch's mesh-sdk head, which no group pull request holds before the pull requests exist.
branch, err := st.sameBranchPin(ctx, t, pin)
if err != nil {
return failed(forgeMachine(err, t.node), fmt.Sprintf("%s#%d: whether mesh-sdk %s is the head of a branch cannot be told: %v%s", t.pull.Repository, t.pull.Number, pin, err, unheardWords(err)))
}
if branch != "" {
st.c.note(fmt.Sprintf("pin accepted: head of mesh-sdk %s", branch))
continue
}
}
nor := "the group has no mesh-sdk pull request"
if sdk != nil {
nor = "the head of the group's mesh-sdk pull request is " + short12(sdk.Head)
@@ -118,35 +103,6 @@ func (st *state) checkPinsOf(ctx context.Context, t *target) verdictPtr {
return nil
}
// sameBranchPin names the branch of the tried repository that points at the tried head and that mesh-sdk also has, at the
// commit the pin names; empty when there is none.
func (st *state) sameBranchPin(ctx context.Context, t *target, pin string) (string, error) {
mine, err := st.j.Forge.BranchHeads(ctx, t.pull.Repository)
if err != nil {
return "", err
}
var same []string
for name, head := range mine {
if head == t.pull.Head && name != "main" {
same = append(same, name)
}
}
if len(same) == 0 {
return "", nil
}
sdk, err := st.j.Forge.BranchHeads(ctx, SDKRepository)
if err != nil {
return "", err
}
sort.Strings(same)
for _, name := range same {
if h, ok := sdk[name]; ok && strings.HasPrefix(h, pin) {
return name, nil
}
}
return "", nil
}
// failureVerdict is a verdict of failure the check ends with.
// pinHint is added to the failure of a Go member's fetch that names mesh-sdk while the group holds a mesh-sdk pull request:
modules/mesh-delivery-checker/internal/check/pins_test.go
-95
View File
@@ -18,15 +18,6 @@ type fakeForge struct {
onTrunk map[string]bool // by commit prefix
err error
errAt map[string]error // by ref: a read at that ref fails with it
// branches are the branches of a repository with the commit each points at.
branches map[string]map[string]string
}
func (f *fakeForge) BranchHeads(_ context.Context, repository string) (map[string]string, error) {
if f.err != nil {
return nil, f.err
}
return f.branches[repository], nil
}
func (f *fakeForge) File(_ context.Context, repository, path, ref string) (string, error) {
@@ -253,39 +244,6 @@ func TestTheSiblingsAreWhereTheTestsStateThem(t *testing.T) { // review D2, ADR
}
}
// 1295/R4g: the node-engine's tests read the controller's manifest, so a mesh-host pull request has mesh-controller beside it:
// at the group member's head when the group holds one, else at its trunk.
func hostCase(t *testing.T, pulls ...delivery.CandidatePull) *Case {
t.Helper()
c, err := delivery.DecodeCandidate(candidateOf(t, "feat/x", pulls...))
if err != nil {
t.Fatal(err)
}
cs := CaseOf(c, map[string]string{"novox/mesh-host": trunkA, controllerRepo: trunkA2})
if cs == nil {
t.Fatal("no case")
}
return cs
}
func TestTheNodeEnginesTestsHaveTheControllerBesideThem(t *testing.T) {
const host = "novox/mesh-host"
b := newBench(t)
b.ex.Trunks = trunksOf(t, map[string]string{host: trunkA, controllerRepo: trunkA2, catalogueRepository: trunkS, SDKRepository: trunkS})
b.judge(t, hostCase(t, pullOf(host, 5, head1)))
if c := besideOf(t, b, host)[controllerRepo]; c.Commit != trunkA2 || c.Ref != "" {
t.Errorf("the controller beside the node-engine alone = %+v, want its trunk %s", c, trunkA2)
}
b = newBench(t)
b.ex.Trunks = trunksOf(t, map[string]string{host: trunkA, controllerRepo: trunkA2, catalogueRepository: trunkS, SDKRepository: trunkS})
b.ex.Forge = &fakeForge{goMods: map[string]string{controllerRepo + " go.mod " + head2: capturedGoMod(t, "cb54756c3865")},
onTrunk: map[string]bool{"cb54756c3865": true}}
b.judge(t, hostCase(t, pullOf(host, 5, head1), pullOf(controllerRepo, 9, head2)))
if c := besideOf(t, b, host)[controllerRepo]; c.Commit != head2 || c.Ref != "refs/pull/9/head" {
t.Errorf("the controller beside the node-engine in a group = %+v, want the member's head %s", c, head2)
}
}
func TestThePinOfTheControllerThatCannotBeReadFailsTheCheck(t *testing.T) { // review D2
b := newBench(t)
b.ex.Trunks = trunksOf(t, map[string]string{"novox/mesh-controller": trunkA})
@@ -355,56 +313,3 @@ func TestAMeshLabHeadGetsACheck(t *testing.T) {
t.Fatalf("mesh-lab was not fetched: %+v", fetched)
}
}
// 1295: the node-engine's step-nine test fails without the controller beside it, so a checker that knows no trunk head of the
// controller fails the check at the placing, loudly, and does not place none.
func TestTheNodeEnginesCheckFailsWhereTheControllersTrunkIsNotKnown(t *testing.T) {
const host = "novox/mesh-host"
b := newBench(t)
b.ex.Trunks = trunksOf(t, map[string]string{host: trunkA, catalogueRepository: trunkS, SDKRepository: trunkS})
v := b.judge(t, hostCase(t, pullOf(host, 5, head1))).Verdict
if v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "mesh-controller") || !strings.Contains(v.Failed.Words, "cannot be placed") {
t.Fatalf("verdict = %+v", v)
}
}
// sameBranchBench is a try of mesh-delivery's feat/r6 at head1 whose go.mod pins the SDK at pin, with the SDK's feat/r6 at head2.
func sameBranchBench(t *testing.T, pin string) (*bench, *Case) {
b := groupBench(t, pin)
b.ex.Forge.(*fakeForge).branches = map[string]map[string]string{
repoA: {"main": trunkA, "feat/r6": head1},
repoSDK: {"main": "cb54756c3865" + strings.Repeat("0", 28), "feat/r6": head2},
}
c := caseOf(t, pullOf(repoA, 7, head1))
c.Try = &TryOptions{}
return b, c
}
func TestATryPinnedToTheHeadOfTheSDKBranchOfTheTriedBranchPasses(t *testing.T) { // 1921/AC1
b, c := sameBranchBench(t, head2[:12])
if v := b.judge(t, c).Verdict; v.State != delivery.VerdictPass {
t.Fatalf("verdict = %+v", v)
}
if n := c.Notes(); len(n) != 1 || n[0] != "pin accepted: head of mesh-sdk feat/r6" {
t.Errorf("notes = %q", n)
}
}
func TestATryPinnedToAnotherCommitIsStillRefused(t *testing.T) { // 1921/AC2
b, c := sameBranchBench(t, "9aaa5c80ac5c")
v := b.judge(t, c).Verdict
if v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "9aaa5c80ac5c") || !strings.Contains(v.Failed.Words, "neither on mesh-sdk's trunk nor the group's head") {
t.Fatalf("verdict = %+v", v)
}
if len(c.Notes()) != 0 {
t.Errorf("notes = %q", c.Notes())
}
}
func TestAMergeCheckWithTheSameBranchPinIsStillRefusedWithoutAGroupPullRequest(t *testing.T) { // 1921/AC3
b, _ := sameBranchBench(t, head2[:12])
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict // no Try: a pull request's check
if v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "the group has no mesh-sdk pull request") {
t.Fatalf("verdict = %+v", v)
}
}
modules/mesh-delivery-checker/internal/check/try.go
-3
View File
@@ -329,9 +329,6 @@ func (tr *Tryer) run(ctx context.Context, key string) {
t.Took = t.Ended.Sub(t.Started).Round(time.Second).String()
t.Levels, t.Machines = v.Levels, v.Machines
t.Said = Report(v, cs, cs.Pulls[0], out.Builds, out.Slowest, out.Judged)
for _, n := range cs.Notes() {
t.Said += " | " + n
}
switch {
case !judged || IsNodesFailure(v):
t.State = TryError
modules/mesh-delivery-checker/manifest_test.go
+2 -2
View File
@@ -96,14 +96,14 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
if want := []state{{Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "tries", History: 1}, {Name: "verdicts", History: 5}}; !reflect.DeepEqual(m.State, want) {
t.Errorf("state = %+v, want %+v", m.State, want)
}
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-controller.assignments",
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains",
"mesh-delivery-assigner.compositions", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates",
"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) {
t.Errorf("reads = %v, want %v", m.Reads, want)
}
// 1887/A5b: the checker reads nothing of the controller's plans-building (ADR 0483 superseded by ADR 0525 §7), calls no
// build module's tool and holds no capacity: it puts check asks on the Go and Node build seats (ADR 0525 §1, §4).
if want := []string{"gitea.gitea_commits", "gitea.gitea_compare", "gitea.gitea_get_file", "gitea.gitea_list_branches"}; !reflect.DeepEqual(sorted(m.Invokes), want) {
if want := []string{"gitea.gitea_commits", "gitea.gitea_compare", "gitea.gitea_get_file"}; !reflect.DeepEqual(sorted(m.Invokes), want) {
t.Errorf("invokes = %v, want only the forge's reads for a mesh-sdk pin (ADR 0484 clause 9)", m.Invokes)
}
if want := []string{"node-build-go", "node-build-node"}; !reflect.DeepEqual(sorted(m.Uses), want) {
modules/mesh-delivery-checker/module.json
+1 -3
View File
@@ -21,7 +21,6 @@
"mesh-build-go.toolchains",
"mesh-build-node.steps",
"mesh-build-node.toolchains",
"mesh-controller.assignments",
"mesh-delivery-assigner.compositions",
"mesh-delivery-builder.builds",
"mesh-delivery-gate.source",
@@ -47,8 +46,7 @@
"invokes": [
"gitea.gitea_commits",
"gitea.gitea_compare",
"gitea.gitea_get_file",
"gitea.gitea_list_branches"
"gitea.gitea_get_file"
],
"own-secrets": {
modules/mesh-delivery-checker/reads-outside
-1
View File
@@ -1 +0,0 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
modules/mesh-delivery-forge-bridge/reads-outside
-1
View File
@@ -1 +0,0 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go
+9 -208
View File
@@ -7,10 +7,8 @@ import (
"encoding/json"
"fmt"
"io"
"maps"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"time"
@@ -440,9 +438,8 @@ func TestIndexerGivesACandidateItsFingerprintWhenTheCheckerCreatesThePreviewsBuc
// TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual: two merged messages of mesh-sdk (its module.json at 9cb4d59,
// whose lib names its paths) differ in the root tree, as a change to go/ alone makes them, and carry the same objects of the
// named paths; the module's kept tree, and so its fingerprint's tree input, is the same after both, its new form
// (fingerprint or reason) does not move, and so the second writes no value at all (issue 1758, I9, ADR 0499 check clause 6
// in miniature; the operator, 2026-10-25: a module's value moves only with the module).
// named paths; the module's kept tree, and so its fingerprint's tree input, is the same after both, and its new form
// (fingerprint or reason) does not move (issue 1758, I9, ADR 0499 check clause 6 in miniature).
func TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual(t *testing.T) {
manifest, err := os.ReadFile("../../internal/index/testdata/sdk-9cb4d59-mesh-sdk.module.json")
if err != nil {
@@ -451,7 +448,7 @@ func TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual(t *testing.T) {
objects := map[string]string{"src": "1111111111111111111111111111111111111111", "test": "2222222222222222222222222222222222222222",
"package.json": "3333333333333333333333333333333333333333", "package-lock.json": "4444444444444444444444444444444444444444", "tsconfig.json": "5555555555555555555555555555555555555555"}
js, ctx, _, modules := startIndexer(t, func(context.Context, jetstream.KeyValue) {})
publish := func(merge, root string, objects map[string]string) {
merged := func(merge, root string) delivery.Module {
data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-sdk", Branch: "feat/x", Number: 70,
Head: "0123456789abcdef0123456789abcdef01234567", Merge: merge, Root: root, Trees: map[string]string{"": root},
Manifests: map[string]string{"": string(manifest)}, Paths: map[string]map[string]string{"": objects}})
@@ -461,8 +458,6 @@ func TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual(t *testing.T) {
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-sdk.merged", data); err != nil {
t.Fatal(err)
}
}
at := func(merge string) delivery.Module {
var m delivery.Module
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err := modules.Get(ctx, "mesh-sdk"); err == nil {
@@ -474,206 +469,12 @@ func TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual(t *testing.T) {
t.Fatalf("modules.mesh-sdk = %+v within 5 s, want the merge %s", m, merge)
return m
}
const firstMerge, secondMerge, thirdMerge = "89abcdef0123456789abcdef0123456789abcdef", "99abcdef0123456789abcdef0123456789abcdef", "a9abcdef0123456789abcdef0123456789abcdef"
publish(firstMerge, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", objects)
first := at(firstMerge)
if first.Inputs2 == nil || !strings.HasPrefix(first.Inputs2.Tree, delivery.NamedTreePrefix) {
t.Fatalf("kept inputs %+v, want a named tree", first.Inputs2)
first := merged("89abcdef0123456789abcdef0123456789abcdef", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
second := merged("99abcdef0123456789abcdef0123456789abcdef", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")
if first.Inputs2 == nil || second.Inputs2 == nil || first.Inputs2.Tree != second.Inputs2.Tree || !strings.HasPrefix(first.Inputs2.Tree, delivery.NamedTreePrefix) {
t.Fatalf("kept trees %+v and %+v, want one named tree", first.Inputs2, second.Inputs2)
}
// The go/-only merge, then one that changes src/: the stream is taken in order, so once the third is written the second
// was heard, and it wrote nothing: the module's value stays at the commit its fingerprint first appeared at (the
// operator, 2026-10-25).
publish(secondMerge, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", objects)
changed := maps.Clone(objects)
changed["src"] = "6666666666666666666666666666666666666666"
publish(thirdMerge, "cccccccccccccccccccccccccccccccccccccccc", changed)
at(thirdMerge)
history, err := modules.History(ctx, "mesh-sdk")
if err != nil {
t.Fatal(err)
}
var commits []string
for _, e := range history {
if m, err := delivery.DecodeModule(e.Value()); err == nil {
commits = append(commits, m.Commit)
if m.Commit == firstMerge && (m.Inputs2 == nil || m.Inputs2.Tree != first.Inputs2.Tree || m.FP2 != first.FP2 || m.None2 != first.None2) {
t.Errorf("the value at the first merge moved: %+v, was %+v", m, first)
}
}
}
if slices.Contains(commits, secondMerge) {
t.Errorf("the go/-only merge wrote a value: the commits written are %v", commits)
}
}
// TestIndexerJudgesAnUnreadableVerdictAgainAtStart: the modules bucket holds avahi as an older SDK wrote it, with the reason
// 'the manifest cannot be read', the inputs the merge carried (the tree, the lock hash by file name) and the digest of its
// manifest, which manifests.<digest> holds and the running SDK reads; the go toolchain is reported. Within 5 s of the start
// and with no repository merge the value carries a fingerprint and no reason (issue 1923).
func TestIndexerJudgesAnUnreadableVerdictAgainAtStart(t *testing.T) { // 1923/AC3
manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json")
if err != nil {
t.Fatal(err)
}
const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"
digest := delivery.ManifestDigest(manifest)
_, ctx, _, modules := startIndexer(t, func(context.Context, jetstream.KeyValue) {}, func(ctx context.Context, js jetstream.JetStream) {
mkey, err := delivery.ManifestsKey(digest)
if err != nil {
t.Fatal(err)
}
mdata, err := delivery.EncodeManifest(delivery.Manifest{Module: "avahi", JSON: string(manifest)})
if err != nil {
t.Fatal(err)
}
manifests, err := js.KeyValue(ctx, delivery.ManifestsBucket)
if err != nil {
t.Fatal(err)
}
if _, err := manifests.Put(ctx, mkey, mdata); err != nil {
t.Fatal(err)
}
stale, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: "89abcdef0123456789abcdef0123456789abcdef", Path: "modules/avahi",
Manifest: digest, None2: "the manifest cannot be read: claims node-build-go, which is a seat in the mesh's own namespace that it does not define",
Inputs2: &delivery.KeptInputs2{Tree: "tree-avahi", Locks: map[string]string{"go.sum": goSum}}})
if err != nil {
t.Fatal(err)
}
held, err := js.KeyValue(ctx, delivery.ModulesBucket)
if err != nil {
t.Fatal(err)
}
if _, err := held.Put(ctx, "avahi", stale); err != nil {
t.Fatal(err)
}
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ToolchainsBucket("mesh-build-go"), History: 1})
if err != nil {
t.Fatal(err)
}
key, err := delivery.ToolchainKey("n1", "go")
if err != nil {
t.Fatal(err)
}
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "go", Version: "go1.27.1", Recipe: "1", At: time.Now()})
if err != nil {
t.Fatal(err)
}
if _, err := kv.Put(ctx, key, value); err != nil {
t.Fatal(err)
}
})
var got delivery.Module
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err := modules.Get(ctx, "avahi"); err == nil {
if m, err := delivery.DecodeModule(e.Value()); err == nil {
if got = m; got.FP2 != "" {
break
}
}
}
}
if got.FP2 == "" || got.None2 != "" {
t.Errorf("avahi = fp2 %q none2 %q within 5 s of the start, want a fingerprint and no reason", got.FP2, got.None2)
}
}
// TestIndexerCountsOnlyTheAssignedBuildNodesToolchains (issue 1940): the toolchains bucket holds the report of n2, where
// mesh-build-go was unassigned (its assignment ended) at another version than n1's, as ace's v26.12.0 stayed beside the
// v26.12.1 of the nodes mesh-build-node runs on. The indexer gives the merged module a fingerprint from n1's toolchain
// alone; when the controller assigns mesh-build-go to n2 again, its report counts and the builders disagree (the guard
// stays); when the assignment ends again, the fingerprint comes back. No restart, no timer: the assignments' watch.
func TestIndexerCountsOnlyTheAssignedBuildNodesToolchains(t *testing.T) {
manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json")
if err != nil {
t.Fatal(err)
}
const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"
var assignments jetstream.KeyValue
js, ctx, _, modules := startIndexer(t, func(context.Context, jetstream.KeyValue) {}, func(ctx context.Context, js jetstream.JetStream) {
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ToolchainsBucket("mesh-build-go"), History: 1})
if err != nil {
t.Fatal(err)
}
for node, version := range map[string]string{"n1": "go1.27.3", "n2": "go1.27.2"} {
key, _ := delivery.ToolchainKey(node, "go")
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "go", Version: version, Recipe: "1", At: time.Now()})
if err != nil {
t.Fatal(err)
}
if _, err := kv.Put(ctx, key, value); err != nil {
t.Fatal(err)
}
}
// The controller's assignments mirror, in its layout (mesh-controller internal/chain Assignment).
if assignments, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "mesh-controller_assignments", History: 1}); err != nil {
t.Fatal(err)
}
for key, value := range map[string]string{
"n1.mesh-build-go": `{"node":"n1","module":"mesh-build-go","assigned":"2026-10-20T08:00:00Z"}`,
"n2.mesh-build-go": `{"node":"n2","module":"mesh-build-go","assigned":"2026-10-20T08:00:00Z","ended":"2026-10-23T08:00:00Z"}`,
} {
if _, err := assignments.Put(ctx, key, []byte(value)); err != nil {
t.Fatal(err)
}
}
})
data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 302,
Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0",
Trees: map[string]string{"modules/avahi": "newtree"},
Manifests: map[string]string{"modules/avahi": string(manifest)},
Locks: map[string]map[string]string{"modules/avahi": {"go.sum": goSum}}})
if err != nil {
t.Fatal(err)
}
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-catalog.merged", data); err != nil {
t.Fatal(err)
}
read := func(ok func(delivery.Module) bool) delivery.Module {
var got delivery.Module
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err := modules.Get(ctx, "avahi"); err == nil {
if m, err := delivery.DecodeModule(e.Value()); err == nil {
if got = m; ok(m) {
break
}
}
}
}
return got
}
agreed := func(m delivery.Module) bool { return m.FP2 != "" && m.Toolchains["go"].Version == "go1.27.3" }
held := read(agreed)
if !agreed(held) {
t.Fatalf("within 5 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27.3 alone", held.FP2, held.None2, held.Toolchains)
}
if _, err := assignments.Put(ctx, "n2.mesh-build-go", []byte(`{"node":"n2","module":"mesh-build-go","assigned":"2026-10-24T08:00:00Z"}`)); err != nil {
t.Fatal(err)
}
// n2's report, written since its assignment, counts: the builders disagree. A fingerprint already computed is kept
// (issue 1940), and only a new one needs the agreement: a merge moving avahi gets none until they agree.
time.Sleep(500 * time.Millisecond)
if e, err := modules.Get(ctx, "avahi"); err != nil {
t.Fatal(err)
} else if m, err := delivery.DecodeModule(e.Value()); err != nil || m.FP2 != held.FP2 {
t.Fatalf("after n2's assignment avahi has fp2 %q, none2 %q: want its fingerprint %q kept", m.FP2, m.None2, held.FP2)
}
moved := strings.Replace(string(data), `"newtree"`, `"movedtree"`, 1)
if moved == string(data) {
t.Fatal("the merge names no tree to move")
}
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-catalog.merged", []byte(moved)); err != nil {
t.Fatal(err)
}
disagree := func(m delivery.Module) bool {
return m.FP2 == "" && strings.HasPrefix(m.None2, "builders disagree: go1.27.2 recipe 1 on n2")
}
if m := read(disagree); !disagree(m) {
t.Fatalf("within 5 s of a merge after n2's assignment avahi has fp2 %q, none2 %q: want the builders to disagree", m.FP2, m.None2)
}
if _, err := assignments.Put(ctx, "n2.mesh-build-go", []byte(`{"node":"n2","module":"mesh-build-go","ended":"2026-10-24T09:00:00Z"}`)); err != nil {
t.Fatal(err)
}
if m := read(agreed); !agreed(m) {
t.Fatalf("within 5 s of n2's unassignment avahi has fp2 %q, none2 %q", m.FP2, m.None2)
if first.FP2 != second.FP2 || first.None2 != second.None2 {
t.Errorf("the new form moved: %q/%q, then %q/%q", first.FP2, first.None2, second.FP2, second.None2)
}
}
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go
+2 -30
View File
@@ -2,9 +2,6 @@
// bridge's messages on the stream FORGE and keeps the candidate groups (candidates.<group>): the pull requests that share
// a branch, their heads and the modules each touches. It reads the controller's builds to know which module a changed
// directory is. It serves no request subject and holds no timer: every reaction is a message or a watch (ADR 0437 §8).
//
// Its build of 2026-10-24 00:37 named a program whose push never reached the artifact store, and a retry sends that
// same build again; this paragraph was added only to give the indexer a new build (issue 1933).
package main
import (
@@ -291,7 +288,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
defer calls.Close()
ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Builds: known, Logf: logger.Printf,
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call)}
Named: namedVia(calls.Call)}
// The gate's source bucket (mesh-delivery-gate_source) comes with the gate (release 3e, novox/hq ADR 0450 §1): its key
// main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does
// not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate.
@@ -313,26 +310,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
go func() { sourceErrs <- follow(ctx, sw, each) }()
}
// Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's
// assignments are settled first, so the toolchains settle through the gate already knowing them.
gate := newToolchainGate(sortedModules(), ix.PutToolchain, logger.Printf)
assignErrs, err := watchAssignments(ctx, js, gate, func(ctx context.Context) {
if err := ix.Recompute(ctx); err != nil {
logger.Printf("writing the modules again after an assignment of a build module changed: %v", err)
}
}, logger)
if err != nil {
return err
}
tools := newToolchainWatch(ctx, js, toolchainSink{
put: gate.Toolchain,
apply: func(ctx context.Context, module, key string, value []byte, deleted bool) error {
if !gate.Toolchain(module, key, value, deleted) {
return nil
}
return ix.Recompute(ctx)
},
recompute: ix.Recompute}, logger)
tools := newToolchainWatch(ctx, js, toolchainSink{put: ix.PutToolchain, apply: ix.ApplyToolchain, recompute: ix.Recompute}, logger)
if err := tools.start(); err != nil {
return err
}
@@ -347,10 +325,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err := ix.RecomputeAtStart(ctx); err != nil {
logger.Printf("computing the new-form fingerprints at start: %v", err)
}
// The modules the trunk no longer holds go from the bucket (issue 1900); a merge does the same from its own message.
if err := ix.Reconcile(ctx); err != nil {
logger.Printf("dropping the modules the trunk does not hold at start: %v", err)
}
// A build module says each time it wrote its toolchains key: its stream, and the indexer's durable consumer on it, are
// made by the controller as the forge's are. Where the stream is not there yet the indexer says so and is started again
// with the build module, as with the gate's source.
@@ -396,8 +370,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return err
case err := <-sourceErrs:
return err
case err := <-assignErrs:
return err
case err := <-tools.errs:
return err
case err := <-previews.errs:
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go
-47
View File
@@ -46,50 +46,3 @@ func namedVia(call toolCaller) func(ctx context.Context, repository, sha string,
return a.Paths, nil
}
}
// trunkVia asks the forge for the module directories of a commit (issue 1900): the keys of the `trees` the same tool
// answers without paths, one per directory that holds a module.json (the root as ""; the tool lists no other).
func trunkVia(call toolCaller) func(ctx context.Context, repository, sha string) ([]string, error) {
return func(_ context.Context, repository, sha string) ([]string, error) {
owner, repo, ok := strings.Cut(delivery.RepositoryName(repository), "/")
if !ok {
return nil, fmt.Errorf("%q is not <owner>/<name>", repository)
}
raw, err := call(commitTreesTool, map[string]any{"owner": owner, "repo": repo, "sha": sha})
if err != nil {
return nil, err
}
var a struct {
Trees map[string]string `json:"trees"`
}
if err := json.Unmarshal(raw, &a); err != nil {
return nil, fmt.Errorf("the answer of %s is not {root, trees}: %w", commitTreesTool, err)
}
dirs := make([]string, 0, len(a.Trees))
for dir := range a.Trees {
dirs = append(dirs, dir)
}
return dirs, nil
}
}
// containsVia asks the forge whether a commit is in another (gitea.gitea_contains: contains is true for the same commit or an ancestor).
func containsVia(call toolCaller) func(ctx context.Context, repository, commit, in string) (bool, error) {
return func(_ context.Context, repository, commit, in string) (bool, error) {
owner, repo, ok := strings.Cut(delivery.RepositoryName(repository), "/")
if !ok {
return false, fmt.Errorf("%q is not <owner>/<name>", repository)
}
raw, err := call("gitea.gitea_contains", map[string]any{"owner": owner, "repo": repo, "commit": commit, "in": in})
if err != nil {
return false, err
}
var a struct {
Contains *bool `json:"contains"`
}
if err := json.Unmarshal(raw, &a); err != nil || a.Contains == nil {
return false, fmt.Errorf("the answer of gitea.gitea_contains names no contains: %v", err)
}
return *a.Contains, nil
}
}
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go
-21
View File
@@ -3,8 +3,6 @@ package main
import (
"context"
"encoding/json"
"slices"
"strings"
"testing"
)
@@ -31,22 +29,3 @@ func TestNamedViaAsksTheTreesToolWithThePathsAndTakesTheirObjects(t *testing.T)
t.Error("an answer missing a path asked was taken")
}
}
func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3
var gotArgs any
call := func(key string, body any) (json.RawMessage, error) {
gotArgs = body
return json.RawMessage(`{"root":"r","trees":{"modules/a":"x","modules/b":"y"}}`), nil
}
dirs, err := trunkVia(call)(context.Background(), "novox/mesh-catalog", strings.Repeat("a", 40))
if err != nil {
t.Fatal(err)
}
slices.Sort(dirs)
if !slices.Equal(dirs, []string{"modules/a", "modules/b"}) {
t.Errorf("dirs = %v", dirs)
}
if _, has := gotArgs.(map[string]any)["paths"]; has {
t.Error("the trunk asked for paths")
}
}
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/standing.go
-209
View File
@@ -1,209 +0,0 @@
package main
// This file is the same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker (issue 1940): it moves
// to mesh-sdk go/delivery/buildrun once mesh-sdk can be delivered again, since a change of mesh-sdk waits on the very
// toolchain agreement this file repairs.
import (
"context"
"encoding/json"
"fmt"
"log"
"maps"
"slices"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provider"
)
// A build module writes <node>.<language> of its toolchains state on every node it runs on, and nothing deletes the key
// when the module is unassigned from a node: the controller writes no module's state, and the module is gone. A report of
// a node the build module no longer runs on is therefore not a builder (issue 1940: mesh-build-node unassigned from ace
// on 2026-10-23 left ace.typescript at v26.12.0, and every Node fingerprint said "builders disagree").
//
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
// key <node>.<module>) hold an open assignment of its build module on its node, and only once it was written since that
// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report
// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node
// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran
// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
// refusal is the guard of novox/hq ADR 0465, and it stays.
type toolchainGate struct {
mu sync.Mutex
modules map[string]bool // the build modules whose toolchains pass the gate
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
logf func(format string, a ...any)
known bool // the assignments are settled
standing map[string]time.Time // <node>.<module> with an open assignment -> since when it stands
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
left map[string]string // <module>/<key> -> why it was said left out, so each reason is said once
}
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{},
raw: map[string]map[string][]byte{}, left: map[string]string{}}
for _, m := range modules {
g.modules[m] = true
}
return g
}
// Toolchain takes one change of a build module's toolchains bucket in and answers whether what it passed on changed.
func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool) bool {
g.mu.Lock()
defer g.mu.Unlock()
if g.raw[module] == nil {
g.raw[module] = map[string][]byte{}
}
if deleted {
delete(g.raw[module], key)
} else {
g.raw[module][key] = value
}
return g.pass(module, key)
}
// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers
// whether what it passed on changed.
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool {
node, module, ok := strings.Cut(key, ".")
if !ok || !g.modules[module] {
return false
}
assigned, open := openAssignment(value)
open = open && !deleted
g.mu.Lock()
defer g.mu.Unlock()
was, stood := g.standing[key]
since := assigned
if since.IsZero() {
// A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it.
since = at
if stood {
since = was
}
}
if open == stood && (!open || since.Equal(was)) {
return false
}
if open {
g.standing[key] = since
} else {
delete(g.standing, key)
}
if !g.known {
return false
}
changed := false
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
if n, _, err := delivery.ParseToolchainKey(k); err == nil && n == node {
changed = g.pass(module, k) || changed
}
}
return changed
}
// Settled says the assignments are read: from now on only the reports of assigned nodes pass. It answers whether what it
// passed on changed.
func (g *toolchainGate) Settled() bool {
g.mu.Lock()
defer g.mu.Unlock()
g.known = true
changed := false
for _, module := range slices.Sorted(maps.Keys(g.raw)) {
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
changed = g.pass(module, k) || changed
}
}
return changed
}
// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not
// known), deleted otherwise. The caller holds mu.
func (g *toolchainGate) pass(module, key string) bool {
value, held := g.raw[module][key]
if !held {
return g.forward(module, key, nil, true)
}
node, _, err := delivery.ParseToolchainKey(key)
why := ""
if err == nil && g.known {
why = g.leftOut(module, node, value)
}
if why == "" {
delete(g.left, module+"/"+key)
return g.forward(module, key, value, false)
}
if g.left[module+"/"+key] != why {
g.left[module+"/"+key] = why
g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why)
}
return g.forward(module, key, nil, true)
}
// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module,
// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse.
// The caller holds mu.
func (g *toolchainGate) leftOut(module, node string, value []byte) string {
since, open := g.standing[node+"."+module]
if !open {
return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node)
}
tc, err := delivery.DecodeToolchain(value)
if err == nil && tc.At.Before(since) {
return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again",
tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node)
}
return ""
}
// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not
// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when
// it was assigned, zero where the value does not say.
func openAssignment(value []byte) (time.Time, bool) {
var a struct {
Assigned time.Time `json:"assigned"`
Ended time.Time `json:"ended"`
State string `json:"state"`
}
if json.Unmarshal(value, &a) != nil {
return time.Time{}, false
}
return a.Assigned, a.Ended.IsZero() && a.State == ""
}
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
// told whenever what the gate passes on changed. Where the bucket cannot be read (not there, or this module may not read
// it) it says so and the gate passes every report on, as before; an error of the watch after that ends on the channel.
func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate, changed func(context.Context), logger *log.Logger) (<-chan error, error) {
errs := make(chan error, 1)
kv, err := js.KeyValue(ctx, provider.AssignmentsBucket)
if err != nil {
// Not fatal: an indexer, builder or checker that stops for it stops every delivery (a grant comes with a delivery).
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
return errs, nil
}
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
w, err := settle(ctx, kv, each)
if err != nil {
return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err)
}
if gate.Settled() {
changed(ctx)
}
go func() {
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) {
changed(ctx)
}
})
}()
return errs, nil
}
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/standing_test.go
-165
View File
@@ -1,165 +0,0 @@
package main
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
import (
"fmt"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
func gateReport(t *testing.T, node, version string) (string, []byte) {
t.Helper()
key, err := delivery.ToolchainKey(node, "typescript")
if err != nil {
t.Fatal(err)
}
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: time.Unix(0, 0).UTC()})
if err != nil {
t.Fatal(err)
}
return key, value
}
func agreedOf(set *buildrun.ToolchainSet) (string, string) {
tc, none := delivery.AgreedToolchain(set.Entries("typescript"))
return tc.Version, none
}
// The live case of issue 1940: mesh-build-node was unassigned from ace, whose report stayed at v26.12.0 while the
// three nodes it runs on report v26.12.1. Without the assignments every report counts (and they disagree); with them
// ace's is left out and the three agree; assigned to ace again, its report counts again and the guard refuses again.
func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
set := &buildrun.ToolchainSet{}
var said []string
gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, f) })
for node, version := range map[string]string{"ace": "v26.12.0", "g14": "v26.12.1", "novox": "v26.12.1", "shanks": "v26.12.1"} {
key, value := gateReport(t, node, version)
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
}
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree") {
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
}
for _, node := range []string{"g14", "novox", "shanks"} {
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{})
}
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{})
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter
if !gate.Settled() {
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
}
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
t.Fatalf("the assigned nodes agree on v26.12.1; got %q, %q", v, none)
}
if len(said) != 1 {
t.Fatalf("ace's report is said left out once; said %v", said)
}
// ace reports again while unassigned (an old value seen again): still left out.
key, value := gateReport(t, "ace", "v26.12.0")
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
t.Fatal("an unassigned node's report changed what was passed on")
}
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) {
t.Fatal("assigning ace again passed nothing on")
}
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
t.Fatalf("builders that run different versions still disagree; got %q", none)
}
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) {
t.Fatal("a pending assignment is no assignment on the machine")
}
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
t.Fatalf("got %q, %q", v, none)
}
gate.Assignment("ace.mesh-build-node", nil, true, time.Time{})
if v, _ := agreedOf(set); v != "v26.12.1" {
t.Fatalf("got %q", v)
}
// A report deleted from the bucket goes whether its node is assigned or not.
key, _ = gateReport(t, "g14", "v26.12.1")
if !gate.Toolchain(buildrun.NodeBuildModule, key, nil, true) {
t.Fatal("deleting g14's report passed nothing on")
}
if n := len(set.Entries("typescript")); n != 2 {
t.Fatalf("novox and shanks are left; got %d entries", n)
}
}
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript
// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node
// writes it again. An assignment whose value names no time stands since the revision that opened it.
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again
report := func(t *testing.T, node, version string, at time.Time) (string, []byte) {
key, _ := gateReport(t, node, version)
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
if err != nil {
t.Fatal(err)
}
return key, value
}
for _, c := range []struct {
name string
assignment string // ace's assignment value
revision time.Time // when the bus wrote it
at time.Time // when ace wrote its report
counts bool
}{
{"no report yet, then one", `{}`, time.Time{}, time.Now(), true},
{"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false},
{"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false},
{"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true},
} {
t.Run(c.name, func(t *testing.T) {
set := &buildrun.ToolchainSet{}
var said []string
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
for _, node := range []string{"g14", "novox", "shanks"} {
gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{})
key, value := gateReport(t, node, "v26.12.1")
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
}
gate.Settled()
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) {
t.Fatal("an assignment with no report passed something on")
}
key, value := report(t, "ace", "v26.12.0", c.at)
if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts {
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
}
// A rewrite of the assignment that does not move it changes nothing, and nothing is said again.
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) {
t.Fatal("the same assignment written again changed what was passed on")
}
v, none := agreedOf(set)
if c.counts {
if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none)
}
return
}
if v != "v26.12.1" || none != "" {
t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none)
}
if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") {
t.Fatalf("said %q, want once why ace's report is left out", said)
}
// ace writes its report again since it was assigned: it counts.
key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute))
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
t.Fatal("ace's new report was not passed on")
}
if n := len(set.Entries("typescript")); n != 4 {
t.Fatalf("four builders report; got %d", n)
}
})
}
}
modules/mesh-delivery-indexer/internal/index/candidates.go
+1 -4
View File
@@ -52,10 +52,7 @@ type Indexer struct {
// manifest's paths (ADR 0495 clause 1, the lead's decision L8 of R4e). Nil asks nothing, and a module naming paths
// has none.
Named func(ctx context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error)
Trunk func(ctx context.Context, repository, sha string) ([]string, error)
// Contains says whether a commit is the second or an ancestor of it (gitea.gitea_contains).
Contains func(ctx context.Context, repository, commit, in string) (bool, error)
Logf func(format string, args ...any)
Logf func(format string, args ...any)
mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach
source sourceNow
modules/mesh-delivery-indexer/internal/index/inputs.go
+1 -1
View File
@@ -69,7 +69,7 @@ func inputsOf(module, raw, tree string, locks map[string]string, base baseFP) mo
func inputsOfNamed(module, raw, tree string, locks, objects map[string]string, base baseFP) moduleInputs {
m, err := catalogue.ParseManifestForInputs([]byte(raw))
if err != nil {
return moduleInputs{None2: None2Unreadable + err.Error()}
return moduleInputs{None2: "the manifest cannot be read: " + err.Error()}
}
if m.Module != module {
return moduleInputs{None2: fmt.Sprintf("the manifest is of module %q, not %q", m.Module, module)}
modules/mesh-delivery-indexer/internal/index/inputs_test.go
+1 -2
View File
@@ -111,8 +111,7 @@ func TestABaseEntersByTheFingerprintOfItsModuleOrNamesItsAbsence(t *testing.T) {
}
// Through the indexer: modules.mesh-tools holds a fingerprint of the new form (a record written by a later release).
ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
mods.put("mesh-tools", mustEncode(t, delivery.Module{Repository: "novox/mesh-tools", // a base held of another repository stays when the catalogue merges (issue 1900)
Commit: merge, None: "x", FP2: fp, Manifest: strings.Repeat("a", 64)}))
mods.put("mesh-tools", mustEncode(t, delivery.Module{Repository: catalogueRepo, Commit: merge, None: "x", FP2: fp, Manifest: strings.Repeat("a", 64)}))
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "t"}), map[string]string{"modules/avahi": aModuleWithABase(t)}, nil)
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
modules/mesh-delivery-indexer/internal/index/modules.go
+1 -132
View File
@@ -2,10 +2,8 @@ package index
import (
"context"
"fmt"
"maps"
"slices"
"strings"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
@@ -22,8 +20,7 @@ import (
// form (Module.FP, None) is no longer written: it was computed from those records (ADR 0453 clause 6). A module the
// bucket holds of another repository is left as it is and said: a move between repositories is the operator's (ADR 0420
// clause 7). Two directories naming one module write the one the bucket holds, or the first by name, and say the other.
// A value the module already holds but for its commit is not written again: its commit stays the one its fingerprint first
// appeared at (writeOverlay), so a merge writes only the modules it changes.
// A value the module already holds is not written again.
func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
digests, err := ix.keepManifests(ctx, f)
if err != nil {
@@ -36,22 +33,18 @@ func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
objects, namedErr := ix.namedObjects(ctx, f)
overlay := map[string]delivery.Module{}
dirOf := map[string]string{}
named, every := map[string]bool{}, true // every module the message's manifests name, and whether each names one
for _, dir := range slices.Sorted(maps.Keys(f.Manifests)) {
raw := f.Manifests[dir]
name := manifestModule(raw)
if name == "" {
ix.logf("%s at %s: the manifest in %q names no module, so no module is written for it", f.Repository, f.Merge, dir)
every = false
continue
}
key, err := delivery.ModulesKey(name)
if err != nil {
every = false
ix.logf("%s: the module %q has no usable key, skipped: %v", f.Repository, name, err)
continue
}
named[key] = true
cur, has := held[key]
if has && cur.Repository != "" && !delivery.SameRepository(cur.Repository, f.Repository) {
ix.logf("%s at %s: %q names the module %s, which the indexer holds of %s; left as it is (a move between repositories is the operator's)", f.Repository, f.Merge, dir, name, cur.Repository)
@@ -71,24 +64,12 @@ func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
} else {
in := inputsOfNamed(name, raw, f.Trees[dir], f.Locks[dir], objects[dir], ix.baseFP(ctx))
next.Inputs2, next.None2, next.ArtifactPaths = in.Kept, in.None2, in.ArtifactPaths
if in.Kept == nil && strings.HasPrefix(in.None2, None2Unreadable) {
// The manifest cannot be read by this SDK; the inputs the merge carried are kept all the same, so a later SDK
// that reads it judges the module again at start with no merge (issue 1923).
next.Inputs2 = unreadInputs(f.Trees[dir], f.Locks[dir], f.Paths[dir])
}
if in.Kept == nil && namedErr[dir] != "" {
next.None2 = in.None2 + ": the forge could not say them: " + namedErr[dir]
}
}
overlay[key] = next
}
// A module of this repository the merge commit holds no module.json for is gone from the trunk, and so is its key
// (issue 1900). A message that carries no manifests (an older writer) or one that cannot be read whole says nothing.
if len(f.Manifests) > 0 && every {
if err := ix.dropRemoved(ctx, f.Repository, held, func(key string, _ delivery.Module) bool { return named[key] }); err != nil {
return err
}
}
// What this merge moved reaches the modules standing on it (a base's tree, build-agent's fingerprint), in the graph's order.
return ix.recomputeWith(ctx, overlay)
}
@@ -168,115 +149,3 @@ func (ix *Indexer) namedObjects(ctx context.Context, f delivery.Forge) (map[stri
}
return objects, nil
}
// dropRemoved deletes, from the modules the bucket holds of the repository, each one the trunk no longer holds (issue 1900):
// held(key, module) says whether the trunk holds it. A module of another repository, or of none known, is never touched.
func (ix *Indexer) dropRemoved(ctx context.Context, repository string, held map[string]delivery.Module, trunkHolds func(key string, m delivery.Module) bool) error {
for _, key := range slices.Sorted(maps.Keys(held)) {
cur := held[key]
if cur.Repository == "" || !delivery.SameRepository(cur.Repository, repository) || trunkHolds(key, cur) {
continue
}
err := write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
if !found {
return nil, false, nil
}
if m, err := delivery.DecodeModule(value); err != nil || !delivery.SameRepository(m.Repository, repository) {
return nil, false, nil
}
return nil, true, nil
})
if err != nil {
return fmt.Errorf("modules.%s: %w", key, err)
}
ix.logf("modules.%s deleted: %s holds no module of that name at its trunk (issue 1900)", key, repository)
}
return nil
}
// Reconcile deletes the modules keys whose module the trunk does not hold, for each repository the latest source names a
// head of: the module directories at that head are asked of the forge (Trunk), and a module of the repository whose
// directory is not among them goes (issue 1900). It is run at the indexer's start; a merge does the same from its own
// message (applyMerged). A repository whose trunk cannot be read, or reads as holding no module at all, is left as it is
// and said: a failure never empties the bucket.
func (ix *Indexer) Reconcile(ctx context.Context) error {
if ix.Modules == nil || ix.Trunk == nil {
return nil
}
ix.mu.Lock()
heads := maps.Clone(ix.source.heads)
ix.mu.Unlock()
held, err := ix.heldModules(ctx)
if err != nil {
return err
}
for _, repository := range slices.Sorted(maps.Keys(heads)) {
mine := map[string]delivery.Module{}
for key, m := range held {
if m.Repository != "" && delivery.SameRepository(m.Repository, repository) {
mine[key] = m
}
}
if len(mine) == 0 {
continue
}
dirs, err := ix.Trunk(ctx, repository, heads[repository])
if err != nil {
ix.logf("%s at %s: the modules of the trunk cannot be read, so none is dropped: %v", repository, heads[repository], err)
continue
}
if len(dirs) == 0 {
ix.logf("%s at %s: the trunk reads as holding no module, so none is dropped", repository, heads[repository])
continue
}
there := map[string]bool{}
for _, d := range dirs {
there[pathOf(strings.Trim(d, "/"))] = true
}
head := heads[repository]
// The source can lag a merge already applied: a key written at a commit the head read does not hold (a merge heard
// first) is not read as removed, only one whose commit the head is, or contains.
stays := func(_ string, m delivery.Module) bool {
if m.Path == "" || there[m.Path] {
return true
}
if m.Commit == head {
return false
}
if m.Commit == "" || ix.Contains == nil {
return true
}
in, err := ix.Contains(ctx, repository, m.Commit, head)
if err != nil {
ix.logf("%s: whether %s is in %s cannot be read, so the module at %s is kept: %v", repository, m.Commit, head, m.Path, err)
return true
}
return !in
}
if err := ix.dropRemoved(ctx, repository, mine, stays); err != nil {
return err
}
}
return nil
}
// None2Unreadable begins the reason of a module whose manifest the running SDK cannot read.
const None2Unreadable = "the manifest cannot be read: "
// unreadInputs are the inputs a merge carried for a module whose manifest cannot be read: the directory's tree, the lock
// hashes by file name (the artifacts that own them are not known without the manifest) and the objects of the named
// paths the message carried; nil when the message carried no tree. Judging the manifest again (rejudged) turns them into
// the inputs of the module (issue 1923).
func unreadInputs(tree string, locks, paths map[string]string) *delivery.KeptInputs2 {
if tree == "" {
return nil
}
in := &delivery.KeptInputs2{Tree: tree}
if len(locks) > 0 {
in.Locks = maps.Clone(locks)
}
if len(paths) > 0 {
in.Paths = maps.Clone(paths)
}
return in
}
modules/mesh-delivery-indexer/internal/index/modules_test.go
+8 -45
View File
@@ -85,51 +85,14 @@ func moduleOf(t *testing.T, st *memStore, key string) (delivery.Module, bool) {
const aInputs = `{"Module":"a","Tree":"oldtree","Bases":["registry.example/base@sha256:aaa"],"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}`
func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value writes nothing); the operator 2026-10-25 (a module's value moves only with the module)
const repo, next = "novox/mesh-delivery", "cccccccccccccccccccccccccccccccccccccccc"
three := map[string]string{"modules/a": "treeA", "modules/b": "treeB", "modules/c": "treeC"}
for _, tc := range []struct {
name string
then func(ix *Indexer) error // after the first merge of a, b and c at merge, stamped by source revision 7
// writes is the values written by then; commit is each module's commit after.
writes int
commit map[string]string
}{
{name: "the same merge again", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge},
then: func(ix *Indexer) error { return ix.Apply(context.Background(), mergedMsg(repo, three)) }},
{name: "a merge touching one of three modules", writes: 1, commit: map[string]string{"a": next, "b": merge, "c": merge},
then: func(ix *Indexer) error {
m := mergedMsg(repo, map[string]string{"modules/a": "treeA2", "modules/b": "treeB", "modules/c": "treeC"})
m.Merge = next
return ix.Apply(context.Background(), m)
}},
{name: "a later source revision of another repository's merge", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge},
then: func(ix *Indexer) error {
return ix.ApplySource(context.Background(), 9, delivery.Source{Group: "g", Heads: map[string]string{repo: merge, "novox/mesh-catalog": otherMerge}})
}},
} {
t.Run(tc.name, func(t *testing.T) {
ix, mods := modulesIndexer(t)
ctx := context.Background()
if err := ix.Apply(ctx, mergedMsg(repo, three)); err != nil {
t.Fatal(err)
}
if err := ix.ApplySource(ctx, 7, sourceOf(repo, merge)); err != nil {
t.Fatal(err)
}
writes := mods.writes
if err := tc.then(ix); err != nil {
t.Fatal(err)
}
if got := mods.writes - writes; got != tc.writes {
t.Errorf("%d values written, want %d", got, tc.writes)
}
for module, want := range tc.commit {
if m, ok := moduleOf(t, mods, module); !ok || m.Commit != want {
t.Errorf("%s = %+v (%v), want commit %.8s", module, m, ok, want)
}
}
})
func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value writes nothing)
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs))
m := mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})
_ = ix.Apply(context.Background(), m)
rev, writes := mods.revs["a"], mods.writes
_ = ix.Apply(context.Background(), m)
if mods.revs["a"] != rev || mods.writes != writes {
t.Errorf("the same merge wrote again (%d writes, was %d)", mods.writes, writes)
}
}
modules/mesh-delivery-indexer/internal/index/previews.go
+5 -17
View File
@@ -112,12 +112,12 @@ func (ix *Indexer) RecomputeCandidates(ctx context.Context) error {
return nil
}
// previewOf is the preview of a head merged onto the trunk head the source heads name: found, or stale when only a preview
// previewOf is the preview of a head merged onto the trunk head the source names now: found, or stale when only a preview
// onto another trunk head exists, or neither (found false, stale false) when the source is not known or none exists.
func (ix *Indexer) previewOf(heads map[string]string, repository, head string) (pv delivery.Preview, found, stale bool) {
func (ix *Indexer) previewOf(repository, head string) (pv delivery.Preview, found, stale bool) {
ix.mu.Lock()
defer ix.mu.Unlock()
trunk := heads[repository]
trunk := ix.source.heads[repository]
if trunk == "" {
return delivery.Preview{}, false, false
}
@@ -137,23 +137,14 @@ func (ix *Indexer) previewOf(heads map[string]string, repository, head string) (
// withPreviews gives each touched module of the candidate's pull requests the fingerprint its preview gives it, or the
// reason it has none, and the candidate its fpset. It answers whether the candidate changed. A module of a pull request with
// no preview keeps what it had.
//
// The candidate's Base is the source revision whose trunk heads the previews were read onto (novox/hq ADR 0437 §5): written
// whenever the fingerprints are, so the builder merges a preview onto the trunk head its fingerprint was computed on (the head
// of the pull request's repository in that revision), never onto a module value's commit, which stays at the commit its
// fingerprint first appeared at.
func (ix *Indexer) withPreviews(ctx context.Context, c *delivery.Candidate) bool {
ix.mu.Lock()
rev, heads := ix.source.rev, ix.source.heads
ix.mu.Unlock()
changed, read := false, false
changed := false
for i := range c.Pulls {
p := &c.Pulls[i]
pv, found, stale := ix.previewOf(heads, p.Repository, p.Head.Head)
pv, found, stale := ix.previewOf(p.Repository, p.Head.Head)
if !found && !stale {
continue
}
read = true
// A touched module's directory: the one the modules bucket holds of the repository, else the preview's directory
// whose manifest is the one the pull request gave the module (a module the pull request adds; ADR 0501 clause 6).
dirs := map[string]string{}
@@ -207,9 +198,6 @@ func (ix *Indexer) withPreviews(ctx context.Context, c *delivery.Candidate) bool
if changed {
c.FPSet = delivery.FingerprintSet(c.Pulls)
}
if read && rev != 0 && c.Base != rev && (changed || c.Base == 0) {
c.Base, changed = rev, true
}
return changed
}
modules/mesh-delivery-indexer/internal/index/previews_test.go
+1 -45
View File
@@ -2,8 +2,6 @@ package index
import (
"context"
"encoding/json"
"os"
"strings"
"testing"
@@ -89,16 +87,9 @@ func TestACandidatesModuleGetsTheFingerprintATrunkModuleOfTheSameTreeHas(t *test
if got.FP2 != trunk.FP2 || got.None2 != "" || got.Manifest != delivery.ManifestDigest([]byte(avahiOnTools(t))) {
t.Fatalf("candidate avahi = %+v, want the trunk's fingerprint %s", got, trunk.FP2)
}
c := candidateOf(t, cands)
if c.FPSet != delivery.FingerprintSet(c.Pulls) || c.FPSet == "" {
if c := candidateOf(t, cands); c.FPSet != delivery.FingerprintSet(c.Pulls) || c.FPSet == "" {
t.Errorf("fpset = %q", c.FPSet)
}
// The candidate names the source revision the preview was read onto (5, whose catalogue head is the preview's trunk):
// the builder merges the preview onto that head, never onto the module value's commit.
if c.Base != 5 {
t.Errorf("candidate base = %d, want the source revision 5 its preview was read onto", c.Base)
}
written(t, writtenCandidate{Source: sourceRev{Rev: 5, Heads: map[string]string{catalogueRepo: trunkNow}}, Candidate: c})
// Another tree gives another fingerprint.
pv := avahiPreview(t, trunkNow, h1)
m := pv.Modules["modules/avahi"]
@@ -200,38 +191,3 @@ func TestAPreviewOfConflictsGivesTheCandidatesModuleTheReasonItDoesNotMerge(t *t
t.Errorf("candidate avahi = %+v, want it not merging and naming the files", got)
}
}
// writtenCandidate is what the builder's tests read of the indexer (mesh-delivery-builder internal/build): a candidate as the
// indexer writes it and the gate's source revision it names, kept in testdata/candidate-written.json so the builder is tested
// on what the indexer writes, never on a candidate made by hand. MESH_WRITE_TESTDATA=1 writes it again.
type writtenCandidate struct {
Source sourceRev `json:"source"`
Candidate delivery.Candidate `json:"candidate"`
}
type sourceRev struct {
Rev uint64 `json:"rev"`
Heads map[string]string `json:"heads"`
}
func written(t *testing.T, w writtenCandidate) {
t.Helper()
data, err := json.MarshalIndent(w, "", " ")
if err != nil {
t.Fatal(err)
}
data = append(data, '\n')
const file = "testdata/candidate-written.json"
if os.Getenv("MESH_WRITE_TESTDATA") == "1" {
if err := os.WriteFile(file, data, 0o644); err != nil {
t.Fatal(err)
}
}
kept, err := os.ReadFile(file)
if err != nil {
t.Fatal(err)
}
if string(kept) != string(data) {
t.Errorf("%s is not what the indexer writes now (MESH_WRITE_TESTDATA=1 writes it again):\n%s", file, data)
}
}
modules/mesh-delivery-indexer/internal/index/remove_test.go
-144
View File
@@ -1,144 +0,0 @@
package index
import (
"context"
"errors"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
func twoModules(t *testing.T) (*Indexer, *memStore) {
t.Helper()
ix, mods := modulesIndexer(t)
if err := ix.Apply(context.Background(), mergedMsg(catalogueRepo, map[string]string{"modules/a": "t", "modules/b": "t"})); err != nil {
t.Fatal(err)
}
for _, k := range []string{"a", "b"} {
if _, ok := mods.vals[k]; !ok {
t.Fatalf("setup: modules.%s not written", k)
}
}
return ix, mods
}
func TestAMergeThatRemovesAModuleDeletesItsKey(t *testing.T) { // 1900/AC1
ix, mods := twoModules(t)
if err := ix.Apply(context.Background(), mergedMsg(catalogueRepo, map[string]string{"modules/a": "t2"})); err != nil {
t.Fatal(err)
}
if _, ok := mods.vals["b"]; ok {
t.Error("modules.b is still held after a merge whose trunk has no module b")
}
if _, ok := mods.vals["a"]; !ok {
t.Error("modules.a, which the trunk still holds, was deleted")
}
}
func TestAMergeLeavesTheModulesOfAnotherRepository(t *testing.T) { // 1900/AC1 (scope)
ix, mods := twoModules(t)
if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/c": "t"})); err != nil {
t.Fatal(err)
}
if _, ok := mods.vals["a"]; !ok {
t.Error("a merge of another repository deleted modules.a")
}
if _, ok := mods.vals["b"]; !ok {
t.Error("a merge of another repository deleted modules.b")
}
}
func TestAMergeWithoutManifestsDeletesNothing(t *testing.T) { // 1900/AC1: older writers send none, which does not say the trunk is empty
ix, mods := twoModules(t)
f := mergedMsg(catalogueRepo, map[string]string{"modules/a": "t2"})
f.Manifests = nil
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
if _, ok := mods.vals["b"]; !ok {
t.Error("a merge message with no manifests deleted modules.b")
}
}
func TestAPathWithoutAModuleJSONIsNeverAModule(t *testing.T) { // 1900/AC2: testdata such as fixture-build
ix, mods := twoModules(t)
f := mergedMsg(catalogueRepo, map[string]string{"modules/a": "t2"})
f.Trees["testdata/fixture-build"] = "t" // a directory in the tree whose module.json the message does not carry
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
if _, ok := mods.vals["fixture-build"]; ok {
t.Error("modules.fixture-build written for a path without a module.json")
}
}
func TestStartingDeletesTheKeysTheTrunkDoesNotHold(t *testing.T) { // 1900/AC3
ix, mods := twoModules(t)
ctx := context.Background()
if err := ix.ApplySource(ctx, 3, sourceOf(catalogueRepo, merge)); err != nil {
t.Fatal(err)
}
var asked string
ix.Trunk = func(_ context.Context, repository, sha string) ([]string, error) {
asked = repository + "@" + sha
return []string{"modules/a"}, nil
}
ix.Contains = func(context.Context, string, string, string) (bool, error) { return true, nil }
if err := ix.Reconcile(ctx); err != nil {
t.Fatal(err)
}
if asked != catalogueRepo+"@"+merge {
t.Errorf("the trunk was asked as %q, want the source's head of the repository", asked)
}
if _, ok := mods.vals["b"]; ok {
t.Error("modules.b is still held though the trunk holds no module at modules/b")
}
if _, ok := mods.vals["a"]; !ok {
t.Error("modules.a was deleted")
}
}
func TestStartingDeletesNothingWhenTheTrunkCannotBeRead(t *testing.T) { // 1900/AC3: failure is loud, never a mass delete
ix, mods := twoModules(t)
ctx := context.Background()
if err := ix.ApplySource(ctx, 3, sourceOf(catalogueRepo, merge)); err != nil {
t.Fatal(err)
}
for name, trunk := range map[string]func(context.Context, string, string) ([]string, error){
"an error": func(context.Context, string, string) ([]string, error) { return nil, errors.New("forge down") },
"no module at all": func(context.Context, string, string) ([]string, error) { return nil, nil },
} {
ix.Trunk = trunk
_ = ix.Reconcile(ctx)
if len(mods.vals) != 2 {
t.Errorf("%s: %d keys held, want 2 untouched", name, len(mods.vals))
}
}
_ = delivery.RootPath
}
func TestAKeyWrittenAheadOfTheSourceHeadStaysAndOneWithoutAPathIsSkipped(t *testing.T) { // 1900/AC3 (review): the source lags a merge already applied
ix, mods := twoModules(t) // written at the merge commit C
ctx := context.Background()
const older = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
if err := ix.ApplySource(ctx, 3, sourceOf(catalogueRepo, older)); err != nil {
t.Fatal(err)
}
mods.put("nopath", mustEncode(t, delivery.Module{Repository: catalogueRepo, Commit: older, None2: "x", Manifest: strings.Repeat("a", 64)}))
ix.Trunk = func(context.Context, string, string) ([]string, error) { return []string{"modules/a"}, nil }
ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { return commit == in, nil } // C is not in B
if err := ix.Reconcile(ctx); err != nil {
t.Fatal(err)
}
for _, k := range []string{"a", "b", "nopath"} {
if _, ok := mods.vals[k]; !ok {
t.Errorf("modules.%s was deleted", k)
}
}
ix.Contains = func(context.Context, string, string, string) (bool, error) { return false, errors.New("forge down") }
_ = ix.Reconcile(ctx)
if _, ok := mods.vals["b"]; !ok {
t.Error("modules.b deleted though whether its commit is in the head cannot be read")
}
}
modules/mesh-delivery-indexer/internal/index/source.go
+3 -5
View File
@@ -32,8 +32,8 @@ func (ix *Indexer) srcOf(repository, merge string) uint64 {
}
// ApplySource takes a revision of the gate's source.main in. An older revision than the latest seen is left. For each
// repository the source names, each module built from it whose fingerprint was computed at the head the source names, and
// that carries no Src yet, is written again with Src the revision (once: a later revision leaves it); a module computed at another commit is left (a merge heard later is stamped
// repository the source names, each module built from it whose fingerprint was computed at the head the source names is
// written again with Src the revision; a module computed at another commit is left (a merge heard later is stamped
// when it is written, applyMerged), and a repository none of the indexer's modules is built from writes nothing. The
// modules of a repository are those the modules bucket holds of it (ADR 0501 clause 6).
func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s delivery.Source) error {
@@ -71,9 +71,7 @@ func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s delivery.Sourc
ix.logf("modules.%s holds what is not a module, left: %v", key, err)
return nil, false, nil
}
// Stamped once, with the first revision naming the commit as the head: a later revision (a merge of another
// repository) leaves the value as it is, so no revision of the source rewrites a module that did not change.
if cur.Repository != repository || cur.Commit != head || cur.Src != 0 {
if cur.Repository != repository || cur.Commit != head || cur.Src == rev {
return nil, false, nil
}
cur.Src = rev
modules/mesh-delivery-indexer/internal/index/source_test.go
+3 -3
View File
@@ -38,15 +38,15 @@ func TestASourceRevisionStampsTheModulesOfTheRepositoriesHeadItNames(t *testing.
if c, ok := moduleOf(t, mods, "c"); ok {
t.Errorf("module c = %+v: written for a head no merge message of the indexer carried", c)
}
// The same revision again changes nothing, and so does a later one: Src is the first revision naming the commit.
// The same revision again changes nothing; a later revision moves Src.
if err := ix.ApplySource(ctx, 7, s); err != nil {
t.Fatal(err)
}
if err := ix.ApplySource(ctx, 9, s); err != nil {
t.Fatal(err)
}
if m, _ := moduleOf(t, mods, "a"); m.Src != 7 {
t.Errorf("module a Src = %d, want 7", m.Src)
if m, _ := moduleOf(t, mods, "a"); m.Src != 9 {
t.Errorf("module a Src = %d, want 9", m.Src)
}
}
modules/mesh-delivery-indexer/internal/index/stale_none2_test.go
-77
View File
@@ -1,77 +0,0 @@
package index
import (
"context"
"strings"
"testing"
)
// unreadableAvahi is avahi's manifest with a field the running parser refuses: the shape of a manifest an older SDK could
// not read.
func unreadableAvahi(t *testing.T) string {
t.Helper()
raw := strings.Replace(catalogueManifest(t, "avahi"), `"version": "1",`, `"version": "1", "scope": "everything",`, 1)
if raw == catalogueManifest(t, "avahi") {
t.Fatal("the fixture has no version line")
}
return raw
}
func TestAMergeOfAnUnreadableManifestKeepsTheInputsBesideTheReason(t *testing.T) { // 1923/AC1
_, mods := newFormIndexer(t, mod{"avahi", unreadableAvahi(t)})
m, ok := moduleOf(t, mods, "avahi")
if !ok || !strings.HasPrefix(m.None2, "the manifest cannot be read: ") || m.FP2 != "" {
t.Fatalf("module = %+v, want the reason 'the manifest cannot be read' and no fingerprint", m)
}
if m.Inputs2 == nil || m.Inputs2.Tree != "tree-avahi" || m.Inputs2.Locks["go.sum"] != avahiGoSum {
t.Errorf("inputs = %+v, want the directory's tree and its lock hashes kept", m.Inputs2)
}
}
func TestAStaleUnreadableVerdictIsJudgedAgainAtStartAndAnotherReasonIsLeft(t *testing.T) { // 1923/AC2
ctx := context.Background()
// The fingerprint a merge of the readable manifest gives.
ixGood, good := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")})
reportToolchain(t, ixGood, "mesh-build-go", "novox", "go", "go1.27.1")
reportToolchain(t, ixGood, "mesh-build-go", "ace", "go", "go1.27.1")
want, _ := moduleOf(t, good, "avahi")
if want.FP2 == "" {
t.Fatalf("the readable manifest gives no fingerprint: %+v", want)
}
// The same module as an older SDK wrote it: the reason, the inputs kept at the merge, the manifest stored by digest.
ix, mods := newFormIndexer(t, mod{"avahi", unreadableAvahi(t)})
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.27.1")
stale, _ := moduleOf(t, mods, "avahi")
stale.Manifest = want.Manifest
ix.Manifests.(*memStore).put(want.Manifest, good2(t, ixGood, want.Manifest))
mods.put("avahi", mustEncode(t, stale))
if err := ix.Recompute(ctx); err != nil {
t.Fatal(err)
}
got, _ := moduleOf(t, mods, "avahi")
if got.FP2 != want.FP2 || got.None2 != "" {
t.Errorf("after the start: fp2 %q none2 %q, want %q and no reason", got.FP2, got.None2, want.FP2)
}
// A manifest still unreadable keeps its value as it was.
ix2, mods2 := newFormIndexer(t, mod{"avahi", unreadableAvahi(t)})
before := string(mods2.vals["avahi"])
if err := ix2.Recompute(ctx); err != nil {
t.Fatal(err)
}
if after := string(mods2.vals["avahi"]); after != before {
t.Errorf("a manifest still unreadable changed the value:\n%s\n%s", before, after)
}
}
// good2 is the stored manifest value of the digest in the indexer that merged the readable manifest.
func good2(t *testing.T, ix *Indexer, digest string) []byte {
t.Helper()
v, _, found, err := ix.Manifests.Get(context.Background(), digest)
if err != nil || !found {
t.Fatalf("manifests.%s: %v %v", digest, found, err)
}
return v
}
modules/mesh-delivery-indexer/internal/index/testdata/candidate-written.json
Vendored
-31
View File
@@ -1,31 +0,0 @@
{
"source": {
"rev": 5,
"heads": {
"novox/mesh-catalog": "5555555555555555555555555555555555555555"
}
},
"candidate": {
"v": 1,
"group": "feat/x",
"base": 5,
"heads": "dfe9fb8f79e8",
"pulls": [
{
"repository": "novox/mesh-catalog",
"number": 301,
"head": "0123456789abcdef0123456789abcdef01234567",
"modules": [
"avahi"
],
"fps": {
"avahi": {
"fp2": "src2:e8d595aea1b13f1f5578ef1f245dab0d28a351c28079ecdcd2276c63a932d7be",
"manifest": "8a3856b6c0566d76f2556fc4f46c5251dc7935517bb6bd15809ded7e5546a0d8"
}
}
}
],
"fpset": "01c840ec276b"
}
}
modules/mesh-delivery-indexer/internal/index/toolchains.go
+22 -93
View File
@@ -112,9 +112,6 @@ type known struct {
languages map[string]string
bases []string
contexts bool
// held is the fingerprint the bucket holds for the module and the toolchains it was computed with (for a merge, the
// value before it): what stillComputed keeps while the builders agree on no toolchain.
held delivery.Module
}
// Recompute writes again every module whose new-form fingerprint, or reason for none, is no longer what it holds: from
@@ -159,41 +156,32 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
}
slices.Sort(keys)
held := map[string]delivery.Module{}
stale := map[string]bool{} // values an SDK that could not read their manifest wrote, judged again (issue 1923)
all := map[string]*known{}
for _, key := range keys {
cur, merged := overlay[key]
var before delivery.Module
value, _, found, err := ix.Modules.Get(ctx, key)
if err != nil {
return fmt.Errorf("modules.%s: %w", key, err)
}
if found {
if before, err = delivery.DecodeModule(value); err != nil && !merged {
ix.logf("modules.%s holds what is not a module, left: %v", key, err)
continue
cur, ok := overlay[key]
if !ok {
value, _, found, err := ix.Modules.Get(ctx, key)
if err != nil {
return fmt.Errorf("modules.%s: %w", key, err)
}
}
if !merged {
if !found {
continue
}
cur = before
if cur, err = delivery.DecodeModule(value); err != nil {
ix.logf("modules.%s holds what is not a module, left: %v", key, err)
continue
}
}
held[key] = cur
if cur.Manifest == "" || cur.Inputs2 == nil {
continue
}
raw, m, err := ix.manifestRaw(ctx, cur.Manifest)
m, err := ix.manifestOf(ctx, cur.Manifest)
if err != nil {
ix.logf("modules.%s: its manifest %s cannot be read, left: %v", key, cur.Manifest, err)
continue
}
if again, ok := ix.rejudged(ctx, m, cur, raw); ok {
cur, stale[key] = again, true
held[key] = cur
}
all[key] = &known{key: key, cur: cur, languages: languagesOf(m), bases: basesOf(m), contexts: hasContext(m), held: before}
all[key] = &known{key: key, cur: cur, languages: languagesOf(m), bases: basesOf(m), contexts: hasContext(m)}
}
tools := ix.agreed()
type result struct {
@@ -258,9 +246,6 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
r.none = strings.Join(missing, "; ")
case noToolchain != "":
r.none = noToolchain
if fp, ok := stillComputed(module, kept, k, toolchains); ok {
r = result{fp: fp, tcs: k.held.Toolchains}
}
default:
r.fp, r.none = delivery.SourceFingerprint2(kept.With(module, toolchains))
if r.fp != "" {
@@ -290,10 +275,10 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
finals[key] = final
continue
}
if !stale[key] && k.cur.FP2 == r.fp && k.cur.None2 == r.none && reflect.DeepEqual(k.cur.Toolchains, r.tcs) {
if k.cur.FP2 == r.fp && k.cur.None2 == r.none && reflect.DeepEqual(k.cur.Toolchains, r.tcs) {
continue
}
digest, inputs, paths := k.cur.Manifest, k.cur.Inputs2, k.cur.ArtifactPaths
digest, inputs := k.cur.Manifest, k.cur.Inputs2
err := write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
if !found {
return nil, false, nil
@@ -306,7 +291,7 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
return nil, false, nil
}
cur.FP2, cur.None2, cur.Toolchains = r.fp, r.none, r.tcs
cur.Inputs2, cur.ArtifactPaths = inputs, paths
cur.Inputs2 = inputs
data, err := delivery.EncodeModule(cur)
return data, false, err
})
@@ -319,12 +304,6 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
// writeOverlay writes the records of a merge, each once and only if it differs from what the bucket holds; finals has
// the ones that were computed, the others are written as the merge gave them.
//
// **A module the merge did not change is not written** (the operator, 2026-10-25: "only if a module has got a new version in
// commit x, should a new module version get created linked to commit x"). A value names the commit its fingerprint first
// appeared at: when the merge gives a module the value it holds but for the commit (and the source revision stamped on it),
// the held value stands, its commit unchanged, so a catalogue merge touching one module writes one key, not one per module of
// the repository, and the controller's take-in hears only that one. The repository's trunk head is the gate's source.main.
func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string]delivery.Module) error {
for _, key := range slices.Sorted(maps.Keys(overlay)) {
next := overlay[key]
@@ -333,7 +312,7 @@ func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string]
}
err := write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
if found {
if cur, err := delivery.DecodeModule(value); err == nil && sameButWhere(cur, next) {
if cur, err := delivery.DecodeModule(value); err == nil && reflect.DeepEqual(cur, withV(next)) {
return nil, false, nil
}
}
@@ -347,48 +326,24 @@ func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string]
return nil
}
// sameButWhere says the held value cur is next but for where it was computed (Commit and Src): the module is unchanged.
func sameButWhere(cur, next delivery.Module) bool {
next = withV(next)
next.Commit, next.Src = cur.Commit, cur.Src
return reflect.DeepEqual(cur, next)
}
// manifestRaw is manifests.<digest>: the manifest as stored and parsed by the running SDK.
func (ix *Indexer) manifestRaw(ctx context.Context, digest string) (string, catalogue.Manifest, error) {
// manifestOf is the parsed manifest manifests.<digest> holds.
func (ix *Indexer) manifestOf(ctx context.Context, digest string) (catalogue.Manifest, error) {
key, err := delivery.ManifestsKey(digest)
if err != nil {
return "", catalogue.Manifest{}, err
return catalogue.Manifest{}, err
}
value, _, found, err := ix.Manifests.Get(ctx, key)
if err != nil {
return "", catalogue.Manifest{}, err
return catalogue.Manifest{}, err
}
if !found {
return "", catalogue.Manifest{}, fmt.Errorf("manifests.%s is not there", key)
return catalogue.Manifest{}, fmt.Errorf("manifests.%s is not there", key)
}
kept, err := delivery.DecodeManifest(value)
if err != nil {
return "", catalogue.Manifest{}, err
return catalogue.Manifest{}, err
}
m, err := catalogue.ParseManifestForInputs([]byte(kept.JSON))
return kept.JSON, m, err
}
// rejudged is a value an SDK that could not read its manifest wrote, as the running SDK judges it: its kept inputs (the
// tree, the lock hashes by file name, the named paths' objects) taken through the manifest as a merge takes them, and the
// reason that gives. It answers false for any other value, and for one whose manifest the running SDK still cannot make
// inputs of (issue 1923).
func (ix *Indexer) rejudged(ctx context.Context, m catalogue.Manifest, cur delivery.Module, raw string) (delivery.Module, bool) {
if cur.FP2 != "" || cur.Inputs2 == nil || !strings.HasPrefix(cur.None2, None2Unreadable) {
return cur, false
}
in := inputsOfNamed(m.Module, raw, cur.Inputs2.Tree, cur.Inputs2.Locks, cur.Inputs2.Paths, ix.baseFP(ctx))
if in.Kept == nil {
return cur, false
}
cur.Inputs2, cur.None2, cur.ArtifactPaths = in.Kept, in.None2, in.ArtifactPaths
return cur, true
return catalogue.ParseManifestForInputs([]byte(kept.JSON))
}
// hasContext is whether an artifact of the manifest reads another repository's tree.
@@ -404,32 +359,6 @@ func hasContext(m catalogue.Manifest) bool {
return false
}
// stillComputed is the fingerprint a module already holds, kept while a language it is built with has no agreed toolchain
// (issue 1940, 2026-10-25 20:56: a build node assigned again brought a stale report into the agreement, every module's
// fingerprint was blanked to "builders disagree", and the controller's take-in refused modules whose fingerprints were
// computed and built). Only computing a new fingerprint needs the agreement: the held one is kept when its kept inputs,
// with the toolchains it was computed with in place of the languages agreed on none, give it again. Any other input that
// moved (a merge, a base, an agreed toolchain) gives another fingerprint, and the module has none until they agree.
func stillComputed(module string, kept delivery.KeptInputs2, k *known, agreed map[string]delivery.Toolchain2) (string, bool) {
if k.held.FP2 == "" {
return "", false
}
toolchains := map[string]delivery.Toolchain2{}
for artifact, language := range k.languages {
if tc, ok := agreed[artifact]; ok {
toolchains[artifact] = tc
continue
}
used, ok := k.held.Toolchains[language]
if !ok {
return "", false
}
toolchains[artifact] = delivery.Toolchain2{Language: language, Version: used.Version, Recipe: used.Recipe}
}
fp, none := delivery.SourceFingerprint2(kept.With(module, toolchains))
return fp, none == "" && fp == k.held.FP2
}
// toolchainsUsed are the toolchains a fingerprint was computed with, by language (an artifact's toolchain is its language's).
func toolchainsUsed(by map[string]delivery.Toolchain2) map[string]delivery.ToolchainUsed {
if len(by) == 0 {
modules/mesh-delivery-indexer/internal/index/toolchains_test.go
+7 -31
View File
@@ -123,16 +123,9 @@ func TestAgreedToolchainChangesAGoModulesReasonWithNoMerge(t *testing.T) { // 14
}
func TestNodesReportingDifferentVersionsLeaveTheReasonNamingBoth(t *testing.T) { // 1444/AC6
// Both reports are read before the module's first fingerprint is computed (a fingerprint computed before they
// disagree is kept: TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith).
ix, mods := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")})
for node, version := range map[string]string{"novox": "go1.27.1", "ace": "go1.26.0"} {
key, _ := delivery.ToolchainKey(node, "go")
ix.PutToolchain("mesh-build-go", key, toolchainValue(t, "go", version, "1"), false)
}
if err := ix.Recompute(context.Background()); err != nil {
t.Fatal(err)
}
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.26.0")
m, _ := moduleOf(t, mods, "avahi")
if !strings.Contains(m.None2, "builders disagree") || !strings.Contains(m.None2, "go1.27.1") || !strings.Contains(m.None2, "go1.26.0") {
t.Errorf("none2 = %q, want 'builders disagree' naming both versions", m.None2)
@@ -267,13 +260,11 @@ func TestAMergeKeepsAFingerprintTheModuleStillHas(t *testing.T) { // 1444 (revie
w := mods.writes
// the same catalogue merged again at a later commit: nothing moved
_, _, f := newFormIndexerUnmerged(t, builderModules(t)...)
f.Merge = otherMerge
if err := ix.Apply(ctx, f); err != nil {
t.Fatal(err)
}
// and the commit stays the one the fingerprint first appeared at (the operator, 2026-10-25)
if after, _ := moduleOf(t, mods, "mesh-tools"); after.FP2 != before.FP2 || after.Commit != before.Commit || mods.writes != w {
t.Errorf("fp2 = %q at %.8s (was %q at %.8s), writes %d more", after.FP2, after.Commit, before.FP2, before.Commit, mods.writes-w)
if after, _ := moduleOf(t, mods, "mesh-tools"); after.FP2 != before.FP2 || mods.writes != w {
t.Errorf("fp2 = %q (was %q), writes %d more", after.FP2, before.FP2, mods.writes-w)
}
}
@@ -297,24 +288,9 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
if n.FP2 == m.FP2 || n.Toolchains["go"].Version != "go1.27.2" {
t.Errorf("after the upgrade: fp2 %q (was %q) toolchains %+v, want the new version with the new fingerprint", n.FP2, m.FP2, n.Toolchains)
}
// Nodes that disagree keep the fingerprint already computed, with its toolchains (issue 1940, 2026-10-25 20:56: a
// stale report blanked every module and the controller's take-in refused modules already built); only a new
// fingerprint needs the agreement: a merge that moves the module's tree gives it none, naming the disagreement.
// Nodes that disagree leave no fingerprint and no toolchains.
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.26.0")
if d, _ := moduleOf(t, mods, "avahi"); d.FP2 != n.FP2 || !reflect.DeepEqual(d.Toolchains, n.Toolchains) || d.None2 != "" {
t.Errorf("disagreeing nodes: fp2 %q toolchains %+v none2 %q, want the fingerprint kept (%q) with %+v", d.FP2, d.Toolchains, d.None2, n.FP2, n.Toolchains)
}
_, _, f := newFormIndexerUnmerged(t, builderModules(t)...)
f.Trees["modules/avahi"] = "tree-avahi-moved"
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
if d, _ := moduleOf(t, mods, "avahi"); d.FP2 != "" || d.Toolchains != nil || !strings.HasPrefix(d.None2, "builders disagree") {
t.Errorf("a new tree while nodes disagree: fp2 %q toolchains %+v none2 %q, want no fingerprint and the disagreement", d.FP2, d.Toolchains, d.None2)
}
// They agree again: the new tree gets its fingerprint.
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.27.2")
if a, _ := moduleOf(t, mods, "avahi"); a.FP2 == "" || a.FP2 == n.FP2 {
t.Errorf("agreed again: fp2 %q (was %q), want a new fingerprint", a.FP2, n.FP2)
if d, _ := moduleOf(t, mods, "avahi"); d.FP2 != "" || d.Toolchains != nil {
t.Errorf("disagreeing nodes: fp2 %q toolchains %+v, want neither", d.FP2, d.Toolchains)
}
}
modules/mesh-delivery-indexer/manifest_test.go
+4 -35
View File
@@ -7,7 +7,6 @@ import (
"os"
"path/filepath"
"reflect"
"regexp"
"slices"
"strings"
"testing"
@@ -81,7 +80,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
if want := []state{{Name: "candidates", History: 10}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}}; !reflect.DeepEqual(m.State, want) {
t.Errorf("state = %+v, want %+v", m.State, want)
}
if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-controller.assignments", "mesh-controller.builds", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) {
if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-controller.builds", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) {
t.Errorf("reads = %v, want %v", m.Reads, want)
}
// "Reads FORGE" is a consumes of the indexer's stream: the broker grants only buckets by reads (mesh-control
@@ -95,10 +94,9 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
if m.SecretsOwner != "mesh-delivery-indexer" {
t.Errorf("secrets-owner = %q", m.SecretsOwner)
}
// Two invokes: the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths:
// ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a
// commit is in another (issue 1900).
if want := []string{"gitea.gitea_commit_trees", "gitea.gitea_contains"}; !reflect.DeepEqual(m.Invokes, want) {
// One invoke: the forge's trees with the named paths' objects, since the indexer is the one reader of a manifest's paths
// (ADR 0499 clauses 2 and 3, ADR 0495 clause 1, the lead's decision L8 of R4e).
if want := []string{"gitea.gitea_commit_trees"}; !reflect.DeepEqual(m.Invokes, want) {
t.Errorf("invokes = %v, want %v", m.Invokes, want)
}
// No emits, no streams, no bus, no seat, no tool (ADR 0437 §4).
@@ -171,32 +169,3 @@ func TestServesNoRequestSubjectAndHoldsNoTicker(t *testing.T) { // 1238/AC9 (ADR
t.Fatalf("read %d source files, want the program's", files)
}
}
// Every forge tool the program calls is granted in invokes, or the bus refuses the call (issue 1900).
func TestEveryGiteaToolTheCodeCallsIsInInvokes(t *testing.T) {
m, _ := load(t)
files, err := filepath.Glob("cmd/mesh-delivery-indexer/*.go")
if err != nil || len(files) == 0 {
t.Fatalf("no program files: %v", err)
}
re := regexp.MustCompile(`gitea\.gitea_[a-z_]+`)
seen := 0
for _, f := range files {
if strings.HasSuffix(f, "_test.go") {
continue
}
src, err := os.ReadFile(f)
if err != nil {
t.Fatal(err)
}
for _, tool := range re.FindAllString(string(src), -1) {
seen++
if !slices.Contains(m.Invokes, tool) {
t.Errorf("%s calls %s, which invokes does not grant", f, tool)
}
}
}
if seen < 2 {
t.Errorf("found %d tool names in the program, want at least the two it calls", seen)
}
}
modules/mesh-delivery-indexer/module.json
+1 -3
View File
@@ -16,14 +16,12 @@
"reads": [
"mesh-build-go.toolchains",
"mesh-build-node.toolchains",
"mesh-controller.assignments",
"mesh-controller.builds",
"mesh-delivery-checker.previews",
"mesh-delivery-gate.source"
],
"invokes": [
"gitea.gitea_commit_trees",
"gitea.gitea_contains"
"gitea.gitea_commit_trees"
],
"own-secrets": {
"broker": "${dir:state}/broker"
modules/mesh-delivery-indexer/reads-outside
-1
View File
@@ -1 +0,0 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
modules/mesh-delivery-teller/cmd/mesh-delivery-teller/assignerasks.go
-213
View File
@@ -1,213 +0,0 @@
package main
import (
"encoding/json"
"fmt"
"slices"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The assigner's asks reach the operator (issue 1906). The assigner writes its asks (the hold of a fingerprint, the restore
// of a data module, the backup before an apply and a move of a module of policy record) in its bucket asks; until now the
// teller only filled them from an answer given in the view, so an ask nobody opened the view for reached nobody: on
// 2026-10-25 seven modules stayed held on every machine and the operator was never asked. Now the teller raises each on the
// operator channel as it raises the gate's merge asks: as the asker mesh-delivery-teller, once per router id, through the same
// records, the same wait for room and the same warrants. The operator's answer, from the phone or from the view, is written
// under the ask's own key in the bucket answers, which the assigner reads (mesh-delivery-assigner pass.go answeredWith); none
// of these asks needs proof of who answered (mesh-sdk go/delivery AskNeedsProof).
// mostAssignerOpen is how many of the assigner's asks the teller keeps open at the router at once. The router keeps at most
// asks.MostOpen open asks of one asker, and the teller asks both the gate's merge asks and the assigner's as one asker, so a burst
// of holds (seven at once on 2026-10-25) would take all the room and a merge Approve could not be asked; one place is always
// left for a merge ask. An assigner ask beyond these waits in the wait for room and is asked when one of them ends.
const mostAssignerOpen = asks.MostOpen - 1
// assignerOpenRecord is the key of the router ids of the assigner's asks the teller asked.
const assignerOpenRecord = "assigner.open"
// assignerOpen is the router ids of the assigner's asks open now: those asked whose ask is neither answered nor ended. An id
// that ended is dropped from the record as it is read.
func (t *Teller) assignerOpen() ([]string, *stdio.StateEntry) {
e, err := t.Store.Get(assignerOpenRecord)
if err != nil || e == nil {
return nil, e
}
var ids []string
if err := json.Unmarshal(e.Value, &ids); err != nil {
t.logf("the record %s cannot be read: %v", assignerOpenRecord, err)
return nil, e
}
return slices.DeleteFunc(ids, func(id string) bool {
k, _, err := t.keptOf(id)
return err == nil && !open(k)
}), e
}
// assignerRoom says whether another of the assigner's asks may be asked now.
func (t *Teller) assignerRoom() bool {
ids, _ := t.assignerOpen()
return len(ids) < mostAssignerOpen
}
// assignerAsked keeps that the assigner's ask with this router id is asked.
func (t *Teller) assignerAsked(id string) {
ids, e := t.assignerOpen()
if !slices.Contains(ids, id) {
ids = append(ids, id)
}
var err error
if e == nil {
_, err = t.Store.Create(assignerOpenRecord, ids)
} else {
_, err = t.Store.Update(assignerOpenRecord, ids, e.Revision)
}
if err != nil {
t.logf("the record %s could not be kept: %v", assignerOpenRecord, err)
}
}
// waitForAssignerRoom puts an assigner ask in the wait for room, once.
func (t *Teller) waitForAssignerRoom(key, id string) {
ws, rec, err := t.waiting()
if err != nil {
t.logf("%v", err)
return
}
if slices.ContainsFunc(ws, func(x waitingAsk) bool { return x.ID == id }) {
return
}
if t.keepWaiting(append(ws, waitingAsk{Key: key, ID: id}), rec) == nil {
t.logf("%s: the assigner's ask %s waits: %d of the assigner's asks are open, the most the teller keeps open so a merge ask always has room", key, id, mostAssignerOpen)
}
}
// isAssignerKey says whether an ask key is one of the assigner's, not the gate's merge ask.
func isAssignerKey(key string) bool {
k, err := delivery.ParseAnyAskKey(key)
return err == nil && k.Kind != delivery.AskKindMerge
}
// assignerAsk reads the assigner's ask at key: found false when the key holds none.
func (t *Teller) assignerAsk(key string) (delivery.AssignerAsk, bool, error) {
if t.Assigner == nil {
return delivery.AssignerAsk{}, false, fmt.Errorf("the teller reads no assigner asks")
}
raw, found, err := t.Assigner.Get(key)
if err != nil || !found {
return delivery.AssignerAsk{}, false, err
}
a, err := delivery.DecodeAssignerAsk(key, raw)
if err != nil {
return delivery.AssignerAsk{}, false, fmt.Errorf("the assigner's key %s holds no ask: %w", key, err)
}
return a, true, nil
}
// raiseAssignerAsk asks the operator the assigner's ask at key, once per router id: an ask answered already (from the view or
// the phone) is not asked; an earlier asking still open is taken back first; an asking already made is only caught up with
// at the router, and one that ended is not asked again, for the router drops a second ask under an id it keeps.
func (t *Teller) raiseAssignerAsk(key string) {
a, found, err := t.assignerAsk(key)
if err != nil {
t.logf("%s: the ask is not raised: %v", key, err)
return
}
if !found {
return
}
id := a.Ask.ID
if raw, _, has, err := t.Answers.Get(delivery.AnswerKey(key)); err != nil {
t.logf("%s: the answers could not be read, so the ask is not raised: %v", key, err)
return
} else if has && answeredAsking(raw) == id {
return
}
prior, grec, err := t.seenAt(gateRecordKey(key))
if err != nil {
t.logf("%s: %v", key, err)
return
}
if grec != nil && prior.ID != id {
t.cancelOpen(key, prior.ID) // a new asking of the same key: the earlier one is taken back
}
k, _, err := t.keptOf(id)
if err != nil {
t.logf("%s: %v", key, err)
return
}
if k != nil {
if grec == nil || prior.ID != id {
_ = t.note(id, key, 0, grec)
}
if open(k) {
t.catchUp(id)
}
return
}
if err := t.note(id, key, 0, grec); err != nil {
t.logf("%s: the record of the ask %s could not be kept, so it is not raised: %v", key, id, err)
return
}
if !t.assignerRoom() {
t.waitForAssignerRoom(key, id)
return
}
if err := t.Asker.Ask(a.Ask); err != nil {
t.logf("%s: the ask %s was not asked: %v", key, id, err)
if kept, _, _ := t.keptOf(id); open(kept) {
_ = t.Store.Delete(keptKey(id)) // kept but not published: forgotten, so the next writing of the key asks it
}
return
}
t.assignerAsked(id)
t.logf("%s: the assigner's ask %s is asked of the operator (%s)", key, id, a.Ask.Headline)
}
// OnAssignerAskGone handles a key the assigner took away from its asks (a hold it lifted by itself): the open router ask of
// its latest asking is taken back, so the operator is not asked about what no longer holds.
func (t *Teller) OnAssignerAskGone(key string) {
t.mu.Lock()
defer t.mu.Unlock()
if !isAssignerKey(key) {
t.logf("the assigner's key %s was taken away and is no ask key of the assigner, and is left", key)
return
}
prior, grec, err := t.seenAt(gateRecordKey(key))
if err != nil || grec == nil {
if err != nil {
t.logf("%s: %v", key, err)
}
return
}
t.cancelOpen(key, prior.ID)
}
// answerAssigner is the act of a warrant for one of the assigner's asks: the ask must still stand at its key under the asking
// and the words the operator answered, and then the answer is written under the ask's key, once. Its options bind no act:
// the assigner reads the option chosen (asker.Client.Take has checked the option, its level and the ask's digest).
func (t *Teller) answerAssigner(key string, o asks.Option, w asks.Warrant) error {
a, found, err := t.assignerAsk(key)
if err != nil {
return err
}
if !found {
t.logf("%s: the ask %s was answered and the assigner has taken it away: the answer changes nothing", key, w.Ask)
return nil
}
if a.Ask.ID != w.Ask {
t.logf("%s: the ask %s was answered, and the assigner has asked again under %s: an answer to an earlier asking changes nothing", key, w.Ask, a.Ask.ID)
return nil
}
if a.Ask.Digest() != w.AskDigest {
return fmt.Errorf("the assigner's ask %s is no longer the ask the operator answered (its digest is %s, the warrant's %s)", key, a.Ask.Digest(), w.AskDigest)
}
body, err := delivery.EncodeAnswer(delivery.Answer{Option: o.ID, Warrant: warrantID(w), Words: w.Says()})
if err != nil {
return err
}
return t.writeAnswer(key, w.Ask, body, w.Says())
}
modules/mesh-delivery-teller/cmd/mesh-delivery-teller/assignerasks_test.go
-209
View File
@@ -1,209 +0,0 @@
package main
import (
"encoding/json"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// 1906/H1: given the assigner's hold ask in its bucket and no answer, when the teller hears the key, then it raises the ask on
// the operator channel as itself, unchanged and once (the key heard again, as a watch replays it, asks nothing more).
func TestAnAssignerAskIsRaisedOnTheOperatorChannelOnce(t *testing.T) {
b := newBed(t)
assigner, _ := withView(b)
key, value, ask := holdAsk(t)
assigner[key] = value
b.t.OnAssignerAsk(key)
b.t.OnAssignerAsk(key)
asked := b.askedOf(asks.AskSubject(delivery.TellerModule))
if len(asked) != 1 {
t.Fatalf("published %d asks, want one; log:\n%s", len(asked), b.logs.String())
}
var got asks.Ask
if err := json.Unmarshal(asked[0].body, &got); err != nil {
t.Fatal(err)
}
if got.ID != ask.ID || got.Digest() != ask.Digest() {
t.Errorf("the router's ask %+v differs from the assigner's", got)
}
}
// 1906/H1: without the view's bucket the assigner's asks are still raised.
func TestAnAssignerAskIsRaisedWithoutTheView(t *testing.T) {
b := newBed(t)
assigner := memGate{}
b.t.Assigner = assigner
key, value, _ := holdAsk(t)
assigner[key] = value
b.t.OnAssignerAsk(key)
if n := len(b.askedOf(asks.AskSubject(delivery.TellerModule))); n != 1 {
t.Fatalf("published %d asks, want one; log:\n%s", n, b.logs.String())
}
}
// 1906/H1: the operator's answer on the phone comes back by the same path as the view's: written under the ask's own key in
// the bucket answers, naming the asking, once; a second warrant writes nothing more and nothing is said as a fault.
func TestTheOperatorsAnswerToAnAssignerAskIsWrittenUnderItsKey(t *testing.T) {
b := newBed(t)
assigner, _ := withView(b)
key, value, ask := holdAsk(t)
assigner[key] = value
b.t.OnAssignerAsk(key)
w := ackFor(ask, "release")
b.deliver(t, w)
raw, ok := b.answers.written[delivery.AnswerKey(key)]
if !ok {
t.Fatalf("no answer under %s; log:\n%s", key, b.logs.String())
}
if a, err := delivery.DecodeAnswer(raw); err != nil || a.Option != "release" || answeredAsking(raw) != ask.ID {
t.Errorf("answer = %+v, %v", a, err)
}
b.deliver(t, w)
if b.answers.writes != 1 || len(b.conds) != 0 {
t.Errorf("%d writes, conditions %+v", b.answers.writes, b.conds)
}
}
// 1906/H1: an answer to an asking the assigner has since taken away or asked again changes nothing.
func TestAnAnswerToAnAssignerAskTakenAwayWritesNothing(t *testing.T) {
b := newBed(t)
assigner, _ := withView(b)
key, value, ask := holdAsk(t)
assigner[key] = value
b.t.OnAssignerAsk(key)
delete(assigner, key)
b.deliver(t, ackFor(ask, "release"))
if b.answers.writes != 0 {
t.Errorf("an answer to a withdrawn ask was written")
}
}
// 1906/H1, H2: an ask answered in the view is taken back from the phone, and an ask the assigner took away (a hold it lifted by
// itself) is taken back too; an ask answered before it is heard is never raised.
func TestAnAssignerAskAnsweredInTheViewOrTakenAwayIsTakenBack(t *testing.T) {
cancelled := func(b *bed, id string) bool {
for _, p := range b.askedOf(asks.CancelSubject(delivery.TellerModule)) {
if string(p.body) == `{"id":"`+id+`"}` {
return true
}
}
return false
}
b := newBed(t)
assigner, view := withView(b)
key, value, ask := holdAsk(t)
assigner[key] = value
b.t.OnAssignerAsk(key)
view[key+".view"] = viewAnswer(t, "release", "")
b.t.OnViewAnswer(key + ".view")
if !cancelled(b, ask.ID) {
t.Errorf("the ask answered in the view was not taken back; log:\n%s", b.logs.String())
}
b = newBed(t)
assigner, _ = withView(b)
assigner[key] = value
b.t.OnAssignerAsk(key)
delete(assigner, key)
b.t.OnAssignerAskGone(key)
if !cancelled(b, ask.ID) {
t.Errorf("the ask taken away was not taken back; log:\n%s", b.logs.String())
}
b = newBed(t)
assigner, view = withView(b)
assigner[key] = value
view[key+".view"] = viewAnswer(t, "keep", "")
b.t.OnAssignerAsk(key)
if n := len(b.askedOf(asks.AskSubject(delivery.TellerModule))); n != 0 {
t.Errorf("an ask answered in the view was raised %d times", n)
}
}
// 1906/H1: an assigner ask refused for want of room waits and is asked again when another of the teller's asks ends, as a merge
// ask does; a refusal for its words is said as a condition about a delivery ask, not a merge.
func TestAnAssignerAskWaitsForRoomAndARefusalIsSaidAsADeliveryAsk(t *testing.T) {
b := newBed(t)
assigner, _ := withView(b)
_, first := b.askOf(t, 1, 1)
b.askOf(t, 2, 2)
b.askOf(t, 3, 3)
key, value, ask := holdAsk(t)
assigner[key] = value
b.t.OnAssignerAsk(key)
b.refuse(t, ask.ID, roomRefusal)
b.deliver(t, warrantFor(first.Ask, "approve"))
got := b.askedIDs()
if len(got) != 5 || got[4] != ask.ID {
t.Fatalf("asks = %v, want the assigner's %s asked again", got, ask.ID)
}
b = newBed(t)
assigner, _ = withView(b)
assigner[key] = value
b.t.OnAssignerAsk(key)
b.refuse(t, ask.ID, "the ask has no headline")
if len(b.conds) != 1 || b.conds[0].Subject.Scope != "delivery" || strings.Contains(b.conds[0].Explanation, "merge") {
t.Errorf("conditions = %+v", b.conds)
}
}
// ackFor is the router's warrant for an option of level acknowledge, as the assigner's asks offer.
func ackFor(a asks.Ask, option string) asks.Warrant {
w := warrantFor(a, option)
w.Level, w.Label = asks.Acknowledge, option
return w
}
// holdAskOf is the assigner's hold ask of module plex at the n-th fingerprint, shaped as holdAsk.
func holdAskOf(t *testing.T, n int) (string, []byte, asks.Ask) {
t.Helper()
key, err := delivery.HoldAskKey("plex", "src2:"+strings.Repeat(string(rune('0'+n)), 64))
if err != nil {
t.Fatal(err)
}
_, _, ask := holdAsk(t)
ask.ID = delivery.AskID(key)
value, err := delivery.EncodeAssignerAsk(key, delivery.AssignerAsk{Ask: ask})
if err != nil {
t.Fatal(err)
}
return key, value, ask
}
// 1906/H5: given seven holds at once (as on 2026-10-25), when they are raised, then at most two are open at the router, a merge
// ask is still asked, and each hold answered lets the next waiting one be asked.
func TestABurstOfHoldsLeavesRoomForAMergeAsk(t *testing.T) {
b := newBed(t)
assigner, _ := withView(b)
var held []asks.Ask
for n := 0; n < 7; n++ {
key, value, ask := holdAskOf(t, n)
assigner[key] = value
b.t.OnAssignerAsk(key)
held = append(held, ask)
}
if got := b.askedIDs(); len(got) != mostAssignerOpen || got[0] != held[0].ID || got[1] != held[1].ID {
t.Fatalf("asked %v, want the first %d holds only; log:\n%s", got, mostAssignerOpen, b.logs.String())
}
_, merge := b.askOf(t, 1, 1)
if got := b.askedIDs(); len(got) != mostAssignerOpen+1 || got[mostAssignerOpen] != merge.Ask.ID {
t.Fatalf("asked %v, want the merge ask asked beside the holds", got)
}
if len(b.dropped) != 0 || len(b.conds) != 0 {
t.Errorf("router dropped %v, conditions %+v", b.dropped, b.conds)
}
b.deliver(t, ackFor(held[0], "release"))
got := b.askedIDs()
if len(got) != mostAssignerOpen+2 || got[len(got)-1] != held[2].ID {
t.Fatalf("asked %v after a hold was answered, want the third hold asked next; log:\n%s", got, b.logs.String())
}
// The merge ask answered frees its own place, and no hold takes it.
b.deliver(t, warrantFor(merge.Ask, "approve"))
if got := b.askedIDs(); len(got) != mostAssignerOpen+2 {
t.Errorf("asked %v after the merge ask was answered, want no further hold", got)
}
}
modules/mesh-delivery-teller/cmd/mesh-delivery-teller/integration_test.go
+2 -148
View File
@@ -524,9 +524,8 @@ func TestTellerFillsAnAskFromTheViewsAnswerOnce(t *testing.T) { // 1297/B10 AC
if a, err := delivery.DecodeAnswer(e.Value()); err != nil || a.Option != "release" || answeredAsking(e.Value()) != ask.ID {
t.Fatalf("answer %+v err %v", a, err)
}
// The view's answer written again with the same option changes nothing under the ask (another option, later, is the
// operator changing their mind: TestTellerTakesAChangedHoldAnswerFromTheView, issue 1906 note 23).
if _, err := viewKV.Put(b.ctx, key+".view", viewAnswer(t, "release", "")); err != nil {
// The view's answer written again changes nothing under the ask.
if _, err := viewKV.Put(b.ctx, key+".view", viewAnswer(t, "keep", "")); err != nil {
t.Fatal(err)
}
time.Sleep(time.Second)
@@ -546,148 +545,3 @@ func TestTellerFillsAnAskFromTheViewsAnswerOnce(t *testing.T) { // 1297/B10 AC
t.Error("the merge's ask was answered from a view key")
}
}
// TestTellerRaisesAnAssignerAskAndWritesTheOperatorsAnswer starts the program against the bus: the assigner's hold ask, put in
// its bucket, is on the operator channel's queue as the teller's ask; the router's warrant for "release" is written under the
// ask's key, where the assigner reads it; a second ask the assigner takes away (a hold it lifted) is cancelled at the router
// (1906/H1, H2 on the real bus).
func TestTellerRaisesAnAssignerAskAndWritesTheOperatorsAnswer(t *testing.T) {
now = time.Now().UTC()
b := startFull(t)
assigner := b.as(t, "n1.mesh-delivery-assigner")
broker := filepath.Join(t.TempDir(), "broker")
cred, _ := json.Marshal(credential{URL: b.srv.ClientURL(), Node: "n1", Module: "mesh-delivery-teller", User: "n1.mesh-delivery-teller", Password: "pw"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
env := map[string]string{"MESH_BROKER_FILE": broker}
logs := &syncBuffer{}
rn := startRunner(b.ctx, func(c context.Context) error { return run(c, func(k string) string { return env[k] }, logs) })
t.Cleanup(func() { rn.finish(t, 10*time.Second) })
key, value, ask := holdAsk(t)
askKV, err := assigner.KeyValue(b.ctx, assignerAsks)
if err != nil {
t.Fatal(err)
}
time.Sleep(500 * time.Millisecond) // the program hears warrants and watches
if _, err := askKV.Put(b.ctx, key, value); err != nil {
t.Fatal(err)
}
seat, err := b.js.Stream(b.ctx, "SEAT_OPERATOR_CHANNEL")
if err != nil {
t.Fatal(err)
}
var raw *jetstream.RawStreamMsg
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if raw, err = seat.GetLastMsgForSubject(b.ctx, seatAsk); err == nil {
break
}
}
if raw == nil {
t.Fatalf("the assigner's ask is not on the seat's queue: %v\n%s", err, logs.String())
}
var asked asks.Ask
if err := json.Unmarshal(raw.Data, &asked); err != nil || asked.ID != ask.ID || asked.Digest() != ask.Digest() {
t.Fatalf("the ask on the queue is %+v (%v), want the assigner's", asked, err)
}
w, _ := json.Marshal(ackFor(ask, "release"))
if err := b.ctl.Publish(seatDecided, w); err != nil {
t.Fatal(err)
}
answers, err := b.js.KeyValue(b.ctx, tellerAnswers)
if err != nil {
t.Fatal(err)
}
var e jetstream.KeyValueEntry
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err = answers.Get(b.ctx, delivery.AnswerKey(key)); err == nil {
break
}
}
if e == nil {
t.Fatalf("no answer under %s: %v\n%s", key, err, logs.String())
}
if a, err := delivery.DecodeAnswer(e.Value()); err != nil || a.Option != "release" || answeredAsking(e.Value()) != ask.ID {
t.Fatalf("answer %+v err %v", a, err)
}
key2, err := delivery.HoldAskKey("plex", "src2:"+strings.Repeat("3", 64))
if err != nil {
t.Fatal(err)
}
ask2 := ask
ask2.ID = delivery.AskID(key2)
value2, err := delivery.EncodeAssignerAsk(key2, delivery.AssignerAsk{Ask: ask2})
if err != nil {
t.Fatal(err)
}
if _, err := askKV.Put(b.ctx, key2, value2); err != nil {
t.Fatal(err)
}
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if strings.Contains(logs.String(), ask2.ID+" is asked") {
break
}
}
if err := askKV.Delete(b.ctx, key2); err != nil {
t.Fatal(err)
}
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if raw, err = seat.GetLastMsgForSubject(b.ctx, seatCancel); err == nil {
break
}
}
if err != nil || string(raw.Data) != `{"id":"`+ask2.ID+`"}` {
t.Errorf("the ask taken away was not cancelled: %v %v\n%s", raw, err, logs.String())
}
}
// TestTellerTakesAChangedHoldAnswerFromTheView starts the program against the bus: a hold ask answered "keep" earlier, and the
// view's answer "release" written now, replace the answer of that asking with "release" (1906/N23 on the real bus).
func TestTellerTakesAChangedHoldAnswerFromTheView(t *testing.T) {
now = time.Now().UTC()
b := startFull(t)
assigner, view := b.as(t, "n1.mesh-delivery-assigner"), b.as(t, "n1.mesh-delivery-view")
broker := filepath.Join(t.TempDir(), "broker")
cred, _ := json.Marshal(credential{URL: b.srv.ClientURL(), Node: "n1", Module: "mesh-delivery-teller", User: "n1.mesh-delivery-teller", Password: "pw"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
key, value, ask := holdAsk(t)
answers, err := b.js.KeyValue(b.ctx, tellerAnswers)
if err != nil {
t.Fatal(err)
}
kept, _ := delivery.EncodeAnswer(delivery.Answer{Option: "keep", Warrant: ask.ID + "@" + time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano), Words: "the operator chose keep"})
if _, err := answers.Put(b.ctx, delivery.AnswerKey(key), kept); err != nil {
t.Fatal(err)
}
askKV, err := assigner.KeyValue(b.ctx, assignerAsks)
if err != nil {
t.Fatal(err)
}
if _, err := askKV.Put(b.ctx, key, value); err != nil {
t.Fatal(err)
}
env := map[string]string{"MESH_BROKER_FILE": broker}
logs := &syncBuffer{}
rn := startRunner(b.ctx, func(c context.Context) error { return run(c, func(k string) string { return env[k] }, logs) })
t.Cleanup(func() { rn.finish(t, 10*time.Second) })
viewKV, err := view.KeyValue(b.ctx, viewAnswers)
if err != nil {
t.Fatal(err)
}
time.Sleep(500 * time.Millisecond) // the program is watching
if _, err := viewKV.Put(b.ctx, key+".view", viewAnswer(t, "release", "it was a flake")); err != nil {
t.Fatal(err)
}
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err := answers.Get(b.ctx, delivery.AnswerKey(key)); err == nil {
if a, err := delivery.DecodeAnswer(e.Value()); err == nil && a.Option == "release" && answeredAsking(e.Value()) == ask.ID {
return
}
}
}
t.Fatalf("the kept hold's answer was not changed to release\n%s", logs.String())
}
modules/mesh-delivery-teller/cmd/mesh-delivery-teller/kv.go
-13
View File
@@ -5,7 +5,6 @@ import (
"encoding/json"
"errors"
"fmt"
"time"
"github.com/nats-io/nats.go/jetstream"
@@ -108,15 +107,3 @@ func (g kvGate) Get(key string) ([]byte, bool, error) {
}
return e.Value(), true, nil
}
// GetAt is Get with the time the value was written.
func (g kvGate) GetAt(key string) ([]byte, time.Time, bool, error) {
e, err := g.kv.Get(g.ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, time.Time{}, false, nil
}
if err != nil {
return nil, time.Time{}, false, err
}
return e.Value(), e.Created(), true, nil
}
modules/mesh-delivery-teller/cmd/mesh-delivery-teller/main.go
+5 -11
View File
@@ -141,20 +141,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// there is said and leaves the teller without that path rather than without the gate's asks (a wait for it needs a timer).
failed := make(chan error, 2)
assignerKV, viewKV := optionalBucket(ctx, js, delivery.AssignerAsksBucket, logger), optionalBucket(ctx, js, delivery.ViewAnswersBucket, logger)
// The assigner's asks are raised on the operator channel whether or not the view's answers can be read (issue 1906).
if viewKV != nil {
t.View = kvGate{ctx: ctx, kv: viewKV}
}
if assignerKV != nil {
t.Assigner = kvGate{ctx: ctx, kv: assignerKV}
go func() {
failed <- watchAsks(ctx, assignerKV, func(key string, _ uint64, _ []byte) { t.OnAssignerAsk(key) }, t.OnAssignerAskGone, nil)
}()
}
if assignerKV != nil && viewKV != nil {
t.Assigner, t.View = kvGate{ctx: ctx, kv: assignerKV}, kvGate{ctx: ctx, kv: viewKV}
go func() {
failed <- watchAsks(ctx, viewKV, func(key string, _ uint64, _ []byte) { t.OnViewAnswer(key) }, func(string) {}, nil)
}()
go func() {
failed <- watchAsks(ctx, assignerKV, func(key string, _ uint64, _ []byte) { t.OnAssignerAsk(key) }, func(string) {}, nil)
}()
}
go func() {
if err := watchAsks(ctx, gate, t.OnAsk, t.OnAskGone, t.OnCurrent); err != nil {
@@ -176,7 +170,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
func optionalBucket(ctx context.Context, js jetstream.JetStream, name string, logger *log.Logger) jetstream.KeyValue {
kv, err := js.KeyValue(ctx, name)
if err != nil {
logger.Printf("the bucket %s cannot be read (%v): the asks and answers it holds are not carried until the teller is started again with it", name, err)
logger.Printf("the bucket %s cannot be read (%v): the view's answers fill no ask until the teller is started again with it", name, err)
return nil
}
return kv

Some files were not shown because too many files have changed in this diff Show More