novox/mesh-delivery
19
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sdk1: the indexer fingerprints by imported content; the checker asks the node-engines to compose #141

Merged
mesh-tools merged 40 commits from feat/sdk1 into main 2026-11-02 20:42:18 +00:00
Conversation 4 Commits 40 Files Changed 208
+15978 -3745
208 changed files with 15978 additions and 3745 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/asks.go
-161
View File
@@ -1,161 +0,0 @@
package main
import (
"context"
"sort"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"mesh-delivery-assigner/internal/assign"
)
// AssignerModule is the asker the assigner's compose asks name.
const AssignerModule = "mesh-delivery-assigner"
// composeAsks are the compose asks the assigner put on the Go build seat's check stream (novox/hq ADR 0482 clause 4,
// issues 1622 and 1623), by what each composes: one ask per key, put once and followed by a watch of its step key until
// it ends, which wakes a pass. No timer: an ask that waits in the stream waits (its backlog is the view's and the
// watchman's to show). A restart forgets them and asks again; the build module keeps a composer built for a commit, so a
// second ask builds nothing.
type composeAsks struct {
ctx context.Context
asker buildrun.Asker
wake func()
logf func(string, ...any)
now func() time.Time
mu sync.Mutex
asks map[string]*composeAsk
}
type composeAsk struct {
id string
cancel context.CancelFunc
ran *assign.Ran
}
func newComposeAsks(ctx context.Context, asker buildrun.Asker, wake func(), logf func(string, ...any)) *composeAsks {
return &composeAsks{ctx: ctx, asker: asker, wake: wake, logf: logf, now: time.Now, asks: map[string]*composeAsk{}}
}
// get is the end of the ask that composes key, or nil and its id while it runs. The first get of a key puts the ask.
// A job its layout refuses ends at once, failed with why: never silent.
func (a *composeAsks) get(key string, job delivery.ComposeJob) (*assign.Ran, string) {
a.mu.Lock()
defer a.mu.Unlock()
if k, ok := a.asks[key]; ok {
return k.ran, k.id
}
ask := delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: AssignerModule,
At: a.now(), Compose: &job}
ctx, cancel := context.WithCancel(a.ctx)
k := &composeAsk{id: ask.ID, cancel: cancel}
a.asks[key] = k
if err := a.asker.Put(ctx, ask); err != nil {
a.logf("the compose ask %s for %s could not be put: %v", ask.ID, key, err)
k.ran = &assign.Ran{State: delivery.StepFailed, Reason: "the compose ask could not be put on the Go build seat: " + err.Error()}
cancel()
return k.ran, k.id
}
go func() {
step, err := a.asker.Wait(ctx, ask)
if ctx.Err() != nil {
return
}
ran := assign.RanOf(step)
if err != nil {
ran = assign.Ran{State: delivery.StepFailed, Reason: "the compose ask's step could not be followed: " + err.Error()}
}
a.mu.Lock()
k.ran = &ran
a.mu.Unlock()
a.wake()
}()
return nil, k.id
}
// sweep forgets, and stops following, every ask whose key no pass asked for this time: a candidate gone or moved on.
func (a *composeAsks) sweep(seen map[string]bool) {
a.mu.Lock()
defer a.mu.Unlock()
for key, k := range a.asks {
if !seen[key] {
k.cancel()
delete(a.asks, key)
}
}
}
// composeAtHead is the composition of a group with a mesh-sdk member: the composer at its head and at each reader's
// mesh-sdk, asked of the Go build seat, and the verdict once every ask ended; until then the verdict waits, naming the
// asks. The keys it used are marked in seen.
func (p *passer) composeAtHead(ctx context.Context, m assign.Mesh, ch assign.Change, seen map[string]bool, began time.Time) delivery.CompositionVerdict {
waiting := func(words string) delivery.CompositionVerdict {
return delivery.CompositionVerdict{Group: ch.Group, Heads: ch.Heads, State: delivery.CompositionWaits, Words: words}
}
head, err := assign.HeadJob(m, ch)
if err != nil {
return delivery.CompositionVerdict{Group: ch.Group, Heads: ch.Heads, State: delivery.CompositionFail, SDK: "mesh-sdk " + ch.SDKHead,
Words: "the compose ask at the group's mesh-sdk head cannot be made: " + err.Error()}
}
var pending []string
key := "head." + ch.Group + "." + ch.Heads + "." + ch.SDKHead
seen[key] = true
headRan, id := p.asks.get(key, head)
if headRan == nil {
pending = append(pending, "mesh-sdk "+ch.SDKHead+" (ask "+id+")")
}
var readers []assign.Reader
var notJudged []string
for _, module := range sortedNames(assign.ReaderModules) {
repository := assign.ReaderModules[module]
var commits []string
member := false
if h := ch.Members[repository]; h != "" {
commits, member = []string{h}, true
} else {
commits = p.w.runningCommits(module)
}
if len(commits) == 0 {
notJudged = append(notJudged, "not judged whether "+module+" reads the group's manifests: the builder's log holds no done "+
"trunk build of it, and the group holds no member of "+repository)
continue
}
for _, commit := range commits {
job, err := assign.ReaderJob(ch, module, repository, commit)
if err != nil {
readers = append(readers, assign.Reader{Module: module, Commit: commit, Member: member,
Ran: assign.Ran{State: delivery.StepFailed, Reason: err.Error()}})
continue
}
rkey := "reader." + ch.Group + "." + ch.Heads + "." + module + "." + commit
seen[rkey] = true
ran, rid := p.asks.get(rkey, job)
if ran == nil {
pending = append(pending, module+" at "+commit+" (ask "+rid+")")
continue
}
readers = append(readers, assign.Reader{Module: module, Commit: commit, Member: member, Ran: *ran})
}
}
if len(pending) > 0 {
return waiting("composing on the Go build seat with the mesh-sdk of " + strings.Join(pending, ", "))
}
v := assign.FromComposer(m, ch, *headRan, readers, time.Since(began))
v.Warnings = append(v.Warnings, notJudged...)
return v
}
func sortedNames(m map[string]string) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compose_integration_test.go
-179
View File
@@ -1,179 +0,0 @@
//go:build integration
package main
import (
"encoding/json"
"errors"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/composer"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// holder is a test holder of the Go build seat: it takes every ask from the seat's check worker, as mesh-build-go does,
// and answers a compose ask by running mesh-sdk's composer over the request filled from the indexer's manifests (the
// linked composer standing for the one the build module builds at the asked commit), writing its step on its node's key.
type holder struct {
mu sync.Mutex
asks []delivery.StepAsk
}
func (h *holder) taken() []delivery.StepAsk {
h.mu.Lock()
defer h.mu.Unlock()
return append([]delivery.StepAsk(nil), h.asks...)
}
func (b *bed) hold(t *testing.T) *holder {
t.Helper()
seat, _ := delivery.AskStream(delivery.AskGo)
worker, _ := delivery.AskWorker(delivery.AskGo, delivery.AskCheck)
subject, _ := delivery.AskSubject(delivery.AskGo, delivery.AskCheck)
cons, err := b.js.CreateOrUpdateConsumer(b.ctx, seat, jetstream.ConsumerConfig{Durable: worker, FilterSubject: subject, AckPolicy: jetstream.AckExplicitPolicy})
if err != nil {
t.Fatal(err)
}
steps := b.buckets[delivery.StepsBucket(goBuildModule)]
h := &holder{}
cc, err := cons.Consume(func(m jetstream.Msg) {
a, err := delivery.DecodeStepAsk(m.Data())
if err != nil || a.Compose == nil {
t.Errorf("the seat was put an ask that is not a compose: %v %s", err, m.Data())
m.Ack()
return
}
h.mu.Lock()
h.asks = append(h.asks, a)
h.mu.Unlock()
req, err := composer.Decode(a.Compose.Request)
if err == nil {
err = composer.Fill(&req, func(digest string) ([]byte, error) {
key, _ := delivery.ManifestsKey(digest)
e, err := b.buckets[delivery.ManifestsBucket].Get(b.ctx, key)
if err != nil {
return nil, err
}
man, err := delivery.DecodeManifest(e.Value())
return []byte(man.JSON), err
})
}
step := delivery.Step{ID: a.ID, Tool: "compose", Node: "ace", State: delivery.StepPassed, Started: time.Now().UTC(), Ended: time.Now().UTC()}
if err != nil {
step.State, step.Reason = delivery.StepFailed, err.Error()
} else if v, err := composer.Run(req); err != nil {
step.State, step.Reason = delivery.StepFailed, err.Error()
} else {
vr, _ := json.Marshal(v)
step.Result, _ = json.Marshal(delivery.ComposeAnswer{Built: true, Key: "composer-0123456789abcdef", Verdict: vr})
}
raw, err := delivery.EncodeStep(step)
if err != nil {
t.Errorf("the step cannot be written: %v", err)
}
key, _ := delivery.AskStepKey("ace", a.ID)
if _, err := steps.Put(b.ctx, key, raw); err != nil {
t.Errorf("the step %s: %v", key, err)
}
m.Ack()
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(cc.Stop)
return h
}
// 1622/AC1-AC3 and 1623 on the bus: started whole, the assigner composes a group holding a mesh-sdk pull request by
// asking the Go build seat for the composer at the member's head, and for the composer at the mesh-sdk the running
// node-engine pins; it waits naming the asks while they run, writes the verdict from their answers naming the mesh-sdk the
// head's composer was built with, and asks nothing more on a later pass of the same heads.
func TestAGroupWithAnSDKMemberIsComposedByTheComposerAtItsHead(t *testing.T) {
b := startBed(t, "")
trunk := b.manifest(t, "web", `{"module":"web","version":"1"}`)
changed := b.manifest(t, "web", `{"module":"web","version":"2"}`)
mod, err := delivery.EncodeModule(delivery.Module{FP2: delivery.SourcePrefix2 + strings.Repeat("a", 64), Manifest: trunk, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
if err != nil {
t.Fatal(err)
}
b.putJSON(t, delivery.ModulesBucket, "web", mod)
b.place(t, "g14", "web")
running := strings.Repeat("3", 40)
b.place(t, "novox", "mesh-host")
b.built(t, "mesh-host", "novox/mesh-host", running)
sdkHead := strings.Repeat("4", 40)
heads := []delivery.Head{{Repository: "novox/mesh-sdk", Number: 80, Head: sdkHead}, {Repository: "novox/web", Number: 7, Head: strings.Repeat("2", 40)}}
hash, _ := delivery.HeadsHash(heads)
pulls := []delivery.CandidatePull{{Head: heads[0]}, {Head: heads[1],
Modules: []string{"web"}, Fps: map[string]delivery.CandidateModule{"web": {FP2: delivery.SourcePrefix2 + strings.Repeat("b", 64), Manifest: changed}}}}
cand, err := delivery.EncodeCandidate(delivery.Candidate{Group: "feat/x", HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
if err != nil {
t.Fatal(err)
}
ckey, _ := delivery.CandidatesKey("feat/x")
b.putJSON(t, delivery.CandidatesBucket, ckey, cand)
b.start(t)
key, _ := delivery.CompositionKey("feat/x", hash)
// No holder takes the asks yet: the composition waits, naming them.
waits := b.verdict(t, key, func(v delivery.CompositionVerdict) bool { return v.State == delivery.CompositionWaits })
if !strings.Contains(waits.Words, "mesh-sdk "+sdkHead) || !strings.Contains(waits.Words, "mesh-host at "+running) {
t.Fatalf("the waiting composition says %q, want the head's and the running node-engine's asks", waits.Words)
}
h := b.hold(t)
v := b.verdict(t, key, func(v delivery.CompositionVerdict) bool { return v.State != delivery.CompositionWaits })
if v.State != delivery.CompositionPass || len(v.Machines) != 1 || v.Machines[0] != "g14" || v.SDK == "" {
t.Fatalf("composition = %+v, want a pass on g14", v)
}
asks := h.taken()
if len(asks) != 2 {
t.Fatalf("the seat was asked %d composes, want two (the head, the running node-engine)", len(asks))
}
byRepo := map[string]delivery.StepAsk{}
for _, a := range asks {
byRepo[a.Compose.Fetch["repository"].(string)] = a
if a.Asker != AssignerModule || a.Kind != delivery.AskCheck || a.Language != delivery.AskGo {
t.Errorf("ask = %+v", a)
}
}
if a := byRepo["novox/mesh-sdk"]; a.Compose == nil || a.Compose.Fetch["commit"] != sdkHead || a.Compose.Dir != "go" {
t.Errorf("the head's ask = %+v", a.Compose)
}
if a := byRepo["novox/mesh-host"]; a.Compose == nil || a.Compose.Fetch["commit"] != running || a.Compose.Dir != delivery.RootPath {
t.Errorf("the running node-engine's ask = %+v", a.Compose)
}
// A later pass of the same heads (woken by any change) asks nothing more.
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
b.putJSON(t, delivery.PresenceBucket, "ace", on)
time.Sleep(500 * time.Millisecond)
if n := len(h.taken()); n != 2 {
t.Errorf("a later pass of the same heads asked again: %d asks", n)
}
}
// verdict waits for the composition under key to satisfy ok, and answers it.
func (b *bed) verdict(t *testing.T, key string, ok func(delivery.CompositionVerdict) bool) delivery.CompositionVerdict {
t.Helper()
var last delivery.CompositionVerdict
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
e, err := b.buckets[delivery.AssignerCompositionsBucket].Get(b.ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
continue
}
if err != nil {
t.Fatal(err)
}
if last, err = delivery.DecodeCompositionVerdict(e.Value()); err != nil {
t.Fatal(err)
}
if ok(last) {
return last
}
}
t.Fatalf("no composition under %s as wanted after 10 s; last %+v", key, last)
return last
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/composed.go
+73
View File
@@ -0,0 +1,73 @@
package main
import (
"sort"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// Every node's published compositions, read only to compare (novox/hq ADR 0593 §8): each node's engine writes its own
// `composed-<node>` (delivery.ComposedRead, one bucket per node), and `mesh-wants.upgrade` with compare names, for every
// node, the composition its own have runs beside the newest it published of each module and of the node, so a person sees
// where they part. The assigner composes nothing and never writes these.
// publishedRow is one composition a node published: the module ("node" for its own), the fingerprint, the digest, or why
// it was refused or cannot be read.
type publishedRow struct {
Key string `json:"key"`
Module string `json:"module"`
FP string `json:"fp,omitempty"`
Digest string `json:"digest,omitempty"`
Refused string `json:"refused,omitempty"`
Unread string `json:"unread,omitempty"`
}
// published is, by node, every composition it published, sorted by key, from every key of the per-node state (a key
// names its node first): each value decoded and digested here, as the node-engine digests it (delivery.ComposedDigestOf).
// A value or key that does not read is named with why, never left out.
func published(values map[string][]byte) map[string][]publishedRow {
byNode := map[string]map[string][]byte{}
for key, raw := range values {
node, _, _ := strings.Cut(key, ".")
if byNode[node] == nil {
byNode[node] = map[string][]byte{}
}
byNode[node][key] = raw
}
out := map[string][]publishedRow{}
for node, keys := range byNode {
var rows []publishedRow
for key, raw := range keys {
row := publishedRow{Key: key}
_, module, fp, isNode, err := delivery.ParseComposedKey(key)
switch {
case err != nil:
row.Unread = err.Error()
case isNode:
row.Module = "node"
default:
row.Module, row.FP = module, fp
}
if err == nil {
c, derr := delivery.DecodeComposed(raw)
switch {
case derr != nil:
row.Unread = derr.Error()
case c.Refused != "":
row.Refused = c.Refused
default:
if d, err := delivery.ComposedDigestOf(c); err != nil {
row.Unread = err.Error()
} else {
row.Digest = d
}
}
}
rows = append(rows, row)
}
sort.Slice(rows, func(i, j int) bool { return rows[i].Key < rows[j].Key })
out[node] = rows
}
return out
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go
+7 -75
View File
@@ -24,10 +24,10 @@ import (
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/compose/judge"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-assigner/internal/judge"
"mesh-delivery-assigner/internal/assign"
)
@@ -290,19 +290,6 @@ func (b *bed) place(t *testing.T, node, module string) {
}
}
// built appends the end of a done trunk build of module at commit to the builder's log, as the builder does.
func (b *bed) built(t *testing.T, module, repository, commit string) {
t.Helper()
raw, err := delivery.EncodeBuildLogEntry(delivery.BuildLogEntry{Event: delivery.BuildEndedEvent, Module: module, FP: delivery.SourcePrefix2 + strings.Repeat("c", 64),
Repository: repository, Directory: delivery.RootPath, Commit: commit, Node: "novox", At: time.Now().UTC(), Outcome: delivery.BuildDone})
if err != nil {
t.Fatal(err)
}
if _, err := b.js.Publish(b.ctx, buscompose.LogSubject(delivery.BuilderModule, delivery.BuilderLog)+"."+module, raw); err != nil {
t.Fatal(err)
}
}
// compose says the node's own composition is the digest d (a letter repeated), as its have names it (`<node>.node`): a
// placed node composes its own (ADR 0571 §6), and the controller's composition is read no more. It answers the digest.
func (b *bed) compose(t *testing.T, node string, d byte) string {
@@ -333,62 +320,6 @@ func (b *bed) manifest(t *testing.T, module, body string) string {
return digest
}
// 1297/AC1: against the real bus, a fixture change touching one module on one machine is composed and answered within
// seconds under the checker's key of the check, naming the machine composed and the mesh-sdk it composed with.
func TestAChangeIsComposedOnTheBus(t *testing.T) {
b := startBed(t, "")
trunk := b.manifest(t, "web", `{"module":"web","version":"1"}`)
changed := b.manifest(t, "web", `{"module":"web","version":"2"}`)
mod, err := delivery.EncodeModule(delivery.Module{FP2: delivery.SourcePrefix2 + strings.Repeat("a", 64), Manifest: trunk, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
if err != nil {
t.Fatal(err)
}
b.putJSON(t, delivery.ModulesBucket, "web", mod)
b.place(t, "g14", "web")
b.place(t, "ace", "other")
heads := []delivery.Head{{Repository: "novox/web", Number: 7, Head: strings.Repeat("2", 40)}}
hash, _ := delivery.HeadsHash(heads)
// 1927/AC1: the candidate carries the fingerprint set the SDK computes, as the indexer writes it.
pulls := []delivery.CandidatePull{{Head: heads[0],
Modules: []string{"web"}, Fps: map[string]delivery.CandidateModule{"web": {FP2: delivery.SourcePrefix2 + strings.Repeat("b", 64), Manifest: changed}}}}
cand, err := delivery.EncodeCandidate(delivery.Candidate{Group: "feat/x", HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
if err != nil {
t.Fatal(err)
}
ckey, _ := delivery.CandidatesKey("feat/x")
b.putJSON(t, delivery.CandidatesBucket, ckey, cand)
began := time.Now()
b.start(t)
key, _ := delivery.CompositionKey("feat/x", hash)
var v delivery.CompositionVerdict
for deadline := time.Now().Add(10 * time.Second); ; time.Sleep(50 * time.Millisecond) {
if time.Now().After(deadline) {
t.Fatalf("no composition under %s after 10 s", key)
}
e, err := b.buckets[delivery.AssignerCompositionsBucket].Get(b.ctx, key)
if err != nil {
continue
}
if v, err = delivery.DecodeCompositionVerdict(e.Value()); err != nil {
t.Fatal(err)
}
break
}
if v.State != delivery.CompositionPass || len(v.Machines) != 1 || v.Machines[0] != "g14" || v.SDK == "" {
t.Fatalf("composition = %+v, want a pass on g14 alone naming its mesh-sdk", v)
}
if took := time.Since(began); took > 5*time.Second {
t.Errorf("the composition took %s", took)
}
// 1622/AC3: a group without a mesh-sdk member builds no composer: no ask is put on the Go build seat.
seat, _ := delivery.AskStream(delivery.AskGo)
if st, err := b.js.Stream(b.ctx, seat); err != nil {
t.Fatal(err)
} else if info, err := st.Info(b.ctx); err != nil || info.State.Msgs != 0 {
t.Errorf("a group without a mesh-sdk member put %d asks on the Go build seat (%v)", info.State.Msgs, err)
}
}
// get waits for a key of one of the bed's buckets and answers its value.
func (b *bed) get(t *testing.T, bucket, key string) []byte {
t.Helper()
@@ -1121,16 +1052,17 @@ func TestAProposalIsAnsweredApprovedAndWritten(t *testing.T) {
t.Error("a declined proposal wrote its key")
}
// ADR 0567 §4: a node that does not answer fails the verdict once its pulse key's time to live ends, heard on the bus
// with no timer of the assigner's, naming it.
// ADR 0593 §8: a node that does not answer is named as giving no answer once its pulse key's time to live ends, heard on
// the bus with no timer of the assigner's, and the proposal is decided on the answers it has: asked, not failed.
q, err = verbs.doPropose(map[string]any{"why": "again", "changes": `[{"op":"take-off","node":"laptop","module":"web"}]`}, "op")
if err != nil {
t.Fatal(err)
}
silent := q.(proposed).Proposal
failed := proposalNow(silent.ID, "failed for a silent node", func(p delivery.PlacementProposal) bool { return p.State == delivery.ProposalFailed })
if failed.Verdict == nil || !strings.Contains(strings.Join(failed.Verdict.Fails, "; "), "laptop (offline) did not answer") {
t.Errorf("the silent node's verdict = %+v", failed.Verdict)
decided := proposalNow(silent.ID, "decided without its silent nodes", func(p delivery.PlacementProposal) bool { return p.State == delivery.ProposalAsked })
if decided.Verdict == nil || decided.Verdict.State != delivery.VerdictPass ||
!strings.Contains(strings.Join(decided.Verdict.Notes, "; "), "no answer from") || !strings.Contains(strings.Join(decided.Verdict.Notes, "; "), "laptop (offline)") {
t.Errorf("the silent node's verdict = %+v", decided.Verdict)
}
// 774/S59 (ADR 0587 §7): a joining node is written on the enrolment's warrant with no node to answer and no ask raised,
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go
+9 -51
View File
@@ -15,19 +15,15 @@ import (
"os"
"os/signal"
"slices"
"sync/atomic"
"syscall"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/announce"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/provisions"
@@ -56,31 +52,23 @@ func main() {
// readBuckets are the buckets of other parts the assigner reads, each named in its manifest's reads (the manifest test
// holds the two lists equal), so the controller's broker grants exactly these.
var readBuckets = []string{
buscompose.ResolvedBucket,
delivery.PartGrantsBucket,
delivery.AnswersBucket,
delivery.BuilderBuildsBucket,
delivery.CandidatesBucket,
delivery.ManifestsBucket,
delivery.ModulesBucket,
delivery.PresenceBucket,
delivery.HaveBucket,
delivery.StepsBucket(goBuildModule),
provisions.ProvisionsBucket,
keptBucket,
}
// readLogs are the logs of other parts the assigner reads, each named in its manifest's reads: the builder's log, whose
// newest done trunk build of a module names the commit a reader of the manifests runs at (novox/hq ADR 0571 clause 1).
var readLogs = []string{delivery.BuildLogRead}
// goBuildModule is the Go build module, which takes the assigner's compose asks on its seat's check stream and writes
// their steps in its steps state (novox/hq ADR 0482 clause 4).
const goBuildModule = "mesh-build-go"
// readFamilies are the states kept one bucket per node the assigner reads, each named in its manifest's reads: every
// node's compositions, which mesh-wants.upgrade compares (novox/hq ADR 0593 §8).
var readFamilies = []string{delivery.ComposedRead}
// ownBuckets are the assigner's own states, as its manifest declares them: the seat mesh-placement's three among them
// (ADR 0567 §1), named after the assigner as every module state is.
var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerCompositionsBucket, delivery.AssignerDifferencesBucket,
var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerDifferencesBucket,
delivery.AssignerPutsBucket, delivery.AssignerWantsBucket, delivery.PlacementBucket, delivery.PlacementProposalsBucket,
delivery.PlacementCandidatesBucket, delivery.PlacementPulsesBucket, assign.FindingsBucket, assign.PoliciesBucket, delivery.SeatHoldersBucket}
@@ -90,7 +78,7 @@ var seatBuckets = []string{delivery.PlacementBucket, delivery.PlacementProposals
delivery.PlacementPulsesBucket}
// keptStates is what the assigner keeps and who may read it (ADR 0495 §2-§3): its wants by the watchman and the view
// (and the node agent from release 6), its differences by the view, its compositions by the checker, its asks by the view
// (and the node agent from release 6), its differences by the view, its asks by the view
// and the teller, its puts by whoever follows one (ADR 0546), and the seat mesh-placement's placement, proposals and
// candidates by the node agents, the grants part, mesh-config and the view (ADR 0567 §9), and its placement by
// the builder, the checker and the indexer, through mesh-sdk's toolchain gate (issue 774, ADR 0549 §3).
@@ -100,7 +88,6 @@ var keptStates = []announce.State{
{Name: delivery.PlacementCandidatesState, Readable: true},
{Name: delivery.PlacementPulsesState, Readable: true},
{Name: "asks", Readable: true},
{Name: "compositions", Readable: true},
{Name: "differences", Readable: true},
{Name: "puts", Readable: true},
{Name: "wants", Readable: true},
@@ -135,11 +122,6 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
continue
}
if name == keptBucket && err != nil {
// mesh-config's held values (ADR 0565 §5): without them a composition lays none, said in the journal.
logf("the bucket %s cannot be opened (%v): no held value is laid on a composition until it is bound", name, err)
continue
}
if name == provisions.ProvisionsBucket && err != nil {
// The seat mesh-provisioner's state, with its own delivery (ADR 0570 §4): bound once the bus says it was made;
// until then no want waits for a provision, as before.
@@ -175,7 +157,7 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
// states and those new with a delivery of their own, bound once made, and the two that choose the grants' source.
func startsWithout(name string) bool {
switch name {
case delivery.SeatHoldersBucket, keptBucket, provisions.ProvisionsBucket, assign.PoliciesBucket, assign.FindingsBucket,
case delivery.SeatHoldersBucket, provisions.ProvisionsBucket, assign.PoliciesBucket, assign.FindingsBucket,
delivery.AssignerPutsBucket:
return true
}
@@ -211,10 +193,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
thisNode = account.Node
logger.Printf("started on the account %s, reading %d buckets", account.User, len(kvs))
findings := findingsOf(ctx, kvs, logger.Printf)
buildLog := func(handle func(stdio.LogEntry) error) error {
return said.ReadLog(delivery.BuilderModule, delivery.BuilderLog).Watch("", stdio.LogFromAll, handle)
}
return follow(ctx, js, conn, kvs, findings.Say(sayVia(ctx, js)), findings, emitting{said}, buildLog, logger)
return follow(ctx, js, conn, kvs, findings.Say(sayVia(ctx, js)), findings, emitting{said}, logger)
}
// findingsOf is where the assigner's said conditions go as findings (issue 1952 WM6): its findings bucket, which the
@@ -246,11 +225,9 @@ func (e emitting) Emit(eventType string, body any) error { return e.Conn.Emit(ev
// watched are the buckets whose every value a pass reads; the manifests are read by digest when a pass needs one, and
// watched only for keys, so a manifest that arrives wakes a composition that waits for it.
var watched = []string{
buscompose.ResolvedBucket,
delivery.PartGrantsBucket,
delivery.AnswersBucket,
delivery.BuilderBuildsBucket,
delivery.CandidatesBucket,
delivery.ModulesBucket,
delivery.PlacementBucket,
delivery.PlacementProposalsBucket,
@@ -259,18 +236,16 @@ var watched = []string{
delivery.PresenceBucket,
delivery.HaveBucket,
delivery.AssignerAsksBucket,
delivery.AssignerCompositionsBucket,
delivery.AssignerDifferencesBucket,
delivery.AssignerWantsBucket,
assign.PoliciesBucket,
provisions.ProvisionsBucket,
keptBucket,
delivery.SeatHoldersBucket,
}
// follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after
// each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends.
func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, kvs buckets, say func(asks.Condition) error, findings assign.Findings, bus caller, buildLog logWatch, logger *log.Logger) error {
func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, kvs buckets, say func(asks.Condition) error, findings assign.Findings, bus caller, logger *log.Logger) error {
w := newWorld()
dirty := make(chan struct{}, 1)
wake := func() {
@@ -279,22 +254,6 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
default:
}
}
// The builder's log (ADR 0571 clause 1): every entry it keeps before the first pass, then each new one as it is appended;
// a module's newer done trunk build makes a pass, as a composition waiting on its commit is judged then. Where it cannot be
// read, a composition names no running reader of the manifests until the assigner starts again, and it is said.
if buildLog != nil {
var first atomic.Bool
if err := buildLog(func(le stdio.LogEntry) error {
if w.putBuilt(le.Value) && first.Load() {
wake()
}
return nil
}); err != nil {
logger.Printf("%s is not read (%v): no composition is judged against a running reader of the manifests until the assigner starts again",
delivery.BuildLogRead, err)
}
first.Store(true)
}
errs := make(chan error, len(watched)+2)
settled := make(chan struct{}, len(watched)+2)
watch := func(name string, opts ...jetstream.WatchOpt) error {
@@ -402,7 +361,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
}
}
// And the two that choose where grants are read from, with the same delivery as the grants part that writes one.
for _, name := range []string{delivery.PartGrantsBucket, assign.PoliciesBucket, provisions.ProvisionsBucket, keptBucket, delivery.SeatHoldersBucket} {
for _, name := range []string{delivery.PartGrantsBucket, assign.PoliciesBucket, provisions.ProvisionsBucket, delivery.SeatHoldersBucket} {
if kvs[name] == nil && js != nil {
name := name
process.AwaitBucket(ctx, js, nc, name, "until then the bucket is not read",
@@ -427,7 +386,6 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
}
}
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, findings: findings, said: map[string]bool{}, answeredAt: map[string]uint64{}}
p.asks = newComposeAsks(ctx, buildrun.Asker{Publisher: buildrun.JetStreamAsks{JS: js}, Waiter: buildrun.JetStreamAsks{JS: js}}, wake, logger.Printf)
if bus != nil && kvs[delivery.AssignerPutsBucket] != nil {
p.acts = newPutActs(bus, wake, logger.Printf)
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go
+5 -5
View File
@@ -40,14 +40,14 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
}
var reads []string
for _, r := range m.Reads {
if slices.Contains(readLogs, r) {
continue // a log, read through its own reader (ADR 0571 clause 1), not a bucket
if slices.Contains(readFamilies, r) {
continue // a state kept one bucket per node, read by its prefix (ADR 0593 §6), not one bucket
}
reads = append(reads, strings.Replace(r, ".", "_", 1))
}
for _, l := range readLogs {
for _, l := range readFamilies {
if !slices.Contains(m.Reads, l) {
t.Errorf("the program reads the log %s, the manifest does not name it", l)
t.Errorf("the program reads the per-node state %s, the manifest does not name it", l)
}
}
var states []string
@@ -113,7 +113,7 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
}
kvs[delivery.AssignerPutsBucket].(*watchKV).closed = true
kvs[delivery.AssignerWantsBucket].(*watchKV).wrote = wrote
go follow(ctx, nil, nil, kvs, func(asks.Condition) error { return nil }, assign.Findings{}, nil, nil, log.New(io.Discard, "", 0))
go follow(ctx, nil, nil, kvs, func(asks.Condition) error { return nil }, assign.Findings{}, nil, log.New(io.Discard, "", 0))
select {
case key := <-wrote:
if key != "want.ace.node" {
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go
-54
View File
@@ -14,7 +14,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provisions"
"git.novox.be/novox/mesh-sdk/go/sdkversion"
"mesh-delivery-assigner/internal/assign"
)
@@ -40,7 +39,6 @@ type passer struct {
unplaced map[string]bool
// answeredAt is, by the id of a backup ask, how many values the have had when its answer was written into the want.
answeredAt map[string]uint64
asks *composeAsks
// now is the clock; nil is time.Now.
now func() time.Time
// neither is, by node and module, each module on neither list whose condition this process raised (issue 1945). Nil until the
@@ -81,7 +79,6 @@ func (p *passer) clock() time.Time {
func (p *passer) pass(ctx context.Context) {
p.placementSeat(ctx)
p.compositions(ctx)
p.wants(ctx)
p.puts(ctx)
p.nodeWants(ctx)
@@ -478,57 +475,6 @@ func (p *passer) sayNeither(now, stale map[string]neitherOne, left map[string]st
}
}
// compositions writes the composition check's verdict of every group the indexer holds a candidate for, under the
// checker's key of that check: waits, with why, while the indexer holds what it composes from only in part.
func (p *passer) compositions(ctx context.Context) {
if p.sdk == "" {
p.sdk = sdkversion.Version()
}
written := p.w.read(delivery.AssignerCompositionsBucket)
trunk := p.w.trunk()
var mesh *assign.Mesh
seen := map[string]bool{}
if p.asks != nil {
defer p.asks.sweep(seen)
}
for key, value := range p.w.read(delivery.CandidatesBucket) {
c, err := delivery.DecodeCandidate(value)
if err != nil {
p.logf("the candidate %s cannot be read, so it is not composed: %v", key, err)
continue
}
out, err := delivery.CompositionKey(c.Group, c.HeadsHash)
if err != nil {
p.logf("the candidate %s names no composition key: %v", key, err)
continue
}
began := time.Now()
ch, waits := p.w.change(ctx, kvGetter(p.kvs[delivery.ManifestsBucket]), c, trunk)
var v delivery.CompositionVerdict
if waits != "" {
v = delivery.CompositionVerdict{Group: c.Group, Heads: c.HeadsHash, State: delivery.CompositionWaits, Words: waits}
} else {
if mesh == nil {
m := p.w.mesh(ctx, kvGetter(p.kvs[delivery.ManifestsBucket]))
mesh = &m
}
if ch.SDKHead != "" && p.asks != nil {
// A group with a mesh-sdk member is composed by the composer built at its head (ADR 0482 clauses 3 to 5).
v = p.composeAtHead(ctx, *mesh, ch, seen, began)
} else {
v = assign.Compose(*mesh, ch, p.sdk, time.Since(began))
}
}
if before, err := delivery.DecodeCompositionVerdict(written[out]); err == nil {
before.Took, before.V = v.Took, 0
if delivery.SameCompositionVerdict(before, v) {
continue
}
}
p.put(ctx, delivery.AssignerCompositionsBucket, out, func() ([]byte, error) { return delivery.EncodeCompositionVerdict(v) })
}
}
// put writes one key of the assigner's own bucket; a write that fails is said and made again at the next pass.
func (p *passer) put(ctx context.Context, bucket, key string, encode func() ([]byte, error)) {
value, err := encode()
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go
+1 -18
View File
@@ -791,35 +791,18 @@ func TestARecordAskOfAHeldBackNodeSurvivesPassesAndItsYesMovesTheWant(t *testing
}
// 774/S26/AC1 (ADR 0549 §5, ADR 0567 §8): the seat's placement alone names where modules run: no controller state is read, so a
// module in an assignment only is no module of its node; before any placement is read nothing is placed and the check composes
// no input; placed, the check composes over the input the nodes compose from (frombus, 1126, ADR 0547 §2).
// module in an assignment only is no module of its node, and before any placement is read nothing is placed.
func aPlacementAloneNamesWhereModulesRun(t *testing.T) {
p := newPasser(buckets{}, time.Date(2026, 10, 26, 18, 0, 0, 0, time.UTC))
p.w.put("mesh-controller_assignments", "g14.b", []byte(`{"node":"g14","module":"b","build":"b-1"}`)) // read no more
if got := p.w.assigned(); len(got) != 0 {
t.Fatalf("before any placement: assigned %v", got)
}
none := func(context.Context, string) ([]byte, error) { return nil, errNotThere }
if m := p.w.mesh(context.Background(), none); m.Input != nil || m.InputUnread != "" {
t.Errorf("before any placement the check composes over %+v (%q)", m.Input, m.InputUnread)
}
placeOn(t, p.w, "g14", "c", "a")
placeOn(t, p.w, "n2", "d")
if got := p.w.assigned(); !slices.Equal(got["g14"], []string{"a", "c"}) || !slices.Equal(got["n2"], []string{"d"}) || len(got) != 2 {
t.Errorf("the placement alone: assigned %v", got)
}
if m := p.w.mesh(context.Background(), none); m.Input != nil || !strings.Contains(m.InputUnread, "g14") {
t.Errorf("a placed node with no record: input %+v, unread %q", m.Input, m.InputUnread)
}
placeOn(t, p.w, "g14", "a", "b", "c")
rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "g14", TunnelKey: "k"}})
p.w.put(delivery.HaveBucket, "g14.node", rec)
rec2, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "n2", TunnelKey: "k2"}})
p.w.put(delivery.HaveBucket, "n2.node", rec2)
m := p.w.mesh(context.Background(), none)
if m.Input == nil || len(m.Input.Nodes) != 2 || !slices.Equal(m.Input.Nodes[0].Assigned, []string{"a", "b", "c"}) || m.Input.Nodes[0].Place.Key != "k" {
t.Errorf("the check's input from the bus: %+v (%q)", m.Input, m.InputUnread)
}
}
// 1126 (ADR 0547 §2 to §4): a node the catalogue places composes its own: its wants name the build alone and its own want
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go
+12 -49
View File
@@ -16,14 +16,12 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/compose"
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
"git.novox.be/novox/mesh-sdk/go/compose/judge"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/handacts"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/judge"
)
// The seat mesh-placement's holder in the assigner's process (novox/hq ADR 0567 §2-§8): each pass follows every open
@@ -354,67 +352,35 @@ func moved(prop delivery.PlacementProposal, revs map[string]uint64) bool {
return false
}
// intended is, by proposal and the revisions it was laid over, what Intended computed, kept while this process runs.
// intendedOf is, by proposal and the revisions it was laid over, what Intended computed, kept while this process runs.
func (p *passer) intendedOf(prop delivery.PlacementProposal, current map[string]delivery.Placement, cands map[string]delivery.PlacementCandidate) map[string]map[string]bool {
key := fmt.Sprint(prop.ID, prop.Base)
if got, ok := p.intended[key]; ok {
return got
}
got, _, _ := p.compute(prop, current, cands)
got, _ := p.compute(prop, current, cands)
p.intended[key] = got
return got
}
// compute composes every node with the current placement and with the proposal laid over it (judge.Intended).
func (p *passer) compute(prop delivery.PlacementProposal, current map[string]delivery.Placement, cands map[string]delivery.PlacementCandidate) (map[string]map[string]bool, []string, []string) {
// compute is what the proposal changes on each node by its placement, and the nodes that answer it: every node with a
// placement or a record (judge.Intended, ADR 0593 §8). The holder composes nothing: each node says what its compositions
// change in its answer.
func (p *passer) compute(prop delivery.PlacementProposal, current map[string]delivery.Placement, cands map[string]delivery.PlacementCandidate) (map[string]map[string]bool, []string) {
laid := map[string]*delivery.Placement{}
for _, node := range prop.Nodes() {
if c, ok := cands[node]; ok && !c.Reached {
laid[node] = c.Placement
}
}
get := kvGetter(p.kvs[delivery.ManifestsBucket])
r := worldReader{w: p.w, manifests: get}
// A read that fails is said, never dropped silently (novox/hq ADR 0578 §3); a refused manifest no longer fails it, and
// its module is held, named in the input's Refused.
var now, with *compose.Input
if in, err := frombus.Input(context.Background(), r, frombus.Mesh{}, frombus.Scope{Every: true}); err == nil {
now = &in
} else if p.logf != nil {
p.logf("proposal %s: the current composition input cannot be read, so what it changes is judged without it: %v", prop.ID, err)
}
if in, err := frombus.Input(context.Background(), r, frombus.Mesh{}, frombus.Scope{Every: true, Placements: laid}); err == nil {
with = &in
} else if p.logf != nil {
p.logf("proposal %s: the composition input with the proposal cannot be read, so what it changes is judged without it: %v", prop.ID, err)
}
if now != nil && len(now.Refused) > 0 && p.logf != nil {
var held []string
for _, m := range slices.Sorted(maps.Keys(now.Refused)) {
held = append(held, m+" ("+now.Refused[m]+")")
}
p.logf("proposal %s: held as refused by the indexer's judge: %s", prop.ID, strings.Join(held, "; "))
}
nodes := map[string]bool{}
for n := range current {
nodes[n] = true
}
if records, err := r.Records(context.Background()); err == nil {
for n := range records {
nodes[n] = true
}
for n := range p.w.records() {
nodes[n] = true
}
names := make([]string, 0, len(nodes))
for n := range nodes {
names = append(names, n)
}
fps := map[string]string{}
for name, m := range p.w.trunk() {
if m.FP2 != "" {
fps[name] = m.FP2
}
}
return judge.Intended(now, with, current, laid, names, fps)
return judge.Intended(current, laid, slices.Sorted(maps.Keys(nodes)))
}
// mustAnswer finds the nodes the proposal must hear from (§3), writes a reached candidate for each whose key it does not
@@ -425,7 +391,7 @@ func (p *passer) mustAnswer(ctx context.Context, prop delivery.PlacementProposal
return // the verbs write the candidates before the proposal: wait to hear them
}
}
intended, must, notes := p.compute(prop, current, cands)
intended, must := p.compute(prop, current, cands)
p.intended[fmt.Sprint(prop.ID, prop.Base)] = intended
for _, node := range must {
if _, ok := cands[node]; ok {
@@ -447,9 +413,6 @@ func (p *passer) mustAnswer(ctx context.Context, prop delivery.PlacementProposal
}
}
prop.Must = must
if len(notes) > 0 {
prop.Words = strings.TrimPrefix(prop.Words+"; "+strings.Join(notes, "; "), "; ")
}
p.writeProposal(ctx, prop, rev)
}
@@ -483,7 +446,7 @@ func (p *passer) lays(ctx context.Context) (func(string) bool, func(string) bool
held[name] = true
}
}
records, _ := worldReader{w: p.w}.Records(ctx)
records := p.w.records()
return func(m string) bool { return held[m] }, func(n string) bool { _, ok := records[n]; return ok }
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go
+13 -1
View File
@@ -367,9 +367,16 @@ func TestTheSeatsVerbs(t *testing.T) {
func wantsVerbs(t *testing.T) {
fpOld, fpNew := delivery.SourcePrefix2+strings.Repeat("1", 64), delivery.SourcePrefix2+strings.Repeat("2", 64)
st := memStore{}
for _, state := range []string{modulesState, manifestsState, policiesState, haveState, putsState, "findings"} {
for _, state := range []string{modulesState, manifestsState, policiesState, haveState, putsState, "findings", delivery.ComposedRead} {
st[state] = &memKV{values: map[string][]byte{}}
}
// ADR 0593 §6, §8: what each node published, every node's bucket as one state: anchor's own composition and its web at
// a fingerprint, and laptop's web refused there.
webKey, _ := delivery.ComposedKey("anchor", "web", fpNew)
refusedKey, _ := delivery.ComposedKey("laptop", "web", fpNew)
st[delivery.ComposedRead].values["anchor.node"], _ = delivery.EncodeComposed(delivery.Composed{Node: "anchor"})
st[delivery.ComposedRead].values[webKey], _ = delivery.EncodeComposed(delivery.Composed{Node: "anchor", Module: "web", FP: fpNew})
st[delivery.ComposedRead].values[refusedKey], _ = delivery.EncodeComposed(delivery.Composed{Node: "laptop", Module: "web", FP: fpNew, Refused: "a required value is unset"})
manifest := func(name, extra string) {
raw := `{"module":"` + name + `","version":"1"` + extra + `}`
digest := delivery.ManifestDigest([]byte(raw))
@@ -409,6 +416,11 @@ func wantsVerbs(t *testing.T) {
got["composed"].(map[string]string)["anchor"] != "sha256:"+strings.Repeat("c", 64) || len(got["composed"].(map[string]string)) != 1 {
t.Errorf("compare = %+v", got)
}
pub, _ := out.(map[string]any)["published"].(map[string][]publishedRow)
if a, l := pub["anchor"], pub["laptop"]; len(a) != 2 || a[0].Module != "node" || a[1].Module != "web" || a[1].FP != fpNew ||
!strings.HasPrefix(a[1].Digest, "sha256:") || len(l) != 1 || l[0].Refused != "a required value is unset" || l[0].Digest != "" {
t.Errorf("published = %+v (unread %v)", pub, out.(map[string]any)["published-unread"])
}
// Read by watch (issue 1953 L5f): the same answer, each watched state started once, and no get per key of the
// large states (the modules, their manifests, the composed record).
watches := map[string]int{}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/wants_tools.go
+7 -1
View File
@@ -20,7 +20,7 @@ import (
// The seat mesh-wants' verbs (novox/hq ADR 0565 §7, §8, §9, §15; issue 1953 D4-D6), served through the tool runner beside the
// put tools and mesh-placement's verbs. `upgrade` reads and sets a person's upgrade policy, kept in the state `policies`
// (each set recorded as a hand act and said as policy-changed), and with compare lists the digest of the composition each node's
// own have names (the controller writes no policy any more, so no module's policy differs); `bus` says what a bus upgrade would do. The assigner's process reads
// own have names beside every composition each node published (ADR 0593 §8, composed.go); `bus` says what a bus upgrade would do. The assigner's process reads
// `policies` when it judges a move and writes the finding policy-split while a split stands.
// The states the verbs read, as the runtime names them.
@@ -158,6 +158,12 @@ func (s *wantsTools) doUpgrade(args map[string]any, by string) (any, error) {
}
}
out := map[string]any{"composed": composed}
// And what every node published (ADR 0593 §8), read only to compare.
if values, err := s.values(delivery.ComposedRead); err != nil {
out["published-unread"] = err.Error()
} else {
out["published"] = published(values)
}
// The assigner's findings, its said conditions as the condition keeper reads them (§15).
if found, err := s.values(health.FindingsState); err != nil {
out["findings-unread"] = err.Error()
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go
+5 -226
View File
@@ -3,20 +3,14 @@ package main
import (
"context"
"errors"
"fmt"
"slices"
"sort"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/compose"
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-assigner/internal/assign"
@@ -249,63 +243,11 @@ func (w *world) trunk() map[string]delivery.Module {
return out
}
// mesh is the mesh as the trunk holds it, for the composition check.
func (w *world) mesh(ctx context.Context, manifests getter) assign.Mesh {
m := assign.Mesh{Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{}, Digests: map[string]string{}}
m.Assigned = w.assigned()
for name, mod := range w.trunk() {
if mod.Manifest == "" {
continue
}
m.Digests[name] = mod.Manifest
// The indexer's verdict on it, carried in its value (ADR 0578 §3, §5): a refused module is held and named as one
// that cannot be read, and its verdict goes into the composer's input.
if v := mod.VerdictOf(name); v != nil {
if refused, _ := delivery.TakeManifestVerdict(v); refused {
if m.Verdicts == nil {
m.Verdicts = map[string]delivery.ManifestVerdict{}
}
m.Verdicts[name] = *v
m.Unreadable[name] = v.Why()
continue
}
}
p := w.manifest(ctx, manifests, mod.Manifest)
if p.err != "" {
m.Unreadable[name] = p.err
continue
}
m.Manifests[name] = p.manifest
}
// Once the catalogue places the nodes, the check composes over the input they compose from (ADR 0547 §2, ADR 0549
// §3): one composer's input for the check and the nodes. An input that cannot be read is said, and the nodes and their
// assignments alone are composed, as before any placement.
if _, seeded := w.placed(); seeded {
in, err := frombus.Input(ctx, worldReader{w: w, manifests: manifests}, frombus.Mesh{}, frombus.Scope{Every: true})
if err != nil {
m.InputUnread = err.Error()
} else {
m.Input = &in
}
}
return m
}
// worldReader is what the assigner watched as frombus reads the bus: the seat's placement, the indexer's modules, each node's
// record in its own have key, the manifests by digest; the check reads no config value, own secret or credential (the
// composition it asks resolves, and stands in for what it does not carry).
type worldReader struct {
w *world
manifests getter
}
func (r worldReader) Placements(context.Context) (map[string]delivery.Placement, error) {
return r.w.placements(), nil
}
func (r worldReader) Records(context.Context) (map[string]delivery.NodeRecord, error) {
// records is each node's record in its own have key (`<node>.node`): the nodes a placement proposal is answered by,
// with those placement names (ADR 0593 §8).
func (w *world) records() map[string]delivery.NodeRecord {
out := map[string]delivery.NodeRecord{}
for key, value := range r.w.read(delivery.HaveBucket) {
for key, value := range w.read(delivery.HaveBucket) {
node, _, isNode, err := delivery.ParseHaveKey(key)
if err != nil || !isNode {
continue
@@ -314,136 +256,7 @@ func (r worldReader) Records(context.Context) (map[string]delivery.NodeRecord, e
out[node] = *h.Record
}
}
return out, nil
}
func (r worldReader) Modules(context.Context) (map[string]delivery.Module, error) {
return r.w.trunk(), nil
}
func (r worldReader) Manifest(ctx context.Context, digest string) ([]byte, error) {
key, err := delivery.ManifestsKey(digest)
if err != nil {
return nil, err
}
raw, err := r.manifests(ctx, key)
if err != nil {
return nil, err
}
m, err := delivery.DecodeManifest(raw)
if err != nil {
return nil, err
}
return []byte(m.JSON), nil
}
// Values is one module's config values on one node as mesh-config resolved them, each a value its source lays
// (catalogue.LaidValue): the holder composes every node a proposal touches with them (ADR 0567 §9).
func (r worldReader) Values(_ context.Context, node, module string) (map[string]any, error) {
var out map[string]any
prefix := node + "." + module + "."
for key, raw := range r.w.read(buscompose.ResolvedBucket) {
name, ok := strings.CutPrefix(key, prefix)
if !ok || strings.Contains(name, ".") {
continue
}
if v, laid := catalogue.LaidValue(raw); laid {
if out == nil {
out = map[string]any{}
}
out[name] = v
}
}
return out, nil
}
func (worldReader) OwnSecrets(context.Context, string) ([]compose.OwnSecretRow, error) {
return nil, nil
}
func (worldReader) Credentials(context.Context, string, []compose.CredentialKey, []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error) {
return nil, nil, nil
}
// Build and Archive are read only for a node composing itself (frombus.Scope without Every): the check of every node
// composes from the manifests the indexer read, and reads no build and no archive.
func (worldReader) Build(context.Context, string, string) (*delivery.BuilderBuild, error) {
return nil, nil
}
func (worldReader) Archive(_ context.Context, url, _ string) ([]byte, error) {
return nil, fmt.Errorf("the composition check reads no archive (%s)", url)
}
func (r worldReader) Served(ctx context.Context, node, module string, keys []string) (map[string]any, error) {
all, err := r.Values(ctx, node, module)
var out map[string]any
for _, k := range keys {
if v, ok := all[k]; ok {
if out == nil {
out = map[string]any{}
}
out[k] = v
}
}
return out, err
}
// change is a candidate's change for the composition check, or why the composition waits for it.
func (w *world) change(ctx context.Context, manifests getter, c delivery.Candidate, trunk map[string]delivery.Module) (assign.Change, string) {
ch := assign.Change{Group: c.Group, Heads: c.HeadsHash, Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{},
Twice: map[string][]string{}, Digests: map[string]string{}, Members: map[string]string{}}
heads := map[string]string{}
for _, p := range c.Pulls {
heads[fmt.Sprintf("%s#%d", p.Repository, p.Number)] = p.Head.Head
}
ch.Held = assign.HeldOf(w.read(keptBucket), heads)
definedIn := map[string]map[string]bool{}
for _, p := range c.Pulls {
if p.Repository == assign.SDKRepository {
ch.SDKHead = p.Head.Head
}
ch.Members[p.Repository] = p.Head.Head
for _, name := range p.Modules {
fp, ok := p.Fps[name]
if !ok || fp.Manifest == "" {
if ok && fp.None2 == noModuleWords {
ch.Removed = append(ch.Removed, name)
continue
}
return ch, fmt.Sprintf("the indexer holds no manifest of %s for %s#%d yet", name, p.Repository, p.Number)
}
if fp.FP2 != "" && fp.FP2 != trunk[name].FP2 {
ch.Rebuilt++
}
got := w.manifest(ctx, manifests, fp.Manifest)
if strings.HasPrefix(got.err, "the indexer holds no manifest") {
return ch, got.err
}
ch.Digests[name] = fp.Manifest
if got.err != "" {
ch.Unreadable[name] = got.err
continue
}
ch.Manifests[got.module] = got.manifest
if definedIn[got.module] == nil {
definedIn[got.module] = map[string]bool{}
}
definedIn[got.module][p.Repository] = true
if t, ok := trunk[got.module]; ok && t.Repository != "" && t.Repository != p.Repository {
definedIn[got.module][t.Repository] = true
}
}
}
for name, repos := range definedIn {
if len(repos) > 1 {
for r := range repos {
ch.Twice[name] = append(ch.Twice[name], r)
}
sort.Strings(ch.Twice[name])
}
}
return ch, ""
return out
}
// online is, by node, whether the watchman holds the node online.
@@ -594,40 +407,6 @@ func (w *world) changed(bucket, key string) uint64 {
return w.changes[bucket+"|"+key]
}
// logWatch follows the builder's log: handle each entry it keeps, then each new one (process.LogReader.Watch).
type logWatch func(handle func(stdio.LogEntry) error) error
// putBuilt takes one entry of the builder's log in: a done trunk build's end (no ref) naming a full commit, newer than the
// module's last, becomes the module's running commit. It answers whether the commit changed.
func (w *world) putBuilt(raw []byte) bool {
e, err := delivery.DecodeBuildLogEntry(raw)
if err != nil || e.Event != delivery.BuildEndedEvent || e.Outcome != delivery.BuildDone || e.Ref != "" || len(e.Commit) != 40 {
return false
}
w.mu.Lock()
defer w.mu.Unlock()
have, ok := w.trunkBuilt[e.Module]
if ok && !e.At.After(have.At) {
return false
}
w.trunkBuilt[e.Module] = e
return !ok || have.Commit != e.Commit
}
// runningCommits is the commit of module's newest done trunk build in the builder's log (ADR 0571 clause 1): the commit a
// reader of the manifests runs at (novox/hq ADR 0482 clause 5, issue 1623); none when the log holds no such build.
func (w *world) runningCommits(module string) []string {
w.mu.Lock()
defer w.mu.Unlock()
if e, ok := w.trunkBuilt[module]; ok {
return []string{e.Commit}
}
return nil
}
// keptBucket is mesh-config's `kept`: the approved held values of open pull requests (ADR 0565 §5).
var keptBucket = "mesh-config_" + delivery.MeshConfigKeptState
// ownComposed is a placed node's own composition, as the digest its own have names (`<node>.node` of the haves, Composed):
// what rule 7 reads. A node whose have names none is not composed yet.
func (w *world) ownComposed(node string) (delivery.ComposedDigest, bool) {
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go
-73
View File
@@ -2,15 +2,12 @@ package main
import (
"context"
"os"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-assigner/internal/assign"
)
// fakeManifests is the indexer's manifests bucket in memory.
@@ -45,76 +42,6 @@ func pull(repo string, number int, modules map[string]delivery.CandidateModule)
return p
}
// 1297/B3b: a candidate's change reads each module's manifest by its digest, counts the modules rebuilt, takes a
// module with no manifest in the preview as removed, a mesh-sdk member's head, and a module the trunk holds from
// another repository as defined twice.
func TestTheChangeOfACandidate(t *testing.T) {
theWorldCountsTheValuesAKeyTakes(t)
f := fakeManifests{}
web := f.add(t, "web", `{"module":"web","version":"2"}`)
mail := f.add(t, "mail", `{"module":"mail","version":"1"}`)
trunk := map[string]delivery.Module{"web": {FP2: fp('a'), Repository: "novox/web"}, "mail": {FP2: fp('c'), Repository: "novox/mail"}}
c := delivery.Candidate{Group: "g", HeadsHash: "0123456789ab", Pulls: []delivery.CandidatePull{
pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {FP2: fp('b'), Manifest: web}, "old": {None2: noModuleWords}}),
pull("novox/other", 2, map[string]delivery.CandidateModule{"mail": {FP2: fp('c'), Manifest: mail}}),
pull(assign.SDKRepository, 3, nil),
}}
ch, waits := newWorld().change(context.Background(), f.get, c, trunk)
if waits != "" {
t.Fatalf("the change waits: %s", waits)
}
if ch.Manifests["web"].Version != "2" || ch.Rebuilt != 1 || !slices.Equal(ch.Removed, []string{"old"}) || ch.SDKHead != strings.Repeat("3", 40) {
t.Errorf("change = %+v", ch)
}
if !slices.Equal(ch.Twice["mail"], []string{"novox/mail", "novox/other"}) || len(ch.Twice) != 1 {
t.Errorf("twice = %v, want mail in novox/mail and novox/other", ch.Twice)
}
}
// 1297/B3b: a candidate whose module has no manifest digest yet, or whose manifest the indexer does not hold yet, waits,
// saying for what; a manifest that does not decode is the change's, not a wait (one with a field the assigner does not
// know is decoded with it set aside, novox/hq ADR 0578 §6).
func TestAChangeWaitsForItsManifests(t *testing.T) {
f := fakeManifests{}
bad := f.add(t, "web", `{"module":"web","version":"2","requires":"x"}`)
for _, c := range []delivery.CandidatePull{
pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {None2: "no preview yet"}}),
pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {FP2: fp('b'), Manifest: strings.Repeat("9", 64)}}),
} {
_, waits := newWorld().change(context.Background(), f.get, delivery.Candidate{Group: "g", HeadsHash: "0123456789ab", Pulls: []delivery.CandidatePull{c}}, nil)
if waits == "" {
t.Errorf("a change without its manifest %+v does not wait", c.Fps)
}
}
ch, waits := newWorld().change(context.Background(), f.get, delivery.Candidate{Group: "g", HeadsHash: "0123456789ab",
Pulls: []delivery.CandidatePull{pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {FP2: fp('b'), Manifest: bad}})}}, nil)
if waits != "" || ch.Unreadable["web"] == "" {
t.Errorf("an unreadable manifest: waits %q, change %+v", waits, ch)
}
}
// A manifest of the change whose config value falls back to a predefined key (novox/hq ADR 0348 §7) is read, not
// unreadable for the catalogue's predefined.json the assigner never holds: the merge gate judges the fallback in the
// catalogue, and the indexer reads it the same way (catalogue.ParseManifestForInputs, issue 1751). A group whose
// mesh-catalog member touched pacman and xterm failed its composition on this (group check 18df81380db3). The manifest is
// xterm's module.json from mesh-catalog at 835bf22b.
func TestAManifestFallingBackToAPredefinedKeyIsReadable(t *testing.T) {
raw, err := os.ReadFile("testdata/catalogue-835bf22b-xterm.module.json")
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(raw), `"predefined"`) {
t.Fatal("the fixture names no predefined key")
}
f := fakeManifests{}
xterm := f.add(t, "xterm", string(raw))
ch, waits := newWorld().change(context.Background(), f.get, delivery.Candidate{Group: "g", HeadsHash: "0123456789ab",
Pulls: []delivery.CandidatePull{pull("novox/mesh-catalog", 1, map[string]delivery.CandidateModule{"xterm": {FP2: fp('b'), Manifest: xterm}})}}, nil)
if waits != "" || ch.Unreadable["xterm"] != "" || ch.Manifests["xterm"].Module != "xterm" {
t.Errorf("xterm: waits %q, unreadable %q, want read", waits, ch.Unreadable["xterm"])
}
}
// Review round 1 (items 3 and 4): the world counts the values a key takes, so a have written again after an answer is seen.
func theWorldCountsTheValuesAKeyTakes(t *testing.T) {
w := newWorld()
modules/mesh-delivery-assigner/go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
modules/mesh-delivery-assigner/go.sum
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
modules/mesh-delivery-assigner/internal/assign/compose.go
-368
View File
@@ -1,368 +0,0 @@
// Package assign is the assigner's judgement, as functions over what it read from the bus: the composition check of a
// change (novox/hq ADR 0446 clauses 1 to 5, ADR 0528 clause 5, issue 1297), the six rules of what each node should
// run (ADR 0429 §6, ADR 0528 clause 3) and the comparison with what the controller sends (ADR 0528 clause 4). It reads
// and writes nothing itself: the program hands it what it watched and writes what it answers.
package assign
import (
"encoding/json"
"fmt"
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/compose"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// SDKRepository is the repository of mesh-sdk: a group with a member there changes the composition itself.
const SDKRepository = "novox/mesh-sdk"
// NotJudged names, in a composition's warnings, the failures of ADR 0446 clause 1 the assigner cannot judge from the
// bus, because what they need is only in the controller's facts snapshot, which is not on the bus yet (issue 1461;
// the design limit of R4g, issue 1297): said on every verdict, never passed silently.
const NotJudged = "not judged until the facts snapshot is on the bus (issue 1461): a consumer left out of a grant, a " +
"credential left bound elsewhere, a stored setting the changed definition cannot keep, and a new module the " +
"node-engine's validator would refuse"
// Mesh is the mesh as the trunk holds it: every module's manifest by name, the manifests that cannot be read with
// why, and each node's open assignments.
type Mesh struct {
Manifests map[string]catalogue.Manifest
Unreadable map[string]string
Assigned map[string][]string
// Digests is every trunk module's manifest digest in the indexer's manifests bucket, readable or not: what a
// composer at another mesh-sdk reads (issue 1622).
Digests map[string]string
// Input is the rest of the mesh as the nodes compose from it, filled by mesh-sdk's frombus from the catalogue's
// placement and the nodes' records (novox/hq ADR 0547 §2, ADR 0549 §3, issue 1126), so the check and the nodes share
// one composer's input; nil before any placement is read, when the nodes and their assignments alone are composed.
Input *compose.Input
// InputUnread says why that input could not be read from the bus, when it could not: the check then composes the
// nodes and their assignments alone, and says so in its warnings.
InputUnread string
// Verdicts are the indexer's refusals of trunk manifests, by module, as their values carry them (novox/hq ADR 0578
// §3): each such module is in Unreadable too, and the composer's input carries the verdict (§5).
Verdicts map[string]delivery.ManifestVerdict
}
// Change is a group's change as the indexer's candidate holds it: the manifest of every module it moves or adds, those
// it cannot read and why, the modules it removes, every module defined twice with where, the heads of a mesh-sdk
// member (empty when none), and how many modules it rebuilds.
type Change struct {
Group, Heads string
Manifests map[string]catalogue.Manifest
Unreadable map[string]string
Removed []string
Twice map[string][]string
SDKHead string
Rebuilt int
// Digests is the manifest digest of every module the change moves or adds, readable by the assigner or not, and
// Members the head of each member by repository: what a composer at the group's mesh-sdk reads (issues 1622, 1623).
Digests map[string]string
Members map[string]string
// Held are the approved held values of the group's pull requests at the heads judged (mesh-config's `kept`, novox/hq ADR
// 0565 §5, ADR 0502 §4): each laid over its module's values on its node in the change's composition, and named.
Held []delivery.KeptValue
}
// HeldOf are the held values kept for the pull requests the candidate names, each at the head it names: one of another
// pull request, or at another head, is never laid (ADR 0502 §4). kept is mesh-config's `kept` by key.
func HeldOf(kept map[string][]byte, heads map[string]string) []delivery.KeptValue {
var out []delivery.KeptValue
for _, raw := range kept {
var k delivery.KeptValue
if json.Unmarshal(raw, &k) != nil || k.Node == "" || k.Module == "" || k.Key == "" {
continue
}
if head, ok := heads[k.Pull]; ok && head == k.Head {
out = append(out, k)
}
}
sort.Slice(out, func(i, j int) bool {
a, b := out[i], out[j]
return a.Pull+a.Node+a.Module+a.Key < b.Pull+b.Node+b.Module+b.Key
})
return out
}
// layHeld lays each held value over its module's values on its node, as a layer of its own after the rest, and answers
// the words that name each; judged first against the head's declaration of its key (catalogue.ValueProblems), a value the
// declaration refuses, or a key it does not declare, is a failure.
func layHeld(in *compose.Input, held []delivery.KeptValue) (said, fails []string) {
for _, k := range held {
var v any
if err := json.Unmarshal(k.Value, &v); err != nil {
said = append(said, fmt.Sprintf("the held value %s of %s on %s for %s cannot be read, so it is not laid: %v", k.Key, k.Module, k.Node, k.Pull, err))
continue
}
d, declared := in.Catalogue[k.Module].Settings[k.Key]
if !declared {
fails = append(fails, fmt.Sprintf("the held value %s of %s on %s for %s: the head declares no such key", k.Key, k.Module, k.Node, k.Pull))
continue
}
if problems := catalogue.ValueProblems(d, v); len(problems) > 0 {
fails = append(fails, fmt.Sprintf("the held value %s of %s on %s for %s does not hold to the head's declaration: %s", k.Key, k.Module,
k.Node, k.Pull, strings.Join(problems, "; ")))
continue
}
at := slices.IndexFunc(in.Nodes, func(n compose.Node) bool { return n.Name == k.Node })
if at < 0 {
said = append(said, fmt.Sprintf("the held value %s of %s for %s names %s, which runs no module of the mesh: not laid", k.Key, k.Module, k.Pull, k.Node))
continue
}
if in.Nodes[at].Settings == nil {
in.Nodes[at].Settings = map[string][]catalogue.Layer{}
}
in.Nodes[at].Settings[k.Module] = append(in.Nodes[at].Settings[k.Module], catalogue.Layer{From: "held for " + k.Pull, Values: map[string]any{k.Key: v}})
said = append(said, fmt.Sprintf("held value laid: %s of %s on %s, held for %s at %.12s, approved by %s (novox/hq ADR 0502 §4)", k.Key, k.Module, k.Node, k.Pull, k.Head, k.Approved))
}
return said, fails
}
// Touched is the machines a change touches (ADR 0446 clause 3): those running a module the change moves, adds or
// removes, those running a consumer of a provision such a module provides (before or after), and those running a
// provider of a provision such a module requires after the change. A module the change adds that is assigned nowhere
// touches no machine. A change of placement is no change of a repository (ADR 0567 §4): the seat
// mesh-placement judges it.
func Touched(m Mesh, c Change) []string {
moved := map[string]bool{}
for name := range c.Manifests {
moved[name] = true
}
for name := range c.Unreadable {
moved[name] = true
}
for _, name := range c.Removed {
moved[name] = true
}
provided, required := map[string]bool{}, map[string]bool{}
for name := range moved {
for _, man := range []catalogue.Manifest{m.Manifests[name], c.Manifests[name]} {
for _, o := range man.Provides {
provided[o.Name] = true
}
}
for _, r := range c.Manifests[name].Requires {
required[r] = true
}
}
var out []string
for node, modules := range m.Assigned {
for _, mod := range modules {
man := m.Manifests[mod]
if moved[mod] || slices.ContainsFunc(man.Requires, func(r string) bool { return provided[r] }) ||
slices.ContainsFunc(man.Provides, func(o catalogue.Offer) bool { return required[o.Name] }) {
out = append(out, node)
break
}
}
}
sort.Strings(out)
return out
}
// input is the composition's input for the mesh's catalogue and assignments: the nodes by name with what is assigned
// to them, and every seat the assigned modules claim as held there. What the bus does not carry (places on the
// private network, settings, credentials, licences) is left empty, so a trunk composition and a change's are made
// with the same gaps and compared (ADR 0446 clause 1: what was already broken is said and fails nothing).
func input(catalog map[string]catalogue.Manifest, assigned map[string][]string) compose.Input {
in := compose.Input{Catalogue: catalog}
nodes := make([]string, 0, len(assigned))
for n := range assigned {
nodes = append(nodes, n)
}
sort.Strings(nodes)
for _, n := range nodes {
in.Nodes = append(in.Nodes, compose.Node{Name: n, Assigned: slices.Clone(assigned[n])})
for _, mod := range assigned[n] {
for _, cl := range catalog[mod].Claims {
scope := cl.Scope
if seat, ok := catalogue.SeatNamed(cl.Name); ok && scope == "" {
scope = seat.Scope
}
in.Holdings = append(in.Holdings, catalogue.Held{Claim: cl.Name, Scope: scope, Node: n, Module: mod})
}
}
}
return in
}
// Compose is the composition check of change c over the mesh m, composed with the mesh-sdk version sdk: every touched
// machine resolved as the trunk holds it and as the change leaves it. It fails on a manifest of the change that cannot
// be read, a module a machine runs that the change removes, a module defined twice, and a machine that resolves on the
// trunk and not with the change; a machine that does not resolve on either is said and fails nothing. It warns past
// CompositionRebuildWarning modules rebuilt, says what it does not judge (NotJudged), and for a group with a mesh-sdk
// member says it composed with its own mesh-sdk, not the member's head (issue 1621). took is how long it took.
func Compose(m Mesh, c Change, sdk string, took time.Duration) delivery.CompositionVerdict {
v := delivery.CompositionVerdict{Group: c.Group, Heads: c.Heads, SDK: sdk, Took: took.Milliseconds()}
var fails []string
failAt := func(machine, words string) {
if v.Machine == "" && machine != "" {
v.Machine = machine
}
fails = append(fails, words)
}
for _, name := range sortedKeys(c.Unreadable) {
failAt("", fmt.Sprintf("the manifest of %s cannot be read: %s", name, c.Unreadable[name]))
}
for _, name := range sortedKeys(c.Twice) {
failAt("", fmt.Sprintf("%s is defined twice, in %s", name, strings.Join(c.Twice[name], " and ")))
}
machines := Touched(m, c)
v.Machines = machines
for _, node := range machines {
for _, name := range c.Removed {
if slices.Contains(m.Assigned[node], name) {
failAt(node, fmt.Sprintf("%s runs %s, which the change removes from its source", node, name))
}
}
}
trunk := input(m.Manifests, m.Assigned)
changed := map[string]catalogue.Manifest{}
for name, man := range m.Manifests {
changed[name] = man
}
for name, man := range c.Manifests {
changed[name] = man
}
for _, name := range c.Removed {
delete(changed, name)
}
after := input(changed, m.Assigned)
laid, heldFails := layHeld(&after, c.Held)
v.Warnings = append(v.Warnings, laid...)
for _, f := range heldFails {
failAt("", f)
}
for _, node := range machines {
_, before := compose.Resolve(trunk, node)
_, now := compose.Resolve(after, node)
switch {
case now == nil:
case before != nil:
v.Warnings = append(v.Warnings, fmt.Sprintf("%s does not compose on the trunk either, so the change is not blamed: %v", node, before))
default:
failAt(node, fmt.Sprintf("%s composes on the trunk and not with the change: %v", node, now))
}
}
if c.SDKHead != "" {
v.Warnings = append(v.Warnings, fmt.Sprintf("composed with the assigner's own mesh-sdk %s, not the group's mesh-sdk head %s, "+
"and a field the running controller cannot read is not judged (issues 1621, 1622, 1623)", sdk, c.SDKHead))
}
return finish(v, m, c, machines, fails)
}
// finish adds what every composition says beside its own judgement (more modules rebuilt than CompositionRebuildWarning,
// what is not judged, the machines not composed, a change that touches none) and its state from fails.
func finish(v delivery.CompositionVerdict, m Mesh, c Change, machines []string, fails []string) delivery.CompositionVerdict {
if c.Rebuilt > delivery.CompositionRebuildWarning {
v.Warnings = append(v.Warnings, fmt.Sprintf("the merge rebuilds %d modules, more than %d", c.Rebuilt, delivery.CompositionRebuildWarning))
}
v.Warnings = append(v.Warnings, NotJudged)
if could := CouldRun(m, c, machines); len(could) > 0 {
v.Warnings = append(v.Warnings, "not judged until the facts snapshot is on the bus (issue 1461, ADR 0537 clause 1): "+
"the machines that could run a moved or added module and are not assigned it: "+strings.Join(could, ", "))
}
if grants := GrantsAlone(m, c, machines); len(grants) > 0 {
v.Warnings = append(v.Warnings, "not judged until the facts snapshot is on the bus (issue 1461, ADR 0537 clause 1): "+
"the machines whose grants alone the change alters: "+strings.Join(grants, ", "))
}
if len(machines) == 0 {
v.Warnings = append(v.Warnings, "the change touches no machine: no module it moves is assigned anywhere")
}
if len(fails) > 0 {
v.State, v.Words = delivery.CompositionFail, strings.Join(fails, "; ")
return v
}
v.State = delivery.CompositionPass
return v
}
// CouldRun is, by name, every machine of the mesh not composed that is not assigned a module the change moves or adds
// (ADR 0537 clause 1): which of them could run it, the bus does not say (capabilities, settings and places are only in
// the facts snapshot), so each is named as not judged.
func CouldRun(m Mesh, c Change, composed []string) []string {
var out []string
for node, modules := range m.Assigned {
if slices.Contains(composed, node) {
continue
}
for name := range c.Manifests {
if !slices.Contains(modules, name) {
out = append(out, node)
break
}
}
}
sort.Strings(out)
return out
}
// grantsOf is what of a manifest makes grants: its bus declarations and the provisions it requires and offers.
func grantsOf(man catalogue.Manifest) string {
var offers []string
for _, o := range man.Provides {
offers = append(offers, o.Name)
}
return fmt.Sprint(man.Requires, offers, man.Reads, man.State, man.Emits, man.Consumes, man.Invokes, man.Claims, man.Grants)
}
// GrantsAlone is, by name, every machine not composed whose grants alone the change alters (ADR 0537 clause 1): for a
// moved module whose grants differ from the trunk's, the machines assigned the bus's holder (whose user list carries
// every module's bus grants) and those assigned a provider of a provision it requires, before or after.
func GrantsAlone(m Mesh, c Change, composed []string) []string {
provisions := map[string]bool{}
altered := false
for name, man := range c.Manifests {
before, had := m.Manifests[name]
if had && grantsOf(before) == grantsOf(man) {
continue
}
altered = true
for _, r := range append(slices.Clone(man.Requires), before.Requires...) {
provisions[r] = true
}
}
if !altered {
return nil
}
var out []string
for node, modules := range m.Assigned {
if slices.Contains(composed, node) {
continue
}
for _, mod := range modules {
man := m.Manifests[mod]
if man.ClaimsSeat("mesh-broker") || slices.ContainsFunc(man.Provides, func(o catalogue.Offer) bool { return provisions[o.Name] }) {
out = append(out, node)
break
}
}
}
sort.Strings(out)
return out
}
// withOverlay is base with every manifest of over laid over it, a copy.
func withOverlay(base, over map[string]catalogue.Manifest) map[string]catalogue.Manifest {
out := make(map[string]catalogue.Manifest, len(base)+len(over))
for name, man := range base {
out[name] = man
}
for name, man := range over {
out[name] = man
}
return out
}
func sortedKeys[V any](m map[string]V) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}
modules/mesh-delivery-assigner/internal/assign/compose_test.go
-169
View File
@@ -1,169 +0,0 @@
package assign
import (
"encoding/json"
"slices"
"strconv"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
func meshOf() Mesh {
return Mesh{
Manifests: map[string]catalogue.Manifest{
"mail": {Module: "mail", Version: "1", Requires: []string{"database"}},
"store": {Module: "store", Version: "1", Provides: []catalogue.Offer{{Name: "database"}}},
"web": {Module: "web", Version: "1"},
},
Assigned: map[string][]string{"ace": {"mail", "store"}, "g14": {"web"}},
}
}
func hasWarning(v delivery.CompositionVerdict, part string) bool {
return slices.ContainsFunc(v.Warnings, func(w string) bool { return strings.Contains(w, part) })
}
// 1297/AC1 (its unit half): a change touching one module on one machine composes that machine alone, passes, names
// the machine and the mesh-sdk it composed with, and says what it did not judge.
func TestAChangeOfOneModuleComposesItsMachine(t *testing.T) {
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"web": {Module: "web", Version: "2"}}}
v := Compose(meshOf(), c, "v0.1.14-x", 0)
if v.State != delivery.CompositionPass || !slices.Equal(v.Machines, []string{"g14"}) || v.SDK != "v0.1.14-x" {
t.Fatalf("composition = %+v, want a pass on g14 alone", v)
}
if !hasWarning(v, "issue 1461") {
t.Errorf("the verdict does not say what it did not judge: %v", v.Warnings)
}
if _, err := delivery.EncodeCompositionVerdict(v); err != nil {
t.Errorf("the verdict cannot be written: %v", err)
}
// 1953/D7 (ADR 0565 §5, ADR 0502 §4): the group's own pull request's held value at its head is laid and named, judged
// against the head's declaration; one of another pull request or another head is not laid.
web, err := catalogue.Decode([]byte(`{"module":"web","version":"2","config":{"port":{"type":"number","default":80,"description":"the port, a number","runtime":true}}}`))
if err != nil {
t.Fatal(err)
}
kept := map[string][]byte{}
for i, k := range []delivery.KeptValue{
{Pull: "novox/x#1", Head: "h1", Node: "g14", Module: "web", Key: "port", Value: json.RawMessage(`81`), Approved: "op"},
{Pull: "novox/x#2", Head: "h2", Node: "g14", Module: "web", Key: "port", Value: json.RawMessage(`82`)},
{Pull: "novox/x#1", Head: "old", Node: "g14", Module: "web", Key: "other", Value: json.RawMessage(`1`)},
} {
kept[strconv.Itoa(i)], _ = json.Marshal(k)
}
held := HeldOf(kept, map[string]string{"novox/x#1": "h1"})
if len(held) != 1 || string(held[0].Value) != "81" {
t.Fatalf("held = %+v, want #1's at its head alone", held)
}
c = Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"web": web}, Held: held}
if v := Compose(meshOf(), c, "x", 0); v.State != delivery.CompositionPass || !hasWarning(v, "held value laid: port of web on g14, held for novox/x#1") {
t.Errorf("composition = %+v", v)
}
c.Held[0].Value = json.RawMessage(`"eighty"`)
if v := Compose(meshOf(), c, "x", 0); v.State != delivery.CompositionFail || !strings.Contains(v.Words, "does not hold to the head's declaration") {
t.Errorf("a held value the declaration refuses: %+v", v)
}
}
// 1297/AC2 and B3c: a change that removes a module a machine runs fails, naming the machine and the module; a machine that
// composes on the trunk and not with the change fails it, naming the machine and why (the consumer's machine is touched when
// its provider changes); what was already broken on the trunk is said and fails nothing. One table (three tests merged).
func TestAChangeThatBreaksAMachineFailsAndWhatWasBrokenFailsNothing(t *testing.T) {
broken := meshOf()
broken.Manifests["store"] = catalogue.Manifest{Module: "store", Version: "1"}
for _, c := range []struct {
name string
mesh Mesh
c Change
state string
must func(delivery.CompositionVerdict) bool
}{
{"removing a module a machine runs", meshOf(), Change{Group: "g", Heads: "0123456789ab", Removed: []string{"web"}}, delivery.CompositionFail,
func(v delivery.CompositionVerdict) bool {
return v.Machine == "g14" && strings.Contains(v.Words, "g14 runs web")
}},
{"a machine the change breaks", meshOf(), Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"store": {Module: "store", Version: "2"}}},
delivery.CompositionFail, func(v delivery.CompositionVerdict) bool {
return v.Machine == "ace" && strings.Contains(v.Words, "database")
}},
{"what was broken before", broken, Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"mail": {Module: "mail", Version: "2", Requires: []string{"database"}}}},
delivery.CompositionPass, func(v delivery.CompositionVerdict) bool { return hasWarning(v, "on the trunk either") }},
} {
if v := Compose(c.mesh, c.c, "sdk", 0); v.State != c.state || !c.must(v) {
t.Errorf("%s: composition = %+v", c.name, v)
}
}
}
// 1297/B3c: an unreadable manifest and a module defined twice fail; more than twelve rebuilt modules warn.
func TestUnreadableTwiceAndWide(t *testing.T) {
c := Change{Group: "g", Heads: "0123456789ab", Unreadable: map[string]string{"web": "unknown field \"scope\""}, Rebuilt: 13}
v := Compose(meshOf(), c, "sdk", 0)
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "web cannot be read") || !hasWarning(v, "rebuilds 13 modules") {
t.Fatalf("composition = %+v", v)
}
c = Change{Group: "g", Heads: "0123456789ab", Twice: map[string][]string{"web": {"novox/a", "novox/b"}}, Rebuilt: 12}
v = Compose(meshOf(), c, "sdk", 0)
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "web is defined twice") || hasWarning(v, "rebuilds") {
t.Fatalf("composition = %+v", v)
}
}
// 1622's criterion as R4g delivers it: a group with a mesh-sdk member is composed with the assigner's own mesh-sdk, and
// the verdict says so, naming the member's head it did not use; a group without one says nothing of it.
func TestAGroupWithAnSDKMemberSaysWhichSDKComposedIt(t *testing.T) {
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"web": {Module: "web", Version: "2"}}, SDKHead: "f625d98713e4"}
v := Compose(meshOf(), c, "v0.1.14-own", 0)
if !hasWarning(v, "not the group's mesh-sdk head f625d98713e4") || !hasWarning(v, "v0.1.14-own") || !hasWarning(v, "1623") {
t.Errorf("the verdict does not say it composed with its own mesh-sdk: %v", v.Warnings)
}
c.SDKHead = ""
if v := Compose(meshOf(), c, "v0.1.14-own", 0); hasWarning(v, "mesh-sdk head") {
t.Errorf("a group without a mesh-sdk member says it has one: %v", v.Warnings)
}
}
// 1297/B3a: the machines touched are those running a moved module or a consumer of what it provides; an added module
// assigned nowhere touches none, and the verdict says so.
func TestTouched(t *testing.T) {
m := meshOf()
if got := Touched(m, Change{Manifests: map[string]catalogue.Manifest{"store": m.Manifests["store"]}}); !slices.Equal(got, []string{"ace"}) {
t.Errorf("touched by store = %v", got)
}
m.Assigned["g14"] = append(m.Assigned["g14"], "mail")
if got := Touched(m, Change{Manifests: map[string]catalogue.Manifest{"store": m.Manifests["store"]}}); !slices.Equal(got, []string{"ace", "g14"}) {
t.Errorf("touched by store with a consumer on g14 = %v", got)
}
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"new": {Module: "new", Version: "1"}}}
if v := Compose(m, c, "sdk", 0); len(v.Machines) != 0 || !hasWarning(v, "touches no machine") || v.State != delivery.CompositionPass {
t.Errorf("an added module assigned nowhere: %+v", v)
}
}
// ADR 0537 clause 1: every verdict names, by machine, those that could run a moved or added module and are not assigned
// it, and those whose grants alone the change alters (the bus's machine, a provider of what the module requires), as not
// judged; shanks, providing what web now requires, is composed and so named in neither. A change that alters no grant
// names none of the second.
func TestTheMachinesNotJudgedAreNamed(t *testing.T) {
m := meshOf()
m.Manifests["bus"] = catalogue.Manifest{Module: "bus", Version: "1", Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: "mesh"}}}
m.Manifests["db"] = catalogue.Manifest{Module: "db", Version: "1", Provides: []catalogue.Offer{{Name: "cache"}}}
m.Assigned["novox"] = []string{"bus"}
m.Assigned["shanks"] = []string{"db"}
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{
"web": {Module: "web", Version: "2", Reads: []string{"mesh-controller.assignments"}, Requires: []string{"cache"}}}}
v := Compose(m, c, "sdk", 0)
if !hasWarning(v, "could run a moved or added module and are not assigned it: ace, novox") || !slices.Contains(v.Machines, "shanks") {
t.Errorf("the machines that could run web are not named: %v", v.Warnings)
}
if !hasWarning(v, "whose grants alone the change alters: novox") {
t.Errorf("the machines whose grants alone change are not named: %v", v.Warnings)
}
c.Manifests["web"] = catalogue.Manifest{Module: "web", Version: "3"}
if v := Compose(meshOf(), c, "sdk", 0); hasWarning(v, "grants alone") {
t.Errorf("a change altering no grant names machines whose grants change: %v", v.Warnings)
}
}
modules/mesh-delivery-assigner/internal/assign/composer.go
-263
View File
@@ -1,263 +0,0 @@
package assign
import (
"encoding/json"
"fmt"
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/compose"
"git.novox.be/novox/mesh-sdk/go/composer"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The cases of a group's composition at its mesh-sdk head: the trunk as it is, and the trunk with the change.
const (
CaseTrunk = "trunk"
CaseChange = "change"
)
// SDKDir is the directory of mesh-sdk's Go module in its repository, where the composer is built at a head; a reader's
// composer is built at its repository's root, at the mesh-sdk its go.mod pins.
const SDKDir = "go"
// The modules whose mesh-sdk reads the manifests a plan sends (ADR 0482 clause 5): the node-engine composes and applies
// them, the controller being gone (novox/hq ADR 0590, issue 1126). Each is judged at its repository.
var ReaderModules = map[string]string{"mesh-host": "novox/mesh-host"}
// HeadJob is the compose job of a group with a mesh-sdk member (ADR 0482 clauses 3 and 4, issue 1622): the composer
// built at the member's head composes the machines the change touches, as the trunk holds them and as the change leaves
// them, every manifest named by its digest in the indexer's bucket, the seats held from the claims (the assigner's input).
func HeadJob(m Mesh, c Change) (delivery.ComposeJob, error) {
trunk, change := map[string]composer.Manifest{}, map[string]composer.Manifest{}
for name, digest := range m.Digests {
trunk[name] = composer.Manifest{Digest: digest}
change[name] = composer.Manifest{Digest: digest}
}
for name, digest := range c.Digests {
change[name] = composer.Manifest{Digest: digest}
}
for _, name := range c.Removed {
delete(change, name)
}
in, fromClaims := plainInput(m.Assigned), true
if m.Input != nil {
// The input the nodes compose from, without its catalogue: the cases carry the manifests (issue 1126).
in = *m.Input
in.Catalogue = nil
fromClaims = len(in.Holdings) == 0
}
// Each trunk manifest the indexer refused goes in with its verdict, so the composer holds and names it (ADR 0578 §5);
// a manifest the change brings has no verdict of the indexer yet, and the checker's judge judges it (§2).
changeVerdicts := map[string]delivery.ManifestVerdict{}
for name, v := range m.Verdicts {
if _, changed := c.Digests[name]; !changed {
changeVerdicts[name] = v
}
}
changeCase := composer.Case{Name: CaseChange, Manifests: change, Verdicts: changeVerdicts}
req := composer.Request{V: composer.Version, Input: in, Depth: composer.DepthResolve,
HoldingsFromClaims: fromClaims, Machines: Touched(m, c),
Cases: []composer.Case{{Name: CaseTrunk, Manifests: trunk, Verdicts: m.Verdicts}, changeCase}}
return job(SDKRepository, c.SDKHead, SDKDir, req, "the mesh-sdk head of "+c.Group)
}
// ReaderJob is the compose job that reads the manifests a group changes with the mesh-sdk a reader module pins at
// commit (ADR 0482 clause 5, issue 1623): no machine is composed, every changed manifest is only read.
func ReaderJob(c Change, module, repository, commit string) (delivery.ComposeJob, error) {
changed := map[string]composer.Manifest{}
for name, digest := range c.Digests {
changed[name] = composer.Manifest{Digest: digest}
}
req := composer.Request{V: composer.Version, Cases: []composer.Case{{Name: CaseChange, Manifests: changed}}}
return job(repository, commit, delivery.RootPath, req, module+" at "+commit)
}
func job(repository, commit, dir string, req composer.Request, reader string) (delivery.ComposeJob, error) {
raw, err := json.Marshal(req)
if err != nil {
return delivery.ComposeJob{}, err
}
return delivery.ComposeJob{Fetch: map[string]any{"repository": repository, "commit": commit}, Dir: dir,
Package: composer.Package, Request: raw, Reader: reader}, nil
}
// plainInput is the composition's input without its catalogue: the nodes by name with what is assigned to them (the
// composer derives the seats held from the claims, as input does).
func plainInput(assigned map[string][]string) compose.Input {
var in compose.Input
nodes := make([]string, 0, len(assigned))
for n := range assigned {
nodes = append(nodes, n)
}
sort.Strings(nodes)
for _, n := range nodes {
in.Nodes = append(in.Nodes, compose.Node{Name: n, Assigned: slices.Clone(assigned[n])})
}
return in
}
// Ran is one compose ask's end: its step's state and reason, and the composer's verdict when the step passed.
type Ran struct {
State string
Reason string
Verdict *composer.Verdict
}
// RanOf reads a compose ask's end from its step value; a step that passed with an answer that cannot be read is a
// failure, named.
func RanOf(step delivery.Step) Ran {
r := Ran{State: step.State, Reason: step.Reason}
if step.State != delivery.StepPassed {
return r
}
var a delivery.ComposeAnswer
var v composer.Verdict
if err := json.Unmarshal(step.Result, &a); err != nil {
return Ran{State: delivery.StepFailed, Reason: "the compose step's answer cannot be read: " + err.Error()}
}
if err := json.Unmarshal(a.Verdict, &v); err != nil || v.V != composer.Version {
return Ran{State: delivery.StepFailed, Reason: fmt.Sprintf("the composer's verdict cannot be read (version %d): %v", v.V, err)}
}
r.Verdict = &v
return r
}
func caseOf(v *composer.Verdict, name string) (composer.CaseVerdict, bool) {
if v == nil {
return composer.CaseVerdict{}, false
}
for _, c := range v.Cases {
if c.Name == name {
return c, true
}
}
return composer.CaseVerdict{}, false
}
// Reader is one module whose mesh-sdk reads the group's manifests, at the commit judged: the group's member head when
// the group holds one in its repository, else a build its node runs.
type Reader struct {
Module, Commit string
Member bool
Ran Ran
}
// FromComposer is the composition check of a group with a mesh-sdk member, judged from the composer built at its head
// (ADR 0482 clauses 3 to 5, issues 1622 and 1623): the same failures as Compose, read from the composer's two cases, the
// verdict naming the mesh-sdk the composer was built with; a trunk manifest the head's mesh-sdk cannot decode fails too
// (the group would deliver a reader that cannot read it). A reader that cannot read a changed manifest is a warning,
// never a failure: it decodes leniently and sets a field it does not know aside (novox/hq ADR 0578 §6, which changes
// ADR 0482 §5).
func FromComposer(m Mesh, c Change, head Ran, readers []Reader, took time.Duration) delivery.CompositionVerdict {
v := delivery.CompositionVerdict{Group: c.Group, Heads: c.Heads, SDK: "mesh-sdk " + c.SDKHead, Took: took.Milliseconds()}
machines := Touched(m, c)
v.Machines = machines
var fails []string
if head.State != delivery.StepPassed {
v.Words = fmt.Sprintf("the composer at mesh-sdk %s did not run: %s %s", c.SDKHead, head.State, head.Reason)
v.State = delivery.CompositionFail
return v
}
if head.Verdict.SDK != "" {
v.SDK = head.Verdict.SDK
}
trunk, okT := caseOf(head.Verdict, CaseTrunk)
change, okC := caseOf(head.Verdict, CaseChange)
if !okT || !okC {
v.State, v.Words = delivery.CompositionFail, "the composer's verdict holds no trunk or no change case"
return v
}
failAt := func(machine, words string) {
if v.Machine == "" && machine != "" {
v.Machine = machine
}
fails = append(fails, words)
}
for _, name := range sortedKeys(c.Digests) {
if r := change.Manifests[name]; !r.Accepted {
failAt("", fmt.Sprintf("the manifest of %s cannot be read by mesh-sdk %s: %s", name, c.SDKHead, r.Refused))
}
}
for _, name := range sortedKeys(trunk.Manifests) {
if _, changed := c.Digests[name]; changed || slices.Contains(c.Removed, name) || trunk.Manifests[name].Accepted {
continue
}
if _, before := m.Unreadable[name]; before {
continue // already unreadable on the trunk: not the group's doing
}
failAt("", fmt.Sprintf("mesh-sdk %s refuses the trunk's manifest of %s, which the group does not change: %s", c.SDKHead, name, trunk.Manifests[name].Refused))
}
for _, name := range sortedKeys(c.Twice) {
failAt("", fmt.Sprintf("%s is defined twice, in %s", name, strings.Join(c.Twice[name], " and ")))
}
for _, node := range machines {
for _, name := range c.Removed {
if slices.Contains(m.Assigned[node], name) {
failAt(node, fmt.Sprintf("%s runs %s, which the change removes from its source", node, name))
}
}
before, now := trunk.Machines[node], change.Machines[node]
switch {
case now.Composes:
case !before.Composes:
v.Warnings = append(v.Warnings, fmt.Sprintf("%s does not compose on the trunk either, so the change is not blamed: %s", node, before.Reason))
default:
failAt(node, fmt.Sprintf("%s composes on the trunk and not with the change: %s", node, now.Reason))
}
}
readerFails, readerWarnings := judgeReaders(c, change, readers)
fails = append(fails, readerFails...)
v.Warnings = append(v.Warnings, readerWarnings...)
if m.InputUnread != "" {
v.Warnings = append(v.Warnings, "composed over the nodes and their assignments alone: the input the nodes compose from "+
"cannot be read from the bus ("+m.InputUnread+")")
}
return finish(v, m, c, machines, fails)
}
// NoComposer says whether a compose step failed because the mesh-sdk it was built at has no composer command (a mesh-sdk
// from before issue 1621): Go's words for a package its module does not hold.
func NoComposer(reason string) bool {
for _, words := range []string{"cannot find package", "no required module provides package", "is not in std", "cannot find module providing package", "does not contain package"} {
if strings.Contains(reason, words) {
return true
}
}
return false
}
func judgeReaders(c Change, head composer.CaseVerdict, readers []Reader) (fails, warnings []string) {
for _, r := range readers {
at := r.Module + " at " + r.Commit
if r.Member {
at = r.Module + "'s member head " + r.Commit
}
if r.Ran.State != delivery.StepPassed {
if NoComposer(r.Ran.Reason) {
warnings = append(warnings, fmt.Sprintf("not judged whether %s reads the group's manifests: the mesh-sdk it pins has no "+
"composer command (before issue 1621)", at))
continue
}
fails = append(fails, fmt.Sprintf("the composer of the mesh-sdk %s pins did not run: %s %s", at, r.Ran.State, r.Ran.Reason))
continue
}
read, ok := caseOf(r.Ran.Verdict, CaseChange)
if !ok {
fails = append(fails, fmt.Sprintf("the composer of the mesh-sdk %s pins answered no change case", at))
continue
}
for _, name := range sortedKeys(c.Digests) {
if !head.Manifests[name].Accepted || read.Manifests[name].Accepted {
continue
}
// ADR 0578 §6: no longer a failure; the reader sets what it does not know aside, and the part that needs it takes
// the newer package (ADR 0573 §2).
warnings = append(warnings, fmt.Sprintf("the manifest of %s uses what %s (mesh-sdk %s) cannot read: %s; it reads the rest "+
"once it takes mesh-sdk %s", name, at, r.Ran.Verdict.SDK, read.Manifests[name].Refused, c.SDKHead))
}
}
return fails, warnings
}
modules/mesh-delivery-assigner/internal/assign/composer_test.go
-225
View File
@@ -1,225 +0,0 @@
package assign
import (
"encoding/json"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/compose"
"git.novox.be/novox/mesh-sdk/go/composer"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
var testTime = time.Date(2026, 10, 24, 3, 0, 0, 0, time.UTC)
const sdkHead = "1111111111111111111111111111111111111111"
// raws are the manifests of meshOf as their repositories hold them, by digest, as the indexer keeps them; `web2` is web
// with a field `shares` that the assigner's own mesh-sdk does not read (the shape of issue 1593).
var raws = map[string]string{
"d-mail": `{"module": "mail", "version": "1", "requires": ["postgres-database"]}`,
"d-store": `{"module": "store", "version": "1", "provides": [{"name": "postgres-database"}]}`,
"d-web": `{"module": "web", "version": "1"}`,
"d-web2": `{"module": "web", "version": "2", "shares": ["photos"]}`,
}
// asTheHeadReads is how mesh-sdk at the group's head reads each manifest: as the assigner's own reads it, but for the
// field it adds, which it accepts. The head's composer is built and run by the Go build module (ADR 0482 clause 4); here
// its verdict is made by mesh-sdk's composer.Run over the same request, with `web2` read as the head reads it.
var asTheHeadReads = map[string]string{"d-web2": `{"module": "web", "version": "2"}`}
func digestedMesh() Mesh {
m := meshOf()
m.Digests = map[string]string{"mail": "d-mail", "store": "d-store", "web": "d-web"}
return m
}
// webShares is a group of mesh-sdk and mesh-catalog whose catalogue member gives web the field only the SDK head reads:
// the assigner's own reader refuses it (Unreadable), as the indexer's manifest is read in world.change.
func webShares(members map[string]string) Change {
// As an older reader refused it; the field set aside by the lenient decoder is named the same (ADR 0578 §6).
m, _ := catalogue.Decode([]byte(raws["d-web2"]))
return Change{Group: "g", Heads: "0123456789ab", SDKHead: sdkHead, Manifests: map[string]catalogue.Manifest{},
Unreadable: map[string]string{"web": m.SetAside()}, Digests: map[string]string{"web": "d-web2"}, Members: members}
}
// run is the composer's step for a job, as the build module answers it: the request filled from the indexer's digests
// (with read standing in for how that mesh-sdk reads a digest), run, and its verdict put in a passed step.
func run(t *testing.T, job delivery.ComposeJob, read map[string]string) Ran {
t.Helper()
req, err := composer.Decode(job.Request)
if err != nil {
t.Fatal(err)
}
if err := composer.Fill(&req, func(d string) ([]byte, error) {
if r, ok := read[d]; ok {
return []byte(r), nil
}
return []byte(raws[d]), nil
}); err != nil {
t.Fatal(err)
}
v, err := composer.Run(req)
if err != nil {
t.Fatal(err)
}
v.SDK = "(devel) " + job.Fetch["commit"].(string)
vr, _ := json.Marshal(v)
ans, _ := json.Marshal(delivery.ComposeAnswer{Built: true, Key: "composer-0", Verdict: vr})
return RanOf(delivery.Step{State: delivery.StepPassed, Result: ans})
}
// 1622/AC1: a group holding a mesh-sdk pull request is composed by the composer built at the member's head: the job
// fetches mesh-sdk at that head, builds mesh-sdk's composer command in its Go module, and hands it the trunk and the
// change by digest, never the manifests' bytes, composing the machines the change touches.
func TestAGroupWithAnSDKMemberAsksTheComposerAtItsHead(t *testing.T) {
c := webShares(nil)
job, err := HeadJob(digestedMesh(), c)
if err != nil {
t.Fatal(err)
}
if job.Fetch["repository"] != SDKRepository || job.Fetch["commit"] != sdkHead || job.Dir != "go" || job.Package != composer.Package {
t.Fatalf("job = %+v", job)
}
ask := delivery.StepAsk{ID: "mh3k2x9qa1b2c5", Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: "mesh-delivery-assigner",
At: testTime, Compose: &job}
if _, err := delivery.EncodeStepAsk(ask); err != nil {
t.Fatalf("the ask is refused by its layout: %v", err)
}
req, err := composer.Decode(job.Request)
if err != nil {
t.Fatal(err)
}
if len(req.Cases) != 2 || req.Cases[0].Name != CaseTrunk || req.Cases[1].Name != CaseChange {
t.Fatalf("cases = %+v", req.Cases)
}
if req.Cases[0].Manifests["web"].Digest != "d-web" || req.Cases[1].Manifests["web"].Digest != "d-web2" || len(req.Cases[1].Manifests["web"].Raw) != 0 {
t.Errorf("the cases do not name web by its trunk and its change digest: %+v", req.Cases)
}
if !slices.Equal(req.Machines, []string{"g14"}) || req.Depth != composer.DepthResolve || !req.HoldingsFromClaims {
t.Errorf("request = machines %v, depth %q, holdings from claims %v", req.Machines, req.Depth, req.HoldingsFromClaims)
}
// 1126 (ADR 0547 §2, ADR 0549 §3): once the catalogue places the nodes, the check composes over the input the nodes
// compose from (frombus), its catalogue left to the cases, and the seats held as placement says.
m := digestedMesh()
m.Input = &compose.Input{Catalogue: map[string]catalogue.Manifest{"web": {Module: "web"}},
Nodes: []compose.Node{{Name: "g14", Assigned: []string{"web"}, Place: compose.Place{Endpoint: "g14.example:51820"}}},
Holdings: []catalogue.Held{{Claim: "web-seat", Scope: "mesh", Node: "g14", Module: "web"}}}
placedJob, err := HeadJob(m, c)
if err != nil {
t.Fatal(err)
}
placed, err := composer.Decode(placedJob.Request)
if err != nil {
t.Fatal(err)
}
if placed.Input.Catalogue != nil || len(placed.Input.Nodes) != 1 || placed.Input.Nodes[0].Place.Endpoint != "g14.example:51820" ||
len(placed.Input.Holdings) != 1 || placed.HoldingsFromClaims {
t.Errorf("the placed check's request = %+v, holdings from claims %v", placed.Input, placed.HoldingsFromClaims)
}
}
// 1622/AC2, AC3: a catalogue member using a field only the SDK head reads passes with the head's composer, and the
// verdict names the mesh-sdk the composer was built with; with the assigner's own mesh-sdk it fails.
func TestAFieldTheGroupAddsPassesWithTheHeadsComposer(t *testing.T) {
c := webShares(map[string]string{"novox/mesh-controller": "2222222222222222222222222222222222222222"})
job, _ := HeadJob(digestedMesh(), c)
head := run(t, job, asTheHeadReads)
v := FromComposer(digestedMesh(), c, head, nil, 0)
if v.State != delivery.CompositionPass || v.SDK != "(devel) "+sdkHead || !slices.Equal(v.Machines, []string{"g14"}) {
t.Fatalf("with the head's composer: %+v", v)
}
if _, err := delivery.EncodeCompositionVerdict(v); err != nil {
t.Errorf("the verdict cannot be written: %v", err)
}
own := Compose(digestedMesh(), c, "v0.1.14-own", 0)
if own.State != delivery.CompositionFail || !strings.Contains(own.Words, "shares") {
t.Fatalf("with the assigner's own mesh-sdk: %+v, want a fail naming the field", own)
}
}
// 1623/AC1, AC2 as ADR 0578 §6 changes them: the group of mesh-sdk and mesh-catalog adding and using a field, without a
// controller member, passes with a warning naming the field and mesh-controller, whose running build pins an older
// mesh-sdk that cannot read it: a reader sets a field it does not know aside, so it fails nothing. With a controller
// member pinned to the SDK head it passes with no such warning.
func TestAFieldTheRunningControllerCannotReadIsAWarning(t *testing.T) {
c := webShares(nil)
job, _ := HeadJob(digestedMesh(), c)
head := run(t, job, asTheHeadReads)
running := "3333333333333333333333333333333333333333"
rjob, err := ReaderJob(c, "mesh-controller", "novox/mesh-controller", running)
if err != nil {
t.Fatal(err)
}
if rjob.Fetch["commit"] != running || rjob.Dir != delivery.RootPath {
t.Fatalf("reader job = %+v", rjob)
}
// The running controller's mesh-sdk, older than the lenient decoder, refuses web2 for its field.
old := Ran{State: delivery.StepPassed, Verdict: &composer.Verdict{V: composer.Version, SDK: "v0.1.13", Cases: []composer.CaseVerdict{
{Name: CaseChange, Manifests: map[string]composer.ManifestRead{"web": {Refused: `json: unknown field "shares"`}}}}}}
v := FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-controller", Commit: running, Ran: old}}, 0)
if v.State != delivery.CompositionPass || !hasWarning(v, "shares") || !hasWarning(v, "mesh-controller at "+running) {
t.Fatalf("without a controller member: %+v, want a pass warning of the field and mesh-controller", v)
}
member := "2222222222222222222222222222222222222222"
pinned := run(t, rjob, asTheHeadReads)
v = FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-controller", Commit: member, Member: true, Ran: pinned}}, 0)
if v.State != delivery.CompositionPass {
t.Fatalf("with a controller member pinned to the head: %+v, want a pass", v)
}
}
// 1623: a reader whose mesh-sdk has no composer command (before issue 1621) is said, not failed; a reader's composer that
// failed for another cause fails the group, loud.
func TestAReaderWithNoComposerIsSaidAndAnotherFailureFails(t *testing.T) {
c := webShares(nil)
job, _ := HeadJob(digestedMesh(), c)
head := run(t, job, asTheHeadReads)
none := Ran{State: delivery.StepFailed, Reason: "go build: no required module provides package git.novox.be/novox/mesh-sdk/go/cmd/mesh-compose"}
v := FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-host", Commit: "abc", Ran: none}}, 0)
if v.State != delivery.CompositionPass || !hasWarning(v, "mesh-host at abc") || !hasWarning(v, "before issue 1621") {
t.Fatalf("a reader with no composer: %+v", v)
}
broke := Ran{State: delivery.StepFailed, Reason: "the fetch could not reach the forge"}
v = FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-host", Commit: "abc", Ran: broke}}, 0)
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "could not reach the forge") {
t.Fatalf("a reader whose composer failed: %+v", v)
}
}
// 1622: the head's composer that did not run fails the composition naming why, never passes it; a trunk manifest the
// head's mesh-sdk refuses fails it too, since the group would deliver a reader that refuses it.
func TestTheHeadsComposerFailingOrRefusingTheTrunkFails(t *testing.T) {
c := webShares(nil)
v := FromComposer(digestedMesh(), c, Ran{State: delivery.StepRefused, Reason: "the guard refuses it"}, nil, 0)
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "the guard refuses it") || !strings.Contains(v.Words, sdkHead) {
t.Fatalf("a head composer that did not run: %+v", v)
}
job, _ := HeadJob(digestedMesh(), c)
strict := map[string]string{"d-web2": asTheHeadReads["d-web2"], "d-mail": `{"module": "mail", "version": "1", "requires": "x"}`}
v = FromComposer(digestedMesh(), c, run(t, job, strict), nil, 0)
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "refuses the trunk's manifest of mail") {
t.Fatalf("a head that refuses a trunk manifest: %+v", v)
}
// ADR 0578 §5: a trunk manifest the indexer refused goes into the composer's input with its verdict, in both cases;
// the composer holds and names it, and the group is not blamed for it.
m := digestedMesh()
m.Verdicts = map[string]delivery.ManifestVerdict{"mail": {Module: "mail", State: delivery.VerdictRefused, Problems: []string{"it is wrong"}}}
m.Unreadable = map[string]string{"mail": m.Verdicts["mail"].Why()}
job, _ = HeadJob(m, c)
var req composer.Request
if err := json.Unmarshal(job.Request, &req); err != nil || !req.Cases[0].Verdicts["mail"].Refused() || !req.Cases[1].Verdicts["mail"].Refused() {
t.Fatalf("the request carries no verdict: %+v %v", req.Cases, err)
}
head := run(t, job, asTheHeadReads)
if trunk, _ := caseOf(head.Verdict, CaseTrunk); !strings.Contains(trunk.Manifests["mail"].Refused, "refused the manifest of mail: it is wrong") {
t.Fatalf("the composer did not name the refusal: %+v", trunk.Manifests["mail"])
}
if v = FromComposer(m, c, head, nil, 0); v.State != delivery.CompositionPass {
t.Fatalf("a refusal on the trunk blamed the group: %+v", v)
}
}
modules/mesh-delivery-assigner/internal/judge/judge.go
+331
View File
@@ -0,0 +1,331 @@
// Package judge is how the seat mesh-placement's holder judges a proposal (novox/hq ADR 0567 §3, §4, as ADR 0593 §8
// changes them): what it changes on each node by its placement (Intended), every node answering it, and the verdict of
// their answers (JudgeAnswers). The assigner's own (ADR 0593 §1), moved back from mesh-sdk's compose/judge: the holder
// composes nothing, since every composition is its node's engine's, and a node answers through the candidates path
// with what it composed. It holds no bus client and no timer: the holder gives it what it read.
package judge
import (
"encoding/json"
"fmt"
"reflect"
"sort"
"strconv"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// AnswerBound is how long the holder waits for a node's answer after its candidate was written: the time to live of the
// node's pulse key (delivery.PulseAnswerKey), whose expiry names it as giving no answer, never passed. The same answer
// bound a check gives a node (ADR 0593 §8, §7: 60 seconds, delivery.ComposeAnswerBound). A variable so a test on the bus
// can shorten it.
var AnswerBound = delivery.ComposeAnswerBound
// Intended is, by node, the modules whose placed entry the proposal changes there, and the nodes that answer it: every
// node (ADR 0593 §8, changing ADR 0567 §4's list of the nodes that must answer), sorted. current is the placement as it
// stands, laid the proposal's placements by node (nil removes the node's key) and nodes every node with a record. A
// node's set holds the modules the proposal adds, removes or whose placed entry differs on that node; a node with no
// key now gets an empty set (§7: the seed is answered as the same). What a node's compositions change beyond its own
// placement it says itself in its answer, marked in the brief as not computed.
func Intended(current map[string]delivery.Placement, laid map[string]*delivery.Placement, nodes []string) (intended map[string]map[string]bool, must []string) {
intended = map[string]map[string]bool{}
all := map[string]bool{}
for _, n := range nodes {
all[n] = true
}
for n := range laid {
all[n] = true
}
for node := range all {
must = append(must, node)
set := map[string]bool{}
intended[node] = set
before, placed := current[node]
after, changesKey := laid[node]
if !changesKey || !placed {
continue // untouched, or new to placement: every answer must be the same
}
var afterModules []delivery.PlacedModule
if after != nil {
afterModules = after.Modules
}
entries := map[string]delivery.PlacedModule{}
for _, m := range before.Modules {
entries[m.Module] = m
}
seen := map[string]bool{}
for _, m := range afterModules {
seen[m.Module] = true
if e, ok := entries[m.Module]; !ok || !reflect.DeepEqual(e, m) {
set[m.Module] = true
}
}
for name := range entries {
if !seen[name] {
set[name] = true
}
}
}
sort.Strings(must)
return intended, must
}
// JudgeAnswers is the verdict of a proposal's answers (§4). answers is by node its HaveNode.Candidates entry for the
// proposal (nil when it has none) and revisions by node the revision of its candidate key: an answer counts when it
// names the proposal and that revision. It fails on a refusal by a node that must answer; a node of Must with no answer
// that counts is waited for while its pulse key stands, and once it has ended (pulsing false: the bound's expiry, heard
// on the bus) is named as giving no answer and the proposal decided on the answers it has (ADR 0593 §8).
// A difference fails nothing: one Intended computed is listed under Intended, one it did not under Unintended, both
// shown in the ask's brief. The seed alone fails on any difference.
func JudgeAnswers(p delivery.PlacementProposal, intended map[string]map[string]bool, answers map[string]*delivery.CandidateAnswer,
revisions map[string]uint64, online map[string]bool, pulsing map[string]bool, now time.Time) delivery.PlacementVerdict {
v := delivery.PlacementVerdict{At: now.UTC(), Intended: map[string][]string{}, Unintended: map[string][]string{}, Carried: map[string][]string{},
Answers: map[string]delivery.CandidateAnswer{}}
for node, set := range intended {
for m := range set {
v.Intended[node] = append(v.Intended[node], m)
}
sort.Strings(v.Intended[node])
}
must := p.Must
if len(must) == 0 {
must = p.Nodes()
}
for _, node := range must {
a := answers[node]
if a == nil || a.Proposal != p.ID || a.Revision == 0 || a.Revision != revisions[node] {
v.Silent = append(v.Silent, node)
continue
}
v.Answers[node] = *a
if a.Refused != "" {
v.Fails = append(v.Fails, fmt.Sprintf("%s cannot compose the proposal: %s", node, a.Refused))
continue
}
for _, r := range a.Refusals {
v.Fails = append(v.Fails, fmt.Sprintf("%s refuses the proposal for %s: %s", node, r.Module, r.Why))
}
want := intended[node]
differs := func(module, words string) {
if p.Seed {
// The seed carries what its nodes found, each named in the brief for the operator who approves it.
v.Carried[node] = append(v.Carried[node], words)
return
}
if !want[module] {
v.Unintended[node] = append(v.Unintended[node], module)
}
}
for _, d := range a.Differs {
differs(d.Module, differenceWords(d))
}
for _, d := range a.Added {
differs(d.Module, "would run "+d.Module)
}
for _, m := range a.Removed {
differs(m, "would stop running "+m)
}
sort.Strings(v.Unintended[node])
}
if len(v.Intended) == 0 {
v.Intended = nil
}
if len(v.Unintended) == 0 {
v.Unintended = nil
}
if len(v.Carried) == 0 {
v.Carried = nil
}
if len(v.Answers) == 0 {
v.Answers = nil
}
var waiting, gone []string
for _, node := range v.Silent {
named := node
if online != nil && !online[node] {
named += " (offline)"
}
if pulsing[node] {
waiting = append(waiting, named)
} else {
gone = append(gone, named)
}
}
if len(waiting) > 0 {
v.Notes = append(v.Notes, fmt.Sprintf("waits for the answers of %s, each for %s from its candidate's write",
strings.Join(waiting, ", "), AnswerBound))
}
if len(gone) > 0 {
// ADR 0593 §8: decided on the answers it has; that node's own composition judges again when it applies (§9).
v.Notes = append(v.Notes, fmt.Sprintf("no answer from %s within %s of its candidate's write: decided on the answers "+
"it has, and that node's own composition judges again when it applies", strings.Join(gone, ", "), AnswerBound))
}
switch {
case len(v.Fails) > 0:
v.State = delivery.VerdictFail
case len(waiting) > 0:
v.State = delivery.VerdictWaits
default:
v.State = delivery.VerdictPass
}
return v
}
// Brief is the verdict as the ask's brief shows it, by node (§5): what the proposal changes and what each node answered
// beside it, marked, and what stays held as the node holds it today, with the proposal's why.
func Brief(p delivery.PlacementProposal, v delivery.PlacementVerdict) string {
var b strings.Builder
fmt.Fprintf(&b, "%s (proposed by %s)", p.Why, p.By)
nodes := map[string]bool{}
for n := range v.Intended {
nodes[n] = true
}
for n := range v.Unintended {
nodes[n] = true
}
for _, n := range p.Nodes() {
nodes[n] = true
}
for n, a := range v.Answers {
if len(a.Held) > 0 {
nodes[n] = true
}
}
for n := range v.Carried {
nodes[n] = true
}
names := make([]string, 0, len(nodes))
for n := range nodes {
names = append(names, n)
}
sort.Strings(names)
for _, n := range names {
fmt.Fprintf(&b, "; %s:", n)
if m := v.Intended[n]; len(m) > 0 {
fmt.Fprintf(&b, " changes %s", strings.Join(m, ", "))
} else {
b.WriteString(" composes the same")
}
if m := v.Unintended[n]; len(m) > 0 {
fmt.Fprintf(&b, "; also differs, not computed: %s", strings.Join(m, ", "))
}
if m := v.Carried[n]; len(m) > 0 {
fmt.Fprintf(&b, "; the seed carries: %s", strings.Join(m, "; "))
}
if held := v.Answers[n].Held; len(held) > 0 {
var names []string
for _, h := range held {
today := h.Today
if today == "" {
today = "held"
}
names = append(names, h.Module+" ("+today+")")
}
fmt.Fprintf(&b, "; stays as today: %s", strings.Join(names, ", "))
}
}
return b.String()
}
// differenceWords names one module's difference: the module, the first field that differs in it with its value as it
// runs and as the seed composes it, each cut to about 60 characters ("(value cut)" where a body shown is cut before the
// field), and how many fields more; or the resource and why when no body is shown.
func differenceWords(d delivery.ModuleDiffers) string {
for _, b := range d.Bodies {
if len(b.Paths) == 0 {
continue
}
path := b.Paths[0]
words := fmt.Sprintf("%s: %s %s → %s", d.Module, fieldName(b.Running, b.Composed, path),
valueAt(b.Running, path), valueAt(b.Composed, path))
if n := len(b.Paths) - 1; n > 0 {
words += fmt.Sprintf(" (and %d more fields)", n)
}
return words
}
if d.Resource != "" {
return fmt.Sprintf("%s: %s (%s)", d.Module, d.Resource, d.Why)
}
return fmt.Sprintf("%s: %s", d.Module, d.Why)
}
// shownValue bounds a value named in a brief.
const shownValue = 60
// walk follows a differing path (delivery.DifferingPaths: keys joined by dots, a key may hold dots, and a JSON value
// written in a string is read as JSON) through a body, answering the keys it passed and the value at its end.
func walk(body, path string) (keys []string, value any, ok bool) {
var v any
if json.Unmarshal([]byte(body), &v) != nil {
return nil, nil, false
}
rest := path
for rest != "" {
if s, isString := v.(string); isString {
var inner any
if json.Unmarshal([]byte(s), &inner) != nil {
return nil, nil, false
}
v = inner
}
switch t := v.(type) {
case map[string]any:
best := ""
for k := range t {
if (rest == k || strings.HasPrefix(rest, k+".")) && len(k) > len(best) {
best = k
}
}
if best == "" {
return nil, nil, false
}
keys, v, rest = append(keys, best), t[best], strings.TrimPrefix(strings.TrimPrefix(rest, best), ".")
case []any:
seg, after, _ := strings.Cut(rest, ".")
n, err := strconv.Atoi(seg)
if err != nil || n < 0 || n >= len(t) {
return nil, nil, false
}
keys, v, rest = append(keys, seg), t[n], after
default:
return nil, nil, false
}
}
return keys, v, true
}
// valueAt is the value at path in a shown body, cut to shownValue, or why it is not shown.
func valueAt(body, path string) string {
if body == "" {
return "(none)"
}
_, v, ok := walk(body, path)
if !ok {
return "(value cut)"
}
s, isString := v.(string)
if !isString {
raw, _ := json.Marshal(v)
s = string(raw)
}
if r := []rune(s); len(r) > shownValue {
s = string(r[:shownValue-1]) + "…"
}
return s
}
// fieldName is the differing field as a person reads it: its parent's name and its own, `PARENT[key]`, from whichever
// body reaches it; the path's end when neither does.
func fieldName(running, composed, path string) string {
for _, body := range []string{running, composed} {
if keys, _, ok := walk(body, path); ok && len(keys) > 0 {
if len(keys) == 1 {
return keys[0]
}
return keys[len(keys)-2] + "[" + keys[len(keys)-1] + "]"
}
}
return path
}
modules/mesh-delivery-assigner/internal/judge/judge_test.go
+91
View File
@@ -0,0 +1,91 @@
package judge
import (
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// ADR 0567 §3, §4 (rows moved from the assigner's check of a pull request's placement, ADR 0559 §5): what a proposal
// changes by node, and the verdict of the answers: differences shown and failing nothing, a refusal and a silent node
// failing, a later answer judging again, the seed failing on any difference.
func TestJudge(t *testing.T) {
current := map[string]delivery.Placement{"anchor": {Node: "anchor", Modules: []delivery.PlacedModule{{Module: "mail"}}}, "server": {Node: "server"}}
laid := map[string]*delivery.Placement{
"anchor": {Node: "anchor", Modules: []delivery.PlacedModule{{Module: "mail", Ports: map[string]int{"25": 2525}}, {Module: "web"}}},
"laptop": {Node: "laptop", Modules: []delivery.PlacedModule{{Module: "web"}}}}
// ADR 0593 §8: every node answers; what the placement changes is read from the placement alone.
got, must := Intended(current, laid, []string{"anchor", "laptop", "server"})
if !got["anchor"]["mail"] || !got["anchor"]["web"] || len(got["laptop"]) != 0 || len(got["server"]) != 0 ||
!slices.Equal(must, []string{"anchor", "laptop", "server"}) {
t.Errorf("intended = %v, must %v: want anchor's mail and web, nothing elsewhere, every node answering", got, must)
}
at := time.Date(2026, 10, 27, 2, 0, 0, 0, time.UTC)
p := delivery.PlacementProposal{ID: "p-one", By: "op", Why: "move web", Written: at, Base: map[string]uint64{"anchor": 3, "laptop": 0},
Must: []string{"anchor", "laptop"}}
revisions := map[string]uint64{"anchor": 7, "laptop": 8}
intended := map[string]map[string]bool{"anchor": {"web": true}}
same := func(rev uint64) *delivery.CandidateAnswer {
return &delivery.CandidateAnswer{Proposal: "p-one", Revision: rev, Verdict: delivery.CandidateSame, Same: []string{"mail"}}
}
differs := func(rev uint64, module string) *delivery.CandidateAnswer {
return &delivery.CandidateAnswer{Proposal: "p-one", Revision: rev, Verdict: delivery.CandidateDiffers,
Differs: []delivery.ModuleDiffers{{Module: module, Why: "the port differs"}}}
}
for _, row := range []struct {
name string
seed bool
answers map[string]*delivery.CandidateAnswer
pulsing map[string]bool
state string
words string
marked string
}{
{"all the same", false, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": same(8)}, nil, delivery.VerdictPass, "", ""},
{"a module added where intended", false, map[string]*delivery.CandidateAnswer{"anchor": {Proposal: "p-one", Revision: 7, Verdict: delivery.CandidateDiffers,
Added: []delivery.ModuleAdded{{Module: "web", Unissued: []string{"its own secret"}}}}, "laptop": same(8)}, nil, delivery.VerdictPass, "", ""},
{"a difference not computed passes, marked", false, map[string]*delivery.CandidateAnswer{"anchor": differs(7, "mail"), "laptop": same(8)}, nil, delivery.VerdictPass, "", "mail"},
{"the seed carries a difference, named", true, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": {Proposal: "p-one", Revision: 8,
Verdict: delivery.CandidateDiffers, Differs: []delivery.ModuleDiffers{{Module: "node-tools", Resource: "node-tools.runtime", Why: "the resource differs",
Bodies: []delivery.ResourceBodies{{ID: "node-tools.runtime", Paths: []string{"env.MESH_OWNED_PATHS./etc/resolv.conf", "env.MESH_TOOL_MODULES"},
Running: `{"env":{"MESH_OWNED_PATHS":"{\"/etc/hosts\":\"hostname\",\"/etc/resolv.conf\":\"networkmanager (node-uplink)\"}"}}`,
Composed: `{"env":{"MESH_OWNED_PATHS":"{\"/etc/hosts\":\"hostname\",\"/etc/resolv.conf\":\"systemd-resolved (node-resolver)\"}","MESH_TOOL_MODULES":"… (cut at 8192 bytes)`}}}}}}, nil, delivery.VerdictPass, "", ""},
{"a refusal", false, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": {Proposal: "p-one", Revision: 8, Verdict: delivery.CandidateRefused,
Refusals: []delivery.ModuleRefused{{Module: "web", Why: "no own secret"}}}}, nil, delivery.VerdictFail, "laptop refuses", ""},
{"a node not answering, within its bound", false, map[string]*delivery.CandidateAnswer{"anchor": same(7)}, map[string]bool{"laptop": true}, delivery.VerdictWaits, "laptop (offline)", ""},
{"an answer of an earlier revision, past its bound, is no answer and decided without it", false, map[string]*delivery.CandidateAnswer{"anchor": same(6), "laptop": same(8)}, nil, delivery.VerdictPass, "no answer from anchor", ""},
{"a module held as today passes, shown in the brief", false, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": {Proposal: "p-one", Revision: 8,
Verdict: delivery.CandidateSame, Held: []delivery.ModuleRefused{{Module: "spiegel", Why: "a required value is unset"}}}}, nil, delivery.VerdictPass, "", ""},
{"its later answer", false, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": same(8)}, nil, delivery.VerdictPass, "", ""},
} {
q := p
q.Seed = row.seed
v := JudgeAnswers(q, intended, row.answers, revisions, map[string]bool{"anchor": true}, row.pulsing, at)
all := strings.Join(append(append([]string{}, v.Fails...), v.Notes...), "; ")
if v.State != row.state || !strings.Contains(all, row.words) {
t.Errorf("%s: %s, %q", row.name, v.State, all)
}
if row.seed && row.state == delivery.VerdictPass && !strings.Contains(Brief(q, v), "the seed carries: node-tools: MESH_OWNED_PATHS[/etc/resolv.conf] networkmanager (node-uplink) → (value cut) (and 1 more fields)") {
t.Errorf("%s: the brief does not name it: %s", row.name, Brief(q, v))
}
if row.seed && row.state == delivery.VerdictPass {
whole := row.answers["laptop"].Differs[0]
whole.Bodies = []delivery.ResourceBodies{{ID: "r", Paths: []string{"env.MESH_OWNED_PATHS./etc/resolv.conf"},
Running: `{"env":{"MESH_OWNED_PATHS":"{\"/etc/resolv.conf\":\"networkmanager (node-uplink)\"}"}}`,
Composed: `{"env":{"MESH_OWNED_PATHS":"{\"/etc/resolv.conf\":\"systemd-resolved (node-resolver)\"}"}}`}}
if got := differenceWords(whole); got != "node-tools: MESH_OWNED_PATHS[/etc/resolv.conf] networkmanager (node-uplink) → systemd-resolved (node-resolver)" {
t.Errorf("the difference is named %q", got)
}
}
if strings.HasPrefix(row.name, "a module held as today") && !strings.Contains(Brief(q, v), "stays as today: spiegel (held)") {
t.Errorf("%s: the brief does not show it: %s", row.name, Brief(q, v))
}
if row.marked != "" && (!slices.Contains(v.Unintended["anchor"], row.marked) || !strings.Contains(Brief(q, v), "not computed: "+row.marked)) {
t.Errorf("%s: the difference is not marked: %+v, %s", row.name, v.Unintended, Brief(q, v))
}
}
}
modules/mesh-delivery-assigner/manifest_test.go
+10 -8
View File
@@ -33,9 +33,11 @@ func TestManifest(t *testing.T) {
}
reads := slices.Clone(m.Reads)
slices.Sort(reads)
want := []string{"mesh-build-go.steps", "mesh-config.kept", "mesh-config.resolved",
"mesh-delivery-builder.build-log", "mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests",
"mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-grants.grants", "mesh-provisioner.provisions", "node-engine.have"}
// ADR 0593 §8: it composes nothing, so it reads no candidate, held or resolved value, compose step or build log; it reads
// every node's published compositions to compare them.
want := []string{"mesh-delivery-builder.builds", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules",
"mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-grants.grants", "mesh-host.composed",
"mesh-provisioner.provisions", "node-engine.have"}
if !reflect.DeepEqual(reads, want) {
t.Errorf("reads = %v, want exactly %v", reads, want)
}
@@ -43,7 +45,7 @@ func TestManifest(t *testing.T) {
for _, s := range m.State {
states[s.Name] = s.History
}
if want := map[string]int{"asks": 5, "compositions": 5, "differences": 64, "puts": 20, "wants": 20,
if want := map[string]int{"asks": 5, "differences": 64, "puts": 20, "wants": 20,
"placement": 64, "proposals": 5, "candidates": 5, "pulses": 0, "findings": 5, "policies": 20, "seat-holders": 5}; !reflect.DeepEqual(states, want) {
t.Errorf("state = %v, want %v", states, want)
}
@@ -53,10 +55,10 @@ func TestManifest(t *testing.T) {
if m.SecretsOwner != "mesh-delivery-assigner" {
t.Errorf("secrets-owner = %q", m.SecretsOwner)
}
// 1906/D3: it says its conditions on the operator channel; 1622: it puts its compose asks on the Go build seat, as the
// checker puts its checks.
if !slices.Equal(m.Uses, []string{"node-build-go", "operator-channel"}) {
t.Errorf("uses = %v, want exactly node-build-go and operator-channel", m.Uses)
// 1906/D3: it says its conditions on the operator channel; it asks the Go build seat nothing since the composition is
// the node-engine's (ADR 0593 §7).
if !slices.Equal(m.Uses, []string{"operator-channel"}) {
t.Errorf("uses = %v, want exactly operator-channel", m.Uses)
}
// ADR 0546: it serves its own tools (put, release, puts) from its bundle through the tool runner (the artifact loads the
// module), says each put's step as the event put, and calls the hand-act seat and the bus node's backup holder.
modules/mesh-delivery-assigner/module.json
+1 -7
View File
@@ -24,22 +24,17 @@
}
],
"reads": [
"mesh-build-go.steps",
"mesh-config.kept",
"mesh-config.resolved",
"mesh-delivery-builder.build-log",
"mesh-delivery-builder.builds",
"mesh-delivery-indexer.candidates",
"mesh-delivery-indexer.manifests",
"mesh-delivery-indexer.modules",
"mesh-delivery-teller.answers",
"mesh-delivery-watchman.presence",
"mesh-grants.grants",
"mesh-host.composed",
"mesh-provisioner.provisions",
"node-engine.have"
],
"uses": [
"node-build-go",
"operator-channel"
],
"invokes": [
@@ -63,7 +58,6 @@
{ "name": "candidates", "history": 5 },
{ "name": "pulses", "key-ttl": true },
{ "name": "asks", "history": 5 },
{ "name": "compositions", "history": 5 },
{ "name": "differences", "history": 64 },
{ "name": "puts", "history": 20 },
{ "name": "wants", "history": 20 },
modules/mesh-delivery-builder/buildrun/asks.go
+316
View File
@@ -0,0 +1,316 @@
package buildrun
import (
"context"
"errors"
"fmt"
"sync"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// **An asker puts an ask on its build seat's stream and waits for its step** (novox/hq ADR 0525 §1, §6; issue 1887 A4,
// A5). It chooses no node: whichever build node pulls the ask writes `<node>.<ask id>` in its build module's steps state,
// and the asker reads the first such key that is not running. The builder and the checker share it.
// AskPublisher puts an encoded ask on subject, de-duplicated by its id.
type AskPublisher interface {
PublishAsk(ctx context.Context, subject, id string, data []byte) error
}
// AskWaiter watches an ask's steps: every value a key `*.<ask id>` of the build module's steps state takes, running or
// ended, on the channel it answers, until ctx ends. A watch, not a poll.
type AskWaiter interface {
WatchAsk(ctx context.Context, module, askID string) (<-chan delivery.Step, error)
}
// Asker puts asks and waits for their steps.
type Asker struct {
Publisher AskPublisher
Waiter AskWaiter
// Bound is how long a running step (a build node took the ask, its key written running) may stand still, with no new phase,
// before the asker stops waiting; a step that keeps moving is waited for up to 3 Bounds from its start (issue 1930); zero waits as long as ctx. An ask waiting in its stream is not bounded: the stream's backlog is no fault
// (ADR 0525 §6, §9), and the bound starts again when another node takes the ask.
Bound time.Duration
}
// Put puts a on its stream: the seat's accept subject of its language and kind.
func (k Asker) Put(ctx context.Context, a delivery.StepAsk) error {
raw, err := delivery.EncodeStepAsk(a)
if err != nil {
return err
}
subject, err := delivery.AskSubject(a.Language, a.Kind)
if err != nil {
return err
}
if err := k.Publisher.PublishAsk(ctx, subject, a.ID, raw); err != nil {
return fmt.Errorf("putting the %s ask %s: %w", a.Kind, a.ID, err)
}
return nil
}
// Wait waits for the step of an ask already put (a builder that started again takes over its own asks with it, never
// putting them twice) and answers it when it ended, whatever its state. The bound runs from the step's start on the node
// that took the ask, read from its running key, so an ask that waits in the stream for a free slot waits as long as ctx.
func (k Asker) Wait(ctx context.Context, a delivery.StepAsk) (delivery.Step, error) {
module, err := delivery.BuildModuleOf(a.Language)
if err != nil {
return delivery.Step{}, err
}
watch, cancel := context.WithCancel(ctx)
defer cancel()
steps, err := k.Waiter.WatchAsk(watch, module, a.ID)
if err != nil {
return delivery.Step{}, err
}
var bound, outerCap <-chan time.Time
var timer, capTimer *time.Timer
defer func() {
if timer != nil {
timer.Stop()
}
if capTimer != nil {
capTimer.Stop()
}
}()
running, since, phase, phaseAt := "", time.Time{}, "", time.Time{}
standStill := func() error {
if phase == "" {
return fmt.Errorf("the ask %s's step on %s did not end within %s of its start: it stood still, saying no phase", a.ID, running, k.Bound)
}
return fmt.Errorf("the ask %s's step on %s did not end within %s: it stood still in %q since %s",
a.ID, running, k.Bound, phase, phaseAt.UTC().Format(time.RFC3339))
}
for {
select {
case <-ctx.Done():
return delivery.Step{}, ctx.Err()
case <-bound:
return delivery.Step{}, standStill()
case <-outerCap:
return delivery.Step{}, fmt.Errorf("the ask %s's step on %s did not end within %s of its start: it kept moving (last phase %q at %s) but never ended",
a.ID, running, 3*k.Bound, phase, phaseAt.UTC().Format(time.RFC3339))
case step, ok := <-steps:
if !ok {
return delivery.Step{}, errors.New("the watch of the ask's step ended")
}
if seen := stepSeenOf(ctx); seen != nil {
seen(step)
}
if step.State != delivery.StepRunning {
return step, nil
}
if k.Bound <= 0 {
continue
}
newRun := step.Node != running || !step.Started.Equal(since)
if !newRun && step.Phase == phase && step.PhaseAt.Equal(phaseAt) {
continue
}
// A node took the ask, or took it again after its run stopped (another node, or the same node's build module
// replaced mid-run: a new start), or its step entered a phase. The bound is on standing still: it runs from the
// step's last movement (its phase's start, else its start), so a step that keeps moving is waited for, up to
// three bounds from its start (issue 1930).
running, since, phase, phaseAt = step.Node, step.Started, step.Phase, step.PhaseAt
last := step.Started
if step.PhaseAt.After(last) {
last = step.PhaseAt
}
left := k.Bound
if !last.IsZero() {
left = time.Until(last.Add(k.Bound))
}
if timer != nil {
timer.Stop()
}
timer = time.NewTimer(max(left, 0))
bound = timer.C
if newRun {
if capTimer != nil {
capTimer.Stop()
}
outer := 3 * k.Bound
if !step.Started.IsZero() {
outer = time.Until(step.Started.Add(3 * k.Bound))
}
capTimer = time.NewTimer(max(outer, 0))
outerCap = capTimer.C
}
}
}
}
type stepSeenKey struct{}
// WithStepSeen is ctx carrying seen, which Asker.Wait hands every value the ask's step key takes while it waits: the node
// that took it, each phase it says while running (delivery.Step.Phase), and its end. A try names its asks' progress so
// (issue 1890). seen runs in the waiting goroutine and must not block.
func WithStepSeen(ctx context.Context, seen func(delivery.Step)) context.Context {
return context.WithValue(ctx, stepSeenKey{}, seen)
}
func stepSeenOf(ctx context.Context) func(delivery.Step) {
seen, _ := ctx.Value(stepSeenKey{}).(func(delivery.Step))
return seen
}
// Ask puts a and waits for its step; a step that did not pass is an error naming its node, its state and its reason.
func (k Asker) Ask(ctx context.Context, a delivery.StepAsk) (delivery.Step, error) {
if err := k.Put(ctx, a); err != nil {
return delivery.Step{}, err
}
return k.Passed(k.Wait(ctx, a))
}
// Passed is step when it passed, else an error naming where it ended and why.
func (k Asker) Passed(step delivery.Step, err error) (delivery.Step, error) {
if err != nil {
return step, err
}
if step.State != delivery.StepPassed {
return step, fmt.Errorf("the ask %s on %s %s: %s", step.ID, step.Node, step.State, step.Reason)
}
return step, nil
}
// JetStreamAsks is the bus side of an Asker: a JetStream publish de-duplicated by the ask's id (Nats-Msg-Id), and a watch
// of `*.<ask id>` in the build module's steps bucket.
type JetStreamAsks struct {
JS jetstream.JetStream
}
// PublishAsk is AskPublisher's.
func (s JetStreamAsks) PublishAsk(ctx context.Context, subject, id string, data []byte) error {
msg := &nats.Msg{Subject: subject, Data: data, Header: nats.Header{}}
msg.Header.Set(jetstream.MsgIDHeader, id)
_, err := s.JS.PublishMsg(ctx, msg)
return err
}
// WatchAsk is AskWaiter's: the watch hands over what the keys hold now, so a step that ended before it began is read at
// once, and every change after.
func (s JetStreamAsks) WatchAsk(ctx context.Context, module, askID string) (<-chan delivery.Step, error) {
pattern, err := delivery.AskStepWatch(askID)
if err != nil {
return nil, err
}
kv, err := s.JS.KeyValue(ctx, delivery.StepsBucket(module))
if err != nil {
return nil, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
}
w, err := kv.Watch(ctx, pattern)
if err != nil {
return nil, err
}
out := make(chan delivery.Step)
go func() {
defer close(out)
defer w.Stop()
for {
select {
case <-ctx.Done():
return
case e, ok := <-w.Updates():
if !ok {
return
}
if e == nil || e.Operation() != jetstream.KeyValuePut {
continue
}
step, err := delivery.DecodeStep(e.Value())
if err != nil {
continue // not a step (a node's pause key never matches an ask id; anything else is not this ask's)
}
select {
case out <- step:
case <-ctx.Done():
return
}
}
}
}()
return out, nil
}
// FakeAsks is a test double of the bus for an asker's tests: each ask put is kept and, when Answer is set, its step is
// written at once with what Answer says a build node wrote (running or ended). Write writes a step later. Safe for
// concurrent use.
type FakeAsks struct {
Answer func(a delivery.StepAsk) delivery.Step
mu sync.Mutex
asks []delivery.StepAsk
steps map[string][]delivery.Step
watchers map[string][]chan delivery.Step
}
// PublishAsk is AskPublisher's.
func (f *FakeAsks) PublishAsk(_ context.Context, _, id string, data []byte) error {
a, err := delivery.DecodeStepAsk(data)
if err != nil {
return err
}
f.mu.Lock()
f.asks = append(f.asks, a)
f.mu.Unlock()
if f.Answer != nil {
f.Write(f.Answer(a))
}
return nil
}
// Write writes an ask's step, as a build node does.
func (f *FakeAsks) Write(step delivery.Step) {
f.mu.Lock()
if f.steps == nil {
f.steps, f.watchers = map[string][]delivery.Step{}, map[string][]chan delivery.Step{}
}
f.steps[step.ID] = append(f.steps[step.ID], step)
ws := append([]chan delivery.Step(nil), f.watchers[step.ID]...)
f.mu.Unlock()
for _, w := range ws {
w <- step
}
}
// End is Write: kept for the tests that end a step.
func (f *FakeAsks) End(step delivery.Step) { f.Write(step) }
// WatchAsk is AskWaiter's: what was written so far, then every write.
func (f *FakeAsks) WatchAsk(ctx context.Context, _ string, askID string) (<-chan delivery.Step, error) {
out := make(chan delivery.Step, 64)
f.mu.Lock()
if f.steps == nil {
f.steps, f.watchers = map[string][]delivery.Step{}, map[string][]chan delivery.Step{}
}
for _, s := range f.steps[askID] {
out <- s
}
f.watchers[askID] = append(f.watchers[askID], out)
f.mu.Unlock()
go func() {
<-ctx.Done()
f.mu.Lock()
ws := f.watchers[askID]
for i, w := range ws {
if w == out {
f.watchers[askID] = append(ws[:i], ws[i+1:]...)
break
}
}
f.mu.Unlock()
}()
return out, nil
}
// Asks are the asks put so far.
func (f *FakeAsks) Asks() []delivery.StepAsk {
f.mu.Lock()
defer f.mu.Unlock()
return append([]delivery.StepAsk(nil), f.asks...)
}
modules/mesh-delivery-builder/buildrun/asks_integration_test.go
+61
View File
@@ -0,0 +1,61 @@
//go:build integration
package buildrun
import (
"context"
"encoding/json"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/testbus"
)
// 1887/A4a, 1887/AC3 on the real bus: an ask put twice under one id is one message, and the asker reads the step a node
// wrote under `<node>.<ask id>`.
func TestAnAskIsPutOnceAndItsStepIsRead(t *testing.T) {
nc, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
defer nc.Close()
js, _ := jetstream.New(nc)
ctx := context.Background()
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "SEAT_NODE_BUILD_GO", Subjects: []string{"mesh.seat.node-build-go.accept.>"},
Retention: jetstream.WorkQueuePolicy}); err != nil {
t.Fatal(err)
}
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.StepsBucket("mesh-build-go")})
if err != nil {
t.Fatal(err)
}
bus := JetStreamAsks{JS: js}
k := Asker{Publisher: bus, Waiter: bus, Bound: 5 * time.Second}
a := goAsk("mvl0frv18d1960")
for i := 0; i < 2; i++ {
if err := k.Put(ctx, a); err != nil {
t.Fatal(err)
}
}
s, _ := js.Stream(ctx, "SEAT_NODE_BUILD_GO")
if info, _ := s.Info(ctx); info.State.Msgs != 1 {
t.Fatalf("%d messages for one ask put twice", info.State.Msgs)
}
go func() {
time.Sleep(100 * time.Millisecond)
key, _ := delivery.AskStepKey("ace", a.ID)
running, _ := delivery.EncodeStep(delivery.Step{ID: a.ID, Tool: "delivery ask", Node: "ace", State: delivery.StepRunning, Started: time.Now()})
_, _ = kv.Put(ctx, key, running)
passed, _ := delivery.EncodeStep(delivery.Step{ID: a.ID, Tool: "delivery ask", Node: "ace", State: delivery.StepPassed, Started: time.Now(),
Result: json.RawMessage(`{"artifacts":[]}`)})
_, _ = kv.Put(ctx, key, passed)
}()
step, err := k.Passed(k.Wait(ctx, a))
if err != nil || step.Node != "ace" {
t.Fatalf("step %+v, %v", step, err)
}
}
modules/mesh-delivery-builder/buildrun/asks_test.go
+250
View File
@@ -0,0 +1,250 @@
package buildrun
import (
"context"
"encoding/json"
"errors"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
)
func goAsk(id string) delivery.StepAsk {
return delivery.StepAsk{ID: id, Kind: delivery.AskDelivery, Language: delivery.AskGo, Asker: "mesh-delivery-builder", At: time.Now(),
Build: &delivery.BuildJob{Module: "m", Path: ".", Fetch: map[string]any{"repository": "novox/x", "commit": strings.Repeat("a", 40)},
Artifacts: []delivery.AskArtifact{{Artifact: "program"}}}}
}
// 1887/A4a: an asker puts the ask once and answers the step a build node wrote for it, naming the node; one that did not
// pass is an error naming the node and why; a wait of an ask already put puts nothing.
func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) {
f := &FakeAsks{Answer: func(a delivery.StepAsk) delivery.Step {
return delivery.Step{ID: a.ID, Node: "ace", State: delivery.StepPassed, Result: []byte(`{"artifacts":[]}`)}
}}
k := Asker{Publisher: f, Waiter: f}
var seen []delivery.Step // issue 1890: a waiter that asks to see each value of the step sees it
step, err := k.Ask(WithStepSeen(context.Background(), func(s delivery.Step) { seen = append(seen, s) }), goAsk("mvl0frv18d1957"))
if err != nil || step.Node != "ace" || len(f.Asks()) != 1 || len(seen) != 1 || seen[0].Node != "ace" {
t.Fatalf("step %+v, %v, asks %d, seen %+v", step, err, len(f.Asks()), seen)
}
f.Answer = func(a delivery.StepAsk) delivery.Step {
return delivery.Step{ID: a.ID, Node: "shanks", State: delivery.StepRefused, Reason: "go 1.27 installed"}
}
if _, err := k.Ask(context.Background(), goAsk("mvl0frv18d1958")); err == nil || !strings.Contains(err.Error(), "shanks refused: go 1.27 installed") {
t.Fatalf("a refused step: %v", err)
}
// review round 2 item 1: an ask that waits in its stream for a free slot is not bounded; once a node writes its step
// running, the step is bounded from its start, and the error names the node.
g := &FakeAsks{}
k = Asker{Publisher: g, Waiter: g, Bound: 20 * time.Millisecond}
waiting, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
defer cancel()
if _, err := k.Wait(waiting, goAsk("mvl0frv18d1959")); err != context.DeadlineExceeded {
t.Fatalf("an ask no node took: %v, want it still waiting when ctx ended", err)
}
g.Write(delivery.Step{ID: "mvl0frv18d1961", Node: "ace", State: delivery.StepRunning, Started: time.Now()})
if _, err := k.Wait(context.Background(), goAsk("mvl0frv18d1961")); err == nil || !strings.Contains(err.Error(), "on ace did not end within") {
t.Fatalf("a step that never ends: %v", err)
}
if len(g.Asks()) != 0 {
t.Fatal("a wait put an ask")
}
// Issue 1930: the bound is on a step that stands still, not on its age. A step that enters a new phase inside the bound is
// waited for until it ends; one that stays in a phase the bound fails at once, naming the phase; one that never stops moving
// ends at three bounds, said as such.
k = Asker{Publisher: g, Waiter: g, Bound: 60 * time.Millisecond}
started := time.Now()
go func() {
for i := range 8 {
g.Write(delivery.Step{ID: "mvl0frv18d1980", Node: "ace", State: delivery.StepRunning, Started: started, Phase: "p" + string(rune('a'+i)), PhaseAt: time.Now()})
time.Sleep(15 * time.Millisecond)
}
g.Write(delivery.Step{ID: "mvl0frv18d1980", Node: "ace", State: delivery.StepPassed})
}()
if step, err := k.Wait(context.Background(), goAsk("mvl0frv18d1980")); err != nil || step.State != delivery.StepPassed {
t.Fatalf("a step that kept moving past its bound: %+v, %v", step, err)
}
g.Write(delivery.Step{ID: "mvl0frv18d1981", Node: "ace", State: delivery.StepRunning, Started: time.Now(), Phase: "integration: tests", PhaseAt: time.Now()})
if _, err := k.Wait(context.Background(), goAsk("mvl0frv18d1981")); err == nil || !strings.Contains(err.Error(), `stood still in "integration: tests"`) {
t.Fatalf("a step that stood still in a phase: %v", err)
}
loop := make(chan struct{})
defer close(loop)
started = time.Now()
go func() {
for i := 0; ; i++ {
select {
case <-loop:
return
case <-time.After(20 * time.Millisecond):
g.Write(delivery.Step{ID: "mvl0frv18d1982", Node: "ace", State: delivery.StepRunning, Started: started, Phase: "q" + string(rune('a'+i%26)), PhaseAt: time.Now()})
}
}
}()
if _, err := k.Wait(context.Background(), goAsk("mvl0frv18d1982")); err == nil || !strings.Contains(err.Error(), "kept moving") {
t.Fatalf("a step that never stops moving: %v", err)
}
// Issue 734: an ask the build seat gave up on (DEAD_LETTERS keeps its copy with the ask's id) ends its wait as failed
// with the words, though a node had taken it; under NotTakenBound one no node takes ends at the bound; a step written
// first is answered as the node wrote it.
dead := &DeadLetters{}
lw := LostAskWaiter{Inner: g, Dead: dead}
k = Asker{Publisher: g, Waiter: lw, Bound: time.Minute}
g.Write(delivery.Step{ID: "mvl0frv18d1970", Node: "ace", State: delivery.StepRunning, Started: time.Now()})
given, ok := GivenUpOf(map[string][]string{"Mesh-Dead-Msg-Id": {"mvl0frv18d1970"}, "Mesh-Dead-Deliveries": {"5"},
"Mesh-Dead-Stream": {"SEAT_NODE_BUILD_GO"}, "Mesh-Dead-Gave-Up": {"2026-10-26T12:26:40Z"}})
if !ok {
t.Fatal("a dead letter's headers were not read")
}
go dead.Heard(given)
step, err = k.Wait(context.Background(), goAsk("mvl0frv18d1970"))
if err != nil || step.State != delivery.StepFailed || !IsLost(step) ||
!strings.Contains(step.Reason, LostWords+" after 5 deliveries at 2026-10-26T12:26:40Z") {
t.Fatalf("a given-up ask: %+v, %v", step, err)
}
if _, err := k.Passed(step, nil); err == nil || !strings.Contains(err.Error(), "on "+LostNode+" failed: "+LostWords) {
t.Fatalf("a given-up ask's error: %v", err)
}
step, err = k.Wait(NotTakenBound(context.Background(), 20*time.Millisecond), goAsk("mvl0frv18d1971"))
if err != nil || !IsNotTaken(step) || IsLost(step) {
t.Fatalf("an ask no node took: %+v, %v", step, err)
}
g.Write(delivery.Step{ID: "mvl0frv18d1972", Node: "g14", State: delivery.StepPassed})
if step, err = k.Wait(NotTakenBound(context.Background(), time.Minute), goAsk("mvl0frv18d1972")); err != nil || step.Node != "g14" {
t.Fatalf("a step written: %+v, %v", step, err)
}
// Review round 1 of issue 734: a hearing the bus refuses at start is deaf and says so, never holds the start up, and
// tries again on the next ask waited for; heard, it says so and hands the given-up asks on.
fol := &refusingFollower{refuse: 1}
var mu sync.Mutex
var told []Hearing
hearing := NewDeadLetterHearing(context.Background(), fol, []string{delivery.AskGo}, dead, nil,
func(h Hearing) { mu.Lock(); told = append(told, h); mu.Unlock() })
hearing.Wait()
if why := hearing.Deaf(); !strings.Contains(why, "SEAT_NODE_BUILD_GO: permissions violation") {
t.Fatalf("a refused hearing says %q", why)
}
k = Asker{Publisher: g, Waiter: LostAskWaiter{Inner: g, Dead: dead, Hearing: hearing}, Bound: time.Minute}
g.Write(delivery.Step{ID: "mvl0frv18d1973", Node: "g14", State: delivery.StepPassed})
if _, err := k.Wait(context.Background(), goAsk("mvl0frv18d1973")); err != nil {
t.Fatal(err)
}
hearing.Wait()
mu.Lock()
if hearing.Deaf() != "" || len(told) != 2 || told[0].Heard || !told[1].Heard || fol.calls() != 2 {
t.Fatalf("after an ask: deaf %q, told %+v, followed %d times", hearing.Deaf(), told, fol.calls())
}
mu.Unlock()
fol.heard(GivenUp{AskID: "mvl0frv18d1974", Deliveries: 5})
step, err = k.Wait(context.Background(), goAsk("mvl0frv18d1974"))
if err != nil || !IsLost(step) {
t.Fatalf("an ask heard given up after the hearing came back: %+v, %v", step, err)
}
// The lost check of 2026-10-28: a step whose own watch went deaf is read at the next nudge (a step of its build module
// ended) and answered; a nudge of another build module reads nothing; a watch that closes is opened again.
d := &FakeAsks{}
heard := &HeardEnds{Inner: deafWatch{}, Read: d}
k = Asker{Publisher: d, Waiter: heard, Bound: time.Minute}
d.Write(delivery.Step{ID: "mvl0frv18d1980", Node: "shanks", State: delivery.StepPassed})
got := make(chan delivery.Step, 1)
go func() { s, _ := k.Wait(context.Background(), goAsk("mvl0frv18d1980")); got <- s }()
for heard.Waiting() == 0 {
time.Sleep(time.Millisecond)
}
heard.Nudge(NodeBuildModule)
select {
case s := <-got:
t.Fatalf("a nudge of another build module answered %+v", s)
case <-time.After(20 * time.Millisecond):
}
heard.Nudge(GoBuildModule)
if s := <-got; s.State != delivery.StepPassed || s.Node != "shanks" {
t.Fatalf("a step its watch never handed over: %+v", s)
}
closing := &HeardEnds{Inner: &closingWatch{inner: d}, Read: d}
if s, err := (Asker{Publisher: d, Waiter: closing}).Wait(context.Background(), goAsk("mvl0frv18d1980")); err != nil || s.State != delivery.StepPassed {
t.Fatalf("a watch that closed once: %+v, %v", s, err)
}
for _, row := range []struct {
h *Hearing
want string
}{{nil, health.StateUnhealthy}, {&told[0], health.StateUnhealthy}, {&told[1], health.StateHealthy}} {
var raw []byte
if row.h != nil {
raw, _ = json.Marshal(row.h)
}
if a := HearingHealth(raw); a.State != row.want || a.Why == "" {
t.Errorf("health of %+v = %+v, want %s", row.h, a, row.want)
}
}
}
// Review (issue 734 path): an ask taken again by the same node (its build module replaced mid-run) is bounded from the
// new run's start, so its pass is answered after the first run's bound would have ended the wait.
func TestARedeliveryToTheSameNodeRestartsTheBound(t *testing.T) {
f := &FakeAsks{}
k := Asker{Publisher: f, Waiter: f, Bound: 60 * time.Millisecond}
const id = "mvl0frv18d1962"
first := time.Now()
f.Write(delivery.Step{ID: id, Node: "ace", State: delivery.StepRunning, Started: first})
go func() {
time.Sleep(15 * time.Millisecond)
f.Write(delivery.Step{ID: id, Node: "ace", State: delivery.StepRunning, Started: time.Now()})
time.Sleep(50 * time.Millisecond) // past the first run's bound, inside the second's
f.Write(delivery.Step{ID: id, Node: "ace", State: delivery.StepPassed, Started: time.Now()})
}()
step, err := k.Wait(context.Background(), goAsk(id))
if err != nil || step.State != delivery.StepPassed {
t.Fatalf("step %+v, %v, want the redelivered run's pass", step, err)
}
}
// refusingFollower refuses the first `refuse` follows, as the bus does before the asker's grant is composed.
type refusingFollower struct {
mu sync.Mutex
refuse int
n int
hand func(GivenUp)
}
func (f *refusingFollower) Follow(_ context.Context, _ string, _ time.Time, heard func(GivenUp)) (func(), error) {
f.mu.Lock()
defer f.mu.Unlock()
f.n++
if f.n <= f.refuse {
return nil, errors.New("permissions violation")
}
f.hand = heard
return func() {}, nil
}
func (f *refusingFollower) calls() int { f.mu.Lock(); defer f.mu.Unlock(); return f.n }
func (f *refusingFollower) heard(g GivenUp) { f.mu.Lock(); h := f.hand; f.mu.Unlock(); h(g) }
// deafWatch is a watch of an ask's step that stays open and hears nothing.
type deafWatch struct{}
func (deafWatch) WatchAsk(context.Context, string, string) (<-chan delivery.Step, error) {
return make(chan delivery.Step), nil
}
// closingWatch closes its first watch at once, and hands the later ones to inner.
type closingWatch struct {
inner AskWaiter
closed bool
}
func (c *closingWatch) WatchAsk(ctx context.Context, module, askID string) (<-chan delivery.Step, error) {
if !c.closed {
c.closed = true
ch := make(chan delivery.Step)
close(ch)
return ch, nil
}
return c.inner.WatchAsk(ctx, module, askID)
}
modules/mesh-delivery-builder/buildrun/choose.go
+88
View File
@@ -0,0 +1,88 @@
package buildrun
import (
"fmt"
"sort"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// ChooseNodes picks up to want build nodes for a language (ADR 0478 clause 3): the nodes whose toolchains key reports the
// version the nodes agree on, the ones with room under their build capacity first (places), then the fewest lanes the
// part has running or chosen there, a tie broken by the node's name. A place is reserved on each node chosen, in the
// same step as the choice, so builds choosing at once do not choose the same full node while another has room; the
// caller takes the place (Places.Acquire) or gives it up (Places.Cancel). A node at its capacity is still chosen when
// the others are too (fewer than want have room): its lane waits for a place. A node qualifies only when qualifies is true
// of it (online and its build module healthy; nil means every node does). It answers the toolchain all the reporting nodes
// agree on, qualifying or not, as the indexer computes it. When nothing is reported, or the nodes report different
// versions (so the indexer agreed none), it fails naming the language and why; when no reporting node qualifies it fails
// naming the nodes. A node whose build module refuses on its toolchain refuses the step, and the build fails naming it.
func ChooseNodes(language string, entries []delivery.NodeToolchain, places *Places, qualifies func(node string) bool, want int) ([]string, delivery.Toolchain2, error) {
tc, none := delivery.AgreedToolchain(entries)
if none != "" {
return nil, delivery.Toolchain2{}, fmt.Errorf("no %s build node qualifies: %s", language, none)
}
nodes, err := choose(language, entries, places, qualifies, want)
return nodes, tc, err
}
// ChooseAnyNodes is ChooseNodes for artifacts that need no toolchain, such as an archive (ADR 0478 clause 1): the
// build module's nodes agree on nothing, so any node of the build module that qualifies is chosen, by capacity and load only.
func ChooseAnyNodes(language string, entries []delivery.NodeToolchain, places *Places, qualifies func(node string) bool, want int) ([]string, error) {
return choose(language, entries, places, qualifies, want)
}
func choose(language string, entries []delivery.NodeToolchain, places *Places, qualifies func(node string) bool, want int) ([]string, error) {
if places == nil {
places = &Places{}
}
nodes := make([]string, 0, len(entries))
var reporting []string
seen := map[string]bool{}
for _, e := range entries {
if seen[e.Node] {
continue
}
seen[e.Node] = true
reporting = append(reporting, e.Node)
if places.CapacityOf(e.Node) > 0 && (qualifies == nil || qualifies(e.Node)) { // a capacity of 0 keeps the part off a node
nodes = append(nodes, e.Node)
}
}
if len(nodes) == 0 {
sort.Strings(reporting)
return nil, fmt.Errorf("no %s build node qualifies: %s report the toolchain, and none is online with a healthy build module and a build capacity", language, strings.Join(reporting, ", "))
}
return places.choose(nodes, want), nil
}
// DigestsDifferPrefix opens the error of a build whose nodes built an artifact to different digests (ADR 0478 clause 4):
// such a key is not built again as a new revision, since the same source on the same toolchains gave two answers and
// building again settles nothing; it waits for another fingerprint.
const DigestsDifferPrefix = "digests differ:"
// Agree is the one digest every node built an artifact to (ADR 0478 clause 4), or an error naming each node and its
// digest when they differ. One node's digest agrees with itself.
func Agree(artifact string, byNode map[string]string) (string, error) {
nodes := make([]string, 0, len(byNode))
for n := range byNode {
nodes = append(nodes, n)
}
sort.Strings(nodes)
var first string
for i, n := range nodes {
if i == 0 {
first = byNode[n]
continue
}
if byNode[n] != first {
msg := fmt.Sprintf("%s the nodes built %s to different digests:", DigestsDifferPrefix, artifact)
for _, n := range nodes {
msg += fmt.Sprintf(" %s %s;", n, byNode[n])
}
return "", fmt.Errorf("%s", msg[:len(msg)-1])
}
}
return first, nil
}
modules/mesh-delivery-builder/buildrun/choose_test.go
+109
View File
@@ -0,0 +1,109 @@
package buildrun
import (
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
const digest = "sha256:2222222222222222222222222222222222222222222222222222222222222222"
func hex64(c byte) string { return strings.Repeat(string(c), 64) }
func tcs(language, version string, nodes ...string) []delivery.NodeToolchain {
var out []delivery.NodeToolchain
for _, n := range nodes {
out = append(out, delivery.NodeToolchain{Node: n, Toolchain: delivery.Toolchain{Language: language, Version: version, Recipe: "1"}})
}
return out
}
func chooseTakesTheAgreedNodesWithFewestRunningStepsTiesByName(t *testing.T) { // 1607/AC4, ADR 0478 clause 3
entries := tcs("go", "go1.27.1", "novox", "ace", "g14", "shanks")
nodes, tc, err := ChooseNodes("go", entries, placesRunning(map[string]int{"novox": 3, "ace": 1, "g14": 1, "shanks": 0}), nil, 2)
if err != nil {
t.Fatal(err)
}
if len(nodes) != 2 || nodes[0] != "shanks" || nodes[1] != "ace" {
t.Errorf("nodes = %v, want [shanks ace]: fewest steps first, a tie by name", nodes)
}
if tc.Version != "go1.27.1" || tc.Recipe != "1" || tc.Language != "go" {
t.Errorf("toolchain = %+v", tc)
}
}
func chooseWithOneNodeAnswersOneNode(t *testing.T) { // 1607/AC4
nodes, _, err := ChooseNodes("go", tcs("go", "go1.27.1", "novox"), nil, nil, 2)
if err != nil || len(nodes) != 1 || nodes[0] != "novox" {
t.Errorf("nodes = %v, err %v, want [novox]", nodes, err)
}
}
func chooseFailsNamingTheLanguageAndTheVersionsWhenNoNodeQualifies(t *testing.T) { // 1607/AC4
_, _, err := ChooseNodes("typescript", nil, nil, nil, 2)
if err == nil || !strings.Contains(err.Error(), "typescript") || !strings.Contains(err.Error(), delivery.ToolchainNoReport) {
t.Errorf("no report: error = %v, want it to name typescript and %q", err, delivery.ToolchainNoReport)
}
entries := append(tcs("go", "go1.27.1", "novox"), tcs("go", "go1.26.0", "ace")...)
_, _, err = ChooseNodes("go", entries, nil, nil, 2)
if err == nil || !strings.Contains(err.Error(), "go1.27.1") || !strings.Contains(err.Error(), "go1.26.0") || !strings.Contains(err.Error(), "go") {
t.Errorf("disagreeing nodes: error = %v, want both versions named", err)
}
}
func agreeNeedsEqualDigestsAndNamesBothNodesWhenTheyDiffer(t *testing.T) { // 1607/AC4, ADR 0478 clause 4
d, err := Agree("tool", map[string]string{"ace": digest, "novox": digest})
if err != nil || d != digest {
t.Errorf("Agree = %q, %v, want %q", d, err, digest)
}
_, err = Agree("tool", map[string]string{"ace": digest, "novox": "sha256:" + hex64('9')})
if err == nil || !strings.Contains(err.Error(), "ace") || !strings.Contains(err.Error(), "novox") || !strings.Contains(err.Error(), digest) || !strings.Contains(err.Error(), hex64('9')) || !strings.Contains(err.Error(), "tool") {
t.Errorf("differing digests: error = %v, want the artifact, both nodes and both digests", err)
}
if d, err := Agree("tool", map[string]string{"ace": digest}); err != nil || d != digest {
t.Errorf("one node: %q, %v", d, err)
}
}
func chooseSkipsNodesThatAreNotOnlineAndFailsNamingThemWhenNoneIs(t *testing.T) { // 1607 review 3, ADR 0478 clause 3
entries := tcs("go", "go1.27.1", "ace", "g14", "novox")
online := func(n string) bool { return n != "g14" } // a sleeping laptop
nodes, tc, err := ChooseNodes("go", entries, placesRunning(map[string]int{"novox": 5}), online, 2)
if err != nil || len(nodes) != 2 || nodes[0] != "ace" || nodes[1] != "novox" || tc.Version != "go1.27.1" {
t.Errorf("nodes = %v, toolchain %+v, err %v, want [ace novox] with g14 left out although it runs nothing", nodes, tc, err)
}
_, tc, err = ChooseNodes("go", entries, nil, func(string) bool { return false }, 2)
if err == nil || !strings.Contains(err.Error(), "online") || !strings.Contains(err.Error(), "g14") || !strings.Contains(err.Error(), "go") {
t.Errorf("none online: error = %v, want it to name the nodes that report and that none is online", err)
}
if tc.Version != "go1.27.1" {
t.Errorf("none online: the agreed toolchain %+v is still answered", tc)
}
}
// placesRunning is Places with these steps counted as running, for a test of the choice.
func placesRunning(running map[string]int) *Places {
return &Places{running: running, changed: make(chan struct{})}
}
func choosePrefersNodesBelowTheirCapacity(t *testing.T) { // round 3
entries := tcs("go", "go1.27.1", "ace", "novox", "shanks")
// shanks has capacity 3 with 2 running: room; ace has the default 1 with 1 running: none; novox has none running.
p := placesRunning(map[string]int{"shanks": 2, "ace": 1})
p.Capacity = map[string]int{"shanks": 3}
nodes, _, err := ChooseNodes("go", entries, p, nil, 2)
if err != nil || len(nodes) != 2 || nodes[0] != "novox" || nodes[1] != "shanks" {
t.Errorf("nodes = %v, err %v, want [novox shanks]: the nodes with room, ace at capacity left out", nodes, err)
}
}
// The choice of the nodes a step runs on and the agreement of their answers, one table of cases (the test budget).
func TestChoosingTheNodesOfAStep(t *testing.T) {
t.Run("choose takes the agreed nodes with fewest running steps ties by name", chooseTakesTheAgreedNodesWithFewestRunningStepsTiesByName)
t.Run("choose with one node answers one node", chooseWithOneNodeAnswersOneNode)
t.Run("choose fails naming the language and the versions when no node qualifies", chooseFailsNamingTheLanguageAndTheVersionsWhenNoNodeQualifies)
t.Run("agree needs equal digests and names both nodes when they differ", agreeNeedsEqualDigestsAndNamesBothNodesWhenTheyDiffer)
t.Run("choose skips nodes that are not online and fails naming them when none is", chooseSkipsNodesThatAreNotOnlineAndFailsNamingThemWhenNoneIs)
t.Run("choose prefers nodes below their capacity", choosePrefersNodesBelowTheirCapacity)
}
modules/mesh-delivery-builder/buildrun/doc.go
+5
View File
@@ -0,0 +1,5 @@
// Package buildrun holds the parts every module that runs steps on the build modules shares: the caller, the steps
// watch and the runner of a long step (ADR 0469), the sets of presence, toolchains and healthy build modules, the
// places a node offers by capacity, and the choice of build nodes (ADR 0478). The builder and the checker of the
// delivery use them; they moved here from the builder's own package so neither keeps a copy (issue 1671, ADR 0484).
package buildrun
modules/mesh-delivery-builder/buildrun/go.mod
+22
View File
@@ -0,0 +1,22 @@
module git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
github.com/nats-io/nats.go v1.54.0
)
require (
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
github.com/google/go-tpm v0.9.8 // indirect
github.com/klauspost/compress v1.20.0 // indirect
github.com/minio/highwayhash v1.0.4 // indirect
github.com/nats-io/jwt/v2 v2.8.1 // indirect
github.com/nats-io/nats-server/v2 v2.11.17 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/time v0.15.0 // indirect
)
modules/mesh-delivery-builder/buildrun/go.sum
+27
View File
@@ -0,0 +1,27 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
modules/mesh-delivery-builder/buildrun/have.go
+56
View File
@@ -0,0 +1,56 @@
package buildrun
import (
"sync"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// HaveSet holds what each node's node-engine says of each module on it (node-engine_have, key `<node>.<module>`): the
// builder reads it to know whether a node's build module is running and healthy, since a build module whose installed
// toolchain is not the one its recipe builds with is unhealthy (mesh-build-go build_go_health) and its steps would
// refuse (ADR 0478 clause 3). A module the node-engine has written no key of is not healthy. Safe for use from the watch
// and from a build at once.
type HaveSet struct {
mu sync.Mutex
healthy map[string]bool // by `<node>.<module>`
}
// Put takes a change of the have bucket in and answers whether a module's healthy state changed. A node's own key
// (`<node>.node`) and a value that is not a have are left out.
func (s *HaveSet) Put(key string, value []byte, deleted bool) bool {
node, module, isNode, err := delivery.ParseHaveKey(key)
if err != nil || isNode {
return false
}
ok := false
if !deleted {
h, err := delivery.DecodeHave(value)
if err != nil {
return false
}
ok = h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK
}
k := node + "." + module
s.mu.Lock()
defer s.mu.Unlock()
if s.healthy[k] == ok {
return false
}
if s.healthy == nil {
s.healthy = map[string]bool{}
}
if ok {
s.healthy[k] = true
} else {
delete(s.healthy, k)
}
return true
}
// Healthy is whether the module on the node was last seen running and healthy.
func (s *HaveSet) Healthy(node, module string) bool {
s.mu.Lock()
defer s.mu.Unlock()
return s.healthy[node+"."+module]
}
modules/mesh-delivery-builder/buildrun/heardends.go
+236
View File
@@ -0,0 +1,236 @@
package buildrun
import (
"context"
"fmt"
"sync"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// **A step's end is never only its own watch's to hear** (the lost check of 2026-10-28: mesh-catalog #553's verdict
// stayed running after its step mvsgthyjed5786 had passed on shanks at 19:09:31Z; the watch of that one key, opened at
// 18:46:05Z, heard neither the step's start nor its end and never closed, while a watch opened at 18:58:48Z heard its own
// step end). HeardEnds stands between the asker (Asker) and the watch of each ask's step key: when it is
// nudged, every ask it waits on reads its step keys now (a read, not the watch) and hands what it reads to its wait, so a
// step that ended is judged even when its own watch went deaf. A watch that closes under a wait is opened again, at most
// reopenTries times. It is nudged on an event (Nudge): any step of the same build module ending (one watch of each steps
// state, HearEnds), a reconnect to the bus, and whatever else the asker hears (the checker: checker_ask_again).
// It holds no timer. Every asker of the build seats waits through it (the checker and the builder, HearEnds).
// StepReader reads, now, every step a build module's steps state holds for an ask: its keys `*.<ask id>`.
type StepReader interface {
AskSteps(ctx context.Context, module, askID string) ([]delivery.Step, error)
}
// reopenTries is how often a wait opens its ask's watch again after it closed, before the wait ends failed.
const reopenTries = 3
// HeardEnds is an AskWaiter over Inner that also reads an ask's steps when nudged. Safe for concurrent use.
type HeardEnds struct {
Inner AskWaiter
Read StepReader
Logf func(format string, args ...any)
mu sync.Mutex
waiting map[string]*waitedAsk // by ask id
}
type waitedAsk struct {
module string
nudge chan struct{}
}
func (h *HeardEnds) logf(format string, args ...any) {
if h.Logf != nil {
h.Logf(format, args...)
}
}
// WatchAsk is AskWaiter's: what Inner's watch hands, and, at each nudge, what the ask's keys hold now.
func (h *HeardEnds) WatchAsk(ctx context.Context, module, askID string) (<-chan delivery.Step, error) {
in, err := h.Inner.WatchAsk(ctx, module, askID)
if err != nil {
return nil, err
}
w := &waitedAsk{module: module, nudge: make(chan struct{}, 1)}
h.mu.Lock()
if h.waiting == nil {
h.waiting = map[string]*waitedAsk{}
}
h.waiting[askID] = w
h.mu.Unlock()
out := make(chan delivery.Step)
go func() {
defer close(out)
defer func() {
h.mu.Lock()
if h.waiting[askID] == w {
delete(h.waiting, askID)
}
h.mu.Unlock()
}()
send := func(s delivery.Step) bool {
select {
case out <- s:
return true
case <-ctx.Done():
return false
}
}
reopened := 0
for {
select {
case <-ctx.Done():
return
case s, ok := <-in:
if !ok {
if ctx.Err() != nil || reopened >= reopenTries {
h.logf("the watch of the ask %s's step on %s closed %d times; its wait ends", askID, module, reopened+1)
return
}
reopened++
if in, err = h.Inner.WatchAsk(ctx, module, askID); err != nil {
h.logf("the watch of the ask %s's step on %s closed and cannot be opened again, so its wait ends: %v", askID, module, err)
return
}
h.logf("the watch of the ask %s's step on %s closed; opened again (%d of %d)", askID, module, reopened, reopenTries)
continue
}
if !send(s) {
return
}
case <-w.nudge:
if h.Read == nil {
continue
}
steps, err := h.Read.AskSteps(ctx, module, askID)
if err != nil {
h.logf("the steps of the ask %s on %s cannot be read: %v", askID, module, err)
continue
}
for _, s := range steps {
if s.State != delivery.StepRunning {
h.logf("the ask %s's step on %s ended %s, read at a nudge: its watch had not handed it over", askID, s.Node, s.State)
}
if !send(s) {
return
}
}
}
}
}()
return out, nil
}
// Nudge has every ask waited on of module read its steps now; module "" nudges every ask.
func (h *HeardEnds) Nudge(module string) {
h.mu.Lock()
defer h.mu.Unlock()
for _, w := range h.waiting {
if module == "" || w.module == module {
select {
case w.nudge <- struct{}{}:
default: // a nudge is pending already
}
}
}
}
// Waiting is how many asks are waited on now.
func (h *HeardEnds) Waiting() int {
h.mu.Lock()
defer h.mu.Unlock()
return len(h.waiting)
}
// AskSteps is StepReader's: the keys `*.<ask id>` of the build module's steps state, read now.
func (s JetStreamAsks) AskSteps(ctx context.Context, module, askID string) ([]delivery.Step, error) {
pattern, err := delivery.AskStepWatch(askID)
if err != nil {
return nil, err
}
kv, err := s.JS.KeyValue(ctx, delivery.StepsBucket(module))
if err != nil {
return nil, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
}
lister, err := kv.ListKeysFiltered(ctx, pattern)
if err != nil {
return nil, err
}
var out []delivery.Step
for key := range lister.Keys() {
e, err := kv.Get(ctx, key)
if err != nil {
continue // deleted between the list and the read
}
if step, err := delivery.DecodeStep(e.Value()); err == nil {
out = append(out, step)
}
}
return out, nil
}
// HearEnds is the HeardEnds every asker of the build seats waits through: over inner (the asks' own watches, or a
// LostAskWaiter's inner), reading with asks, and nudged at each step end of each of modules (one watch of each steps state,
// updates only; one that closes is opened again, and said and left after reopenTries closes in a row that heard nothing). The asker nudges it at a reconnect too (Nudge("")).
func HearEnds(ctx context.Context, asks JetStreamAsks, inner AskWaiter, modules []string, logf func(string, ...any)) *HeardEnds {
if logf == nil {
logf = func(string, ...any) {}
}
h := &HeardEnds{Inner: inner, Read: asks, Logf: logf}
for _, module := range modules {
kv, err := asks.JS.KeyValue(ctx, delivery.StepsBucket(module))
if err != nil {
logf("the steps of %s cannot be read (%v): an ask whose own watch goes deaf is read only at a reconnect", module, err)
continue
}
go h.follow(ctx, kv, module)
}
return h
}
func (h *HeardEnds) follow(ctx context.Context, kv jetstream.KeyValue, module string) {
silent := 0 // closes in a row having heard nothing
for ctx.Err() == nil {
w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly())
if err != nil {
h.logf("the steps of %s cannot be watched, so an ask whose own watch goes deaf is read only at a reconnect: %v", module, err)
return
}
heard := false
for e := range w.Updates() {
if e == nil || e.Operation() != jetstream.KeyValuePut {
continue
}
heard = true
if step, err := delivery.DecodeStep(e.Value()); err == nil && step.State != delivery.StepRunning {
h.Nudge(module)
}
}
_ = w.Stop()
if ctx.Err() != nil {
return
}
h.Nudge(module) // a step may have ended while the watch was closing
if heard {
silent = 0
} else if silent++; silent >= reopenTries {
h.logf("the watch of the steps of %s closed %d times in a row having heard nothing, so it is left: an ask whose own watch goes deaf is read only at a reconnect", module, silent)
return
}
h.logf("the watch of the steps of %s closed; opened again", module)
}
}
// AskSteps is StepReader's: the last step written for the ask, if any.
func (f *FakeAsks) AskSteps(_ context.Context, _ string, askID string) ([]delivery.Step, error) {
f.mu.Lock()
defer f.mu.Unlock()
if s := f.steps[askID]; len(s) > 0 {
return []delivery.Step{s[len(s)-1]}, nil
}
return nil, nil
}
modules/mesh-delivery-builder/buildrun/lostask.go
+503
View File
@@ -0,0 +1,503 @@
package buildrun
import (
"context"
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/process"
)
// **An ask the build seat gave up on ends its step** (novox/hq issue 734). A build seat's worker offers an ask
// delivery.AskMaxDeliver times; one handed out that often without an answer (a bus reload that drops every hand-out, a node
// that dies taking it each time) is given up by the server, and the controller keeps a copy of it in DEAD_LETTERS under
// mesh.events.dead.<seat stream>.<worker>, with the ask's message id (its ask id), how often it was handed out and when it
// was given up. No build node writes a step for it, so an asker that waits for the step waits for ever: a try stays
// running and holds a try place, a check's verdict stays running and its pull request waits.
//
// An asker of the build seats (the checker, the builder) hears those copies (DeadLetterHearing: an ordered consumer on
// DEAD_LETTERS filtered to the build seats' streams, granted to a user of a seat pulled by kind, buscompose
// DeadLetterReads) into DeadLetters, and LostAskWaiter ends the waiting step at once as failed on the node LostNode, its
// reason saying LostWords; the asker decides what follows (the checker ends a try as error and asks a check again, the
// builder asks its part of a build again).
//
// Belt and braces, where the asker says so in the context (NotTakenBound; the checker's tries): an ask no build node took
// within a bound ends the same way, saying NotTakenWords. An ask waits in the stream as long as the backlog is otherwise
// (ADR 0525 §9): a backlog is no fault.
// LostNode is the node a step ended for a lost ask names: no build node wrote it.
const LostNode = "bus"
// The words of a step ended for its ask.
const (
LostWords = "its ask was given up by the build seat"
NotTakenWords = "no build node took its ask"
)
// The headers the controller keeps a given-up message's copy with in DEAD_LETTERS (mesh-controller internal/link
// deadletters.go): the original message id (an ask's is its ask id), how often it was handed out, when it was given up.
const (
DeadHeaderMsgID = "Mesh-Dead-Msg-Id"
DeadHeaderDeliveries = "Mesh-Dead-Deliveries"
DeadHeaderGaveUp = "Mesh-Dead-Gave-Up"
DeadHeaderStream = "Mesh-Dead-Stream"
)
// DeadLettersStream is the stream the controller keeps given-up messages in, and DeadLetterFilter the subjects of one seat
// stream's copies there (buscompose DeadLetterSubject).
const DeadLettersStream = "DEAD_LETTERS"
// DeadLetterFilter is the subjects DEAD_LETTERS keeps a seat stream's given-up messages under.
func DeadLetterFilter(stream string) string { return "mesh.events.dead." + stream + ".>" }
// SeatStream is the stream of a build seat's language (buscompose seatStreamName of node-build-<language>).
func SeatStream(language string) string { return "SEAT_NODE_BUILD_" + strings.ToUpper(language) }
// GivenUp is one ask a build seat gave up on, as DEAD_LETTERS keeps it.
type GivenUp struct {
AskID string
Stream string
Deliveries int
At time.Time
}
// GivenUpOf reads a DEAD_LETTERS copy's headers; false when it names no message id.
func GivenUpOf(header map[string][]string) (GivenUp, bool) {
get := func(k string) string {
if v := header[k]; len(v) > 0 {
return strings.TrimSpace(v[0])
}
return ""
}
g := GivenUp{AskID: get(DeadHeaderMsgID), Stream: get(DeadHeaderStream)}
if g.AskID == "" {
return GivenUp{}, false
}
g.Deliveries, _ = strconv.Atoi(get(DeadHeaderDeliveries))
g.At, _ = time.Parse(time.RFC3339Nano, get(DeadHeaderGaveUp))
return g, true
}
// Words are what a step ended for the given-up ask says.
func (g GivenUp) Words() string {
at := "at an unknown time"
if !g.At.IsZero() {
at = "at " + g.At.UTC().Format(time.RFC3339)
}
return fmt.Sprintf("%s after %d deliveries %s (kept in %s); ask again", LostWords, g.Deliveries, at, DeadLettersStream)
}
// keptGivenUp is how many given-up asks DeadLetters remembers for a waiter that starts after the copy was heard.
const keptGivenUp = 1024
// DeadLetters holds the given-up asks heard, and hands each to the waiter of its ask. Safe for concurrent use.
type DeadLetters struct {
mu sync.Mutex
heard map[string]GivenUp
order []string
watchers map[string][]chan GivenUp
}
// Heard takes one given-up ask in.
func (d *DeadLetters) Heard(g GivenUp) {
d.mu.Lock()
if d.heard == nil {
d.heard, d.watchers = map[string]GivenUp{}, map[string][]chan GivenUp{}
}
if _, ok := d.heard[g.AskID]; !ok {
d.order = append(d.order, g.AskID)
if len(d.order) > keptGivenUp {
delete(d.heard, d.order[0])
d.order = d.order[1:]
}
}
d.heard[g.AskID] = g
ws := d.watchers[g.AskID]
delete(d.watchers, g.AskID)
d.mu.Unlock()
for _, w := range ws {
w <- g // buffered, one each
}
}
// watch answers a channel that gets the ask's given-up copy once (at once when it was heard already), and its cancel.
func (d *DeadLetters) watch(askID string) (<-chan GivenUp, func()) {
ch := make(chan GivenUp, 1)
d.mu.Lock()
defer d.mu.Unlock()
if d.heard == nil {
d.heard, d.watchers = map[string]GivenUp{}, map[string][]chan GivenUp{}
}
if g, ok := d.heard[askID]; ok {
ch <- g
return ch, func() {}
}
d.watchers[askID] = append(d.watchers[askID], ch)
return ch, func() {
d.mu.Lock()
defer d.mu.Unlock()
ws := d.watchers[askID]
for i, w := range ws {
if w == ch {
d.watchers[askID] = append(ws[:i], ws[i+1:]...)
break
}
}
if len(d.watchers[askID]) == 0 {
delete(d.watchers, askID)
}
}
}
type notTakenKey struct{}
// NotTakenBound says, in ctx, that an ask no build node took within bound ends its step (a try's, never a check's).
func NotTakenBound(ctx context.Context, bound time.Duration) context.Context {
return context.WithValue(ctx, notTakenKey{}, bound)
}
func notTakenBoundOf(ctx context.Context) time.Duration {
d, _ := ctx.Value(notTakenKey{}).(time.Duration)
return d
}
// LostAskWaiter is an AskWaiter that ends an ask's step when the build seat gave the ask up (DeadLetters), or, under
// NotTakenBound, when no build node took it within the bound: it hands the asker a failed step whose reason says why, on
// the node "bus". Every step the build module writes is handed on as it is.
type LostAskWaiter struct {
Inner AskWaiter
Dead *DeadLetters
// Hearing, when set, is asked to hear DEAD_LETTERS again each time an ask is waited for while it cannot: an ask put is
// the event that makes hearing matter.
Hearing interface {
Deaf() string
Again()
}
Now func() time.Time
}
func (w LostAskWaiter) now() time.Time {
if w.Now != nil {
return w.Now()
}
return time.Now()
}
// WatchAsk is AskWaiter's.
func (w LostAskWaiter) WatchAsk(ctx context.Context, module, askID string) (<-chan delivery.Step, error) {
if w.Hearing != nil && w.Hearing.Deaf() != "" {
w.Hearing.Again()
}
inner, err := w.Inner.WatchAsk(ctx, module, askID)
if err != nil {
return nil, err
}
var dead <-chan GivenUp
cancel := func() {}
if w.Dead != nil {
dead, cancel = w.Dead.watch(askID)
}
// The not-taken bound is a context's (ADR 0437 §8: no timer), ended when a node takes the ask.
var notTaken <-chan struct{}
bound := notTakenBoundOf(ctx)
stopBound := func() {}
if bound > 0 {
var nt context.Context
nt, stopBound = context.WithTimeout(context.Background(), bound)
notTaken = nt.Done()
}
out := make(chan delivery.Step)
go func() {
defer close(out)
defer cancel()
defer func() { stopBound() }()
send := func(s delivery.Step) bool {
select {
case out <- s:
return true
case <-ctx.Done():
return false
}
}
ended := func(reason string) delivery.Step {
at := w.now().UTC()
return delivery.Step{ID: askID, Node: LostNode, State: delivery.StepFailed, Reason: reason, Ended: at}
}
for {
select {
case <-ctx.Done():
return
case s, ok := <-inner:
if !ok {
return
}
if notTaken != nil {
stopBound() // a node took it: its step is bounded by the asker from its start
notTaken = nil
}
if !send(s) {
return
}
case g := <-dead:
dead = nil
send(ended(g.Words()))
return
case <-notTaken:
send(ended(fmt.Sprintf("%s within %s of its put; it was lost or waits behind a backlog; ask again", NotTakenWords, bound)))
return
}
}
}()
return out, nil
}
// IsLost says whether a step was ended by LostAskWaiter because the build seat gave its ask up (LostWords).
func IsLost(s delivery.Step) bool { return s.Node == LostNode && strings.Contains(s.Reason, LostWords) }
// IsNotTaken says whether a step was ended by LostAskWaiter because no build node took its ask within the bound.
func IsNotTaken(s delivery.Step) bool {
return s.Node == LostNode && strings.Contains(s.Reason, NotTakenWords)
}
// **The asker hears DEAD_LETTERS, tries again when it cannot, and says whether it hears** (issue 734). The read is
// granted with the asker's seat traffic (buscompose DeadLetterReads); an asker started before its grant was composed is
// refused, and an asker that only tried once would stay deaf until it starts again, silently. DeadLetterHearing tries in
// the background, never holding the start up, and again on an event (Again): the asker's reconnect to the bus, and every
// ask waited for while it is deaf (LostAskWaiter.Hearing); no timer. Each change of whether it hears is said (Said), so
// the module's health tool says it: a deaf asker is not healthy.
// Hearing is what a DeadLetterHearing says: whether it hears every build seat stream it follows, and why not.
type Hearing struct {
Heard bool `json:"heard"`
Streams []string `json:"streams"`
Why string `json:"why,omitempty"`
At time.Time `json:"at"`
}
// HearingKey is the key a module keeps its Hearing under, in its own state (the checker's and the builder's hearing).
const HearingKey = "dead-letters"
// DeadLetterFollower starts following one seat stream's copies in DEAD_LETTERS from since, handing each given-up ask to
// heard; it answers the stop, or why it cannot.
type DeadLetterFollower interface {
Follow(ctx context.Context, stream string, since time.Time, heard func(GivenUp)) (func(), error)
}
// JetStreamDeadLetters is the bus side of a DeadLetterFollower: an ordered consumer on DEAD_LETTERS filtered to the
// stream's copies, which resumes where it was after a reconnect and, reset before it heard any, starts at since again.
type JetStreamDeadLetters struct{ JS jetstream.JetStream }
// Follow is DeadLetterFollower's.
func (j JetStreamDeadLetters) Follow(ctx context.Context, stream string, since time.Time, heard func(GivenUp)) (func(), error) {
cons, err := j.JS.OrderedConsumer(ctx, DeadLettersStream, jetstream.OrderedConsumerConfig{
FilterSubjects: []string{DeadLetterFilter(stream)}, DeliverPolicy: jetstream.DeliverByStartTimePolicy, OptStartTime: &since})
if err != nil {
return nil, err
}
cc, err := cons.Consume(func(m jetstream.Msg) {
if g, ok := GivenUpOf(m.Headers()); ok {
heard(g)
}
})
if err != nil {
return nil, err
}
return cc.Stop, nil
}
// DeadLetterHearing follows the build seats' given-up asks into Dead. Make it with NewDeadLetterHearing.
type DeadLetterHearing struct {
follower DeadLetterFollower
streams []string
dead *DeadLetters
logf func(string, ...any)
said func(Hearing)
ctx context.Context
since time.Time
mu sync.Mutex
stops map[string]func()
why map[string]string
trying bool
again bool
told *Hearing
wg sync.WaitGroup
}
// NewDeadLetterHearing starts hearing the given-up asks of each language's build seat stream, in the background. said
// is told each change of whether it hears (nil tells nothing); ctx ends it.
func NewDeadLetterHearing(ctx context.Context, follower DeadLetterFollower, languages []string, dead *DeadLetters,
logf func(string, ...any), said func(Hearing)) *DeadLetterHearing {
h := &DeadLetterHearing{follower: follower, dead: dead, logf: logf, said: said, ctx: ctx, since: time.Now(),
stops: map[string]func(){}, why: map[string]string{}}
for _, l := range languages {
s := SeatStream(l)
h.streams = append(h.streams, s)
h.why[s] = "not tried yet"
}
if h.logf == nil {
h.logf = func(string, ...any) {}
}
h.Again()
return h
}
// Deaf says why a stream is not heard, or "" when every one is.
func (h *DeadLetterHearing) Deaf() string {
h.mu.Lock()
defer h.mu.Unlock()
return h.deafLocked()
}
func (h *DeadLetterHearing) deafLocked() string {
var out []string
for _, s := range h.streams {
if why, deaf := h.why[s]; deaf {
out = append(out, s+": "+why)
}
}
if len(out) == 0 {
return ""
}
return DeadLettersStream + " is not heard, so an ask a build seat gives up on is waited for until its bound: " +
strings.Join(out, "; ")
}
// Again tries, in the background, every stream not heard yet; a call while it tries makes it try once more after.
func (h *DeadLetterHearing) Again() {
h.mu.Lock()
if h.trying {
h.again = true
h.mu.Unlock()
return
}
h.trying = true
h.mu.Unlock()
h.wg.Add(1)
go func() {
defer h.wg.Done()
for {
h.try()
h.mu.Lock()
if !h.again || h.ctx.Err() != nil {
h.trying = false
h.mu.Unlock()
return
}
h.again = false
h.mu.Unlock()
}
}()
}
// Wait returns when no try runs.
func (h *DeadLetterHearing) Wait() { h.wg.Wait() }
// Stop ends every stream's following.
func (h *DeadLetterHearing) Stop() {
h.mu.Lock()
defer h.mu.Unlock()
for s, stop := range h.stops {
stop()
delete(h.stops, s)
}
}
func (h *DeadLetterHearing) try() {
for _, s := range h.streams {
h.mu.Lock()
_, deaf := h.why[s]
h.mu.Unlock()
if !deaf || h.ctx.Err() != nil {
continue
}
stream := s
stop, err := h.follower.Follow(h.ctx, stream, h.since, func(g GivenUp) {
h.logf("the ask %s on %s was given up after %d deliveries at %s: the wait for its step ends", g.AskID, g.Stream,
g.Deliveries, g.At.Format(time.RFC3339))
h.dead.Heard(g)
})
h.mu.Lock()
if err != nil {
h.why[stream] = err.Error()
h.logf("the asks %s gives up on are not heard (%s of %s); tried again at the next reconnect or ask: %v",
stream, DeadLetterFilter(stream), DeadLettersStream, err)
} else {
delete(h.why, stream)
h.stops[stream] = stop
h.logf("the asks %s gives up on are heard from %s", stream, DeadLettersStream)
}
h.mu.Unlock()
}
h.tell()
}
// tell says the hearing when it changed.
func (h *DeadLetterHearing) tell() {
h.mu.Lock()
why := h.deafLocked()
now := Hearing{Heard: why == "", Streams: append([]string(nil), h.streams...), Why: why, At: time.Now().UTC()}
same := h.told != nil && h.told.Heard == now.Heard && h.told.Why == now.Why
if !same {
h.told = &now
}
h.mu.Unlock()
if !same && h.said != nil {
h.said(now)
}
}
// HearingState is the name of the state an asker keeps its Hearing in (`<module>_hearing`, key HearingKey), which its
// health tool reads.
const HearingState = "hearing"
// HearingKeeper is where a Hearing is kept: a jetstream.KeyValue, or a process.AwaitedBucket for a state that may not be
// on the bus at the part's start (issue 1790), which keeps the latest until it appears and answers process.ErrNotYet.
type HearingKeeper interface {
Put(ctx context.Context, key string, value []byte) (uint64, error)
}
// SayHearing is a DeadLetterHearing's said that keeps each Hearing in kv under HearingKey; nil kv keeps nothing and
// says so once. A write kv keeps for later (process.ErrNotYet) is not an error: the awaited bucket said it waits.
func SayHearing(ctx context.Context, kv HearingKeeper, logf func(string, ...any)) func(Hearing) {
if kv == nil {
if logf != nil {
logf("whether %s is heard is not kept: the state %s is not there, so the health tool cannot say it", DeadLettersStream, HearingState)
}
return nil
}
return func(h Hearing) {
data, err := json.Marshal(h)
if err == nil {
_, err = kv.Put(ctx, HearingKey, data)
}
if err != nil && !errors.Is(err, process.ErrNotYet) && logf != nil {
logf("whether %s is heard could not be kept: %v", DeadLettersStream, err)
}
}
}
// HearingHealth is the health answer of an asker's kept Hearing (raw, empty when none is kept): healthy while it hears
// DEAD_LETTERS, unhealthy saying why while it does not or has not said.
func HearingHealth(raw []byte) health.Answer {
if len(raw) == 0 {
return health.Unhealthy("it has not said whether it hears " + DeadLettersStream + " (issue 734): it is starting, or its state " + HearingState + " is not there")
}
var h Hearing
if err := json.Unmarshal(raw, &h); err != nil {
return health.Unhealthy("what it keeps of hearing " + DeadLettersStream + " is not a hearing: " + err.Error())
}
if !h.Heard {
return health.Unhealthy(h.Why + " (since " + h.At.Format(time.RFC3339) + "; tried again at its next reconnect or ask)")
}
return health.Healthy("it hears " + DeadLettersStream + " for " + strings.Join(h.Streams, ", "))
}
modules/mesh-delivery-builder/buildrun/modules.go
+7
View File
@@ -0,0 +1,7 @@
package buildrun
// The build modules the mesh runs, by the language each builds (ADR 0469).
const (
GoBuildModule = "mesh-build-go"
NodeBuildModule = "mesh-build-node"
)
modules/mesh-delivery-builder/buildrun/places.go
+145
View File
@@ -0,0 +1,145 @@
package buildrun
import (
"context"
"fmt"
"sort"
"sync"
)
// DefaultCapacity is the build capacity of a node that has no setting: one build step at a time.
const DefaultCapacity = 1
// Places counts the build steps a part runs on each node and holds a lane back while its node is at its
// capacity (the operator's decision of 2026-10-21; ADR 0478 clause 3). A lane is the fetch and the builds of one build
// module on one node, run one step at a time, so a lane holds one place while it runs. A node is chosen for a lane by
// reserving a place on it at the moment of the choice (ChooseNodes), so that builds choosing at once see each other's
// choices and none waits on a full node while another has room. A lane waits for its place by watching the ends of the
// other lanes, not by a timer: ending a lane wakes the waiting ones.
type Places struct {
// Capacity is the number of steps at once by node; a node not in it has DefaultCapacity.
Capacity map[string]int
mu sync.Mutex
running map[string]int // lanes holding a place
reserved map[string]int // lanes chosen for the node that do not hold a place yet
changed chan struct{} // closed, and replaced, each time a lane ends
}
// CapacityOf is how many steps the part runs on the node at once: DefaultCapacity for a node the setting does not
// name, and 0 for one it names so, where the part never builds.
func (p *Places) CapacityOf(node string) int {
if n, ok := p.Capacity[node]; ok && n >= 0 {
return n
}
return DefaultCapacity
}
// Running is the steps in flight now, by node.
func (p *Places) Running() map[string]int {
p.mu.Lock()
defer p.mu.Unlock()
out := make(map[string]int, len(p.running))
for n, c := range p.running {
if c > 0 {
out[n] = c
}
}
return out
}
// Reserved is the lanes chosen for a node that hold no place yet, by node.
func (p *Places) Reserved() map[string]int {
p.mu.Lock()
defer p.mu.Unlock()
out := make(map[string]int, len(p.reserved))
for n, c := range p.reserved {
if c > 0 {
out[n] = c
}
}
return out
}
func (p *Places) init() {
if p.running == nil {
p.running = map[string]int{}
}
if p.reserved == nil {
p.reserved = map[string]int{}
}
if p.changed == nil {
p.changed = make(chan struct{})
}
}
// Acquire takes a place on the node, waiting while the node is at its capacity, and answers the function that gives it
// back. A place reserved for the node by a choice is the one taken. It returns an error when ctx ends first, and then
// gives up the reservation too.
func (p *Places) Acquire(ctx context.Context, node string) (release func(), err error) {
for {
p.mu.Lock()
p.init()
if p.running[node] < p.CapacityOf(node) {
p.running[node]++
if p.reserved[node] > 0 {
p.reserved[node]--
}
p.mu.Unlock()
var once sync.Once
return func() {
once.Do(func() {
p.mu.Lock()
p.running[node]--
close(p.changed)
p.changed = make(chan struct{})
p.mu.Unlock()
})
}, nil
}
wait := p.changed
p.mu.Unlock()
select {
case <-wait:
case <-ctx.Done():
p.Cancel(node)
return nil, fmt.Errorf("waiting for a place on %s: %w", node, ctx.Err())
}
}
}
// Cancel gives up a place reserved for the node that will not be taken.
func (p *Places) Cancel(node string) {
p.mu.Lock()
defer p.mu.Unlock()
p.init()
if p.reserved[node] > 0 {
p.reserved[node]--
}
}
// choose orders the nodes (those with room first, counting the lanes running and the lanes reserved, then the fewest of
// them, then by name), keeps the first want and reserves a place on each, all at once.
func (p *Places) choose(nodes []string, want int) []string {
p.mu.Lock()
defer p.mu.Unlock()
p.init()
load := func(n string) int { return p.running[n] + p.reserved[n] }
sort.SliceStable(nodes, func(i, j int) bool {
ri, rj := load(nodes[i]) < p.CapacityOf(nodes[i]), load(nodes[j]) < p.CapacityOf(nodes[j])
if ri != rj {
return ri
}
if load(nodes[i]) != load(nodes[j]) {
return load(nodes[i]) < load(nodes[j])
}
return nodes[i] < nodes[j]
})
if len(nodes) > want {
nodes = nodes[:want]
}
for _, n := range nodes {
p.reserved[n]++
}
return nodes
}
modules/mesh-delivery-builder/buildrun/places_test.go
+104
View File
@@ -0,0 +1,104 @@
package buildrun
import (
"context"
"strings"
"testing"
"time"
)
func acquireAsync(p *Places, node string) chan func() {
got := make(chan func(), 1)
go func() {
release, err := p.Acquire(context.Background(), node)
if err != nil {
close(got)
return
}
got <- release
}()
return got
}
func TestThreeStepsOnANodeOfCapacityTwoRunTwoAtOnceAndTheThirdStartsWhenOneEnds(t *testing.T) { // round 3
p := &Places{Capacity: map[string]int{"ace": 2}}
a, b := acquireAsync(p, "ace"), acquireAsync(p, "ace")
ra, rb := <-a, <-b
third := acquireAsync(p, "ace")
select {
case <-third:
t.Fatal("a third step started on a node of capacity 2 with two running")
case <-time.After(10 * time.Millisecond):
}
if p.Running()["ace"] != 2 {
t.Errorf("running = %v, want 2 on ace", p.Running())
}
ra()
select {
case r := <-third:
r()
case <-time.After(2 * time.Second):
t.Fatal("the third step did not start when one ended")
}
rb()
}
func TestANodeWithNoSettingRunsOneStepAtATime(t *testing.T) { // round 3
p := &Places{Capacity: map[string]int{"shanks": 3}}
first := <-acquireAsync(p, "g14")
second := acquireAsync(p, "g14")
select {
case <-second:
t.Fatal("two steps on a node without a setting")
case <-time.After(10 * time.Millisecond):
}
first()
r := <-second
r()
if p.CapacityOf("g14") != 1 || p.CapacityOf("shanks") != 3 {
t.Errorf("capacities = %d, %d", p.CapacityOf("g14"), p.CapacityOf("shanks"))
}
}
func TestAWaitingStepStopsWhenItsContextEnds(t *testing.T) { // round 3
p := &Places{}
r := <-acquireAsync(p, "ace")
defer r()
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { _, err := p.Acquire(ctx, "ace"); done <- err }()
cancel()
select {
case err := <-done:
if err == nil {
t.Error("Acquire returned no error for a cancelled wait")
}
case <-time.After(2 * time.Second):
t.Fatal("a cancelled wait did not return")
}
}
// The operator's decision of 2026-10-21 13:40: the builder builds on shanks and ace only; a capacity of 0 keeps it off a node.
func TestACapacityOfZeroKeepsThePartOffTheNodeAndANodeNotNamedKeepsOne(t *testing.T) { // issue 1661
p := &Places{Capacity: map[string]int{"shanks": 3, "ace": 2, "g14": 0, "novox": 0}}
if p.CapacityOf("g14") != 0 || p.CapacityOf("novox") != 0 || p.CapacityOf("ace") != 2 || p.CapacityOf("other") != DefaultCapacity {
t.Errorf("capacities = %d %d %d %d, want 0 0 2 and the default for a node not named", p.CapacityOf("g14"), p.CapacityOf("novox"), p.CapacityOf("ace"), p.CapacityOf("other"))
}
entries := tcs("go", "go1.27.1", "ace", "g14", "novox", "shanks", "other")
for i := 0; i < 6; i++ {
nodes, _, err := ChooseNodes("go", entries, p, nil, 2)
if err != nil {
t.Fatal(err)
}
for _, n := range nodes {
if n == "g14" || n == "novox" {
t.Fatalf("choice %d: %v includes a node of capacity 0", i, nodes)
}
}
}
// Only nodes of capacity 0 report: nothing qualifies, and the build says why.
_, _, err := ChooseNodes("go", tcs("go", "go1.27.1", "g14", "novox"), p, nil, 2)
if err == nil || !strings.Contains(err.Error(), "capacity") {
t.Errorf("only capacity-0 nodes: error = %v, want it to name the capacity", err)
}
}
modules/mesh-delivery-builder/buildrun/presence.go
+53
View File
@@ -0,0 +1,53 @@
package buildrun
import (
"sync"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// PresenceSet holds which nodes are online, as the watchman writes it (mesh-delivery-watchman_presence, key `<node>`, the
// node-engine's connection): a node asleep or off the bus is offline and its build module cannot answer. A node the
// watchman has no key of is not online. Safe for use from the watch and from a build at once.
type PresenceSet struct {
mu sync.Mutex
online map[string]bool
}
// Put takes a change of the presence bucket in and answers whether a node's online state changed. A part's key and a
// value that is not a presence are left out.
func (s *PresenceSet) Put(key string, value []byte, deleted bool) bool {
node, part, _, err := delivery.ParsePresenceKey(key)
if err != nil || part != "" {
return false
}
on := false
if !deleted {
p, err := delivery.DecodePresence(value)
if err != nil {
return false
}
on = p.Online
}
s.mu.Lock()
defer s.mu.Unlock()
if s.online[node] == on {
return false
}
if s.online == nil {
s.online = map[string]bool{}
}
if on {
s.online[node] = true
} else {
delete(s.online, node)
}
return true
}
// Online is whether the watchman last saw the node online.
func (s *PresenceSet) Online(node string) bool {
s.mu.Lock()
defer s.mu.Unlock()
return s.online[node]
}
modules/mesh-delivery-builder/buildrun/presence_test.go
+43
View File
@@ -0,0 +1,43 @@
package buildrun
import (
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
func presenceValue(t *testing.T, online bool) []byte {
t.Helper()
b, err := delivery.EncodePresence(delivery.Presence{Online: online})
if err != nil {
t.Fatal(err)
}
return b
}
func TestPresenceSetKnowsOnlyNodesTheWatchmanSawOnline(t *testing.T) { // 1607 review 3
var s PresenceSet
if s.Online("ace") {
t.Error("a node the watchman never wrote is online")
}
if !s.Put("ace", presenceValue(t, true), false) || !s.Online("ace") {
t.Error("ace online was not taken in")
}
if s.Put("ace", presenceValue(t, true), false) {
t.Error("the same presence again counted as a change")
}
if !s.Put("ace", presenceValue(t, false), false) || s.Online("ace") {
t.Error("ace going offline was not taken in")
}
s.Put("g14", presenceValue(t, true), false)
if !s.Put("g14", nil, true) || s.Online("g14") {
t.Error("a deleted key is still online")
}
// A part's instance (part.<part>.<instance>) is not a node.
if s.Put("part.mesh-delivery-builder.novox", presenceValue(t, true), false) || s.Online("part") {
t.Error("a part's key was taken for a node")
}
if s.Put("novox", []byte("junk"), false) {
t.Error("junk counted")
}
}
modules/mesh-delivery-builder/buildrun/standing.go
+225
View File
@@ -0,0 +1,225 @@
package buildrun
import (
"context"
"errors"
"fmt"
"maps"
"slices"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
)
// A build module writes <node>.<language> of its toolchains state on every node it runs on, and nothing deletes the key
// when the module leaves a node: no part writes another module's state, and the module is gone. A report of a node the
// build module no longer runs on is therefore no builder's (issue 1940: mesh-build-node taken off ace on 2026-10-23 left
// ace.typescript at v26.12.0, and every Node fingerprint said "builders disagree").
//
// ToolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
// ToolchainSet): an entry is passed on only while the placement (the seat mesh-placement's state `placement`, held by
// the assigner, delivery.PlacementBucket, read by delivery.Placed; ADR 0567, ADR 0549 §3) places its build module on
// its node, and only once it was written since that placement; it is passed on as deleted otherwise, said once, and
// passed on when its node writes it again. A report written before the placement is one an earlier run of the module
// left (issue 1940, 2026-10-25 20:56: mesh-build-node put on ace again brought ace.typescript v26.12.0 of 2026-10-23
// back into the agreement before the module ran there). Until the placement is settled, and while no node's placement
// is held (before the assigner's adopt writes the first one, or where the state cannot be read), every entry is passed
// on. Builders that run and report different versions still disagree: that refusal is the guard of novox/hq ADR 0465,
// and it stays.
//
// This replaces the copies of the gate the indexer, the builder and the checker each kept over the controller's
// assignments (issue 1940's subtask, issue 774: those go with the controller's lease, ADR 0549 §6).
type ToolchainGate struct {
mu sync.Mutex
modules map[string]bool // the build modules whose toolchains pass the gate
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
logf func(format string, a ...any)
placed *delivery.Placed
known bool // the placement is settled
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
left map[string]string // <module>/<key> -> why it was said left out, so each reason is said once
}
// NewToolchainGate is a gate over the toolchains of modules, passing what stands on to forward and saying with logf what
// it leaves out.
func NewToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *ToolchainGate {
g := &ToolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, placed: delivery.NewPlaced(),
raw: map[string]map[string][]byte{}, left: map[string]string{}}
for _, m := range modules {
g.modules[m] = true
}
return g
}
// Toolchain takes one change of a build module's toolchains bucket in and answers whether what it passed on changed.
func (g *ToolchainGate) Toolchain(module, key string, value []byte, deleted bool) bool {
g.mu.Lock()
defer g.mu.Unlock()
if g.raw[module] == nil {
g.raw[module] = map[string][]byte{}
}
if deleted {
delete(g.raw[module], key)
} else {
g.raw[module][key] = value
}
return g.pass(module, key)
}
// Placement takes one revision of the seat mesh-placement's placement in (at is when the bus wrote it) and answers whether what it
// passed on changed. A value that does not read is said, and places nothing on its node.
func (g *ToolchainGate) Placement(key string, value []byte, deleted bool, at time.Time) bool {
g.mu.Lock()
defer g.mu.Unlock()
was := g.placed.Seeded()
node, changed, err := g.placed.Put(key, value, deleted, at)
if err != nil {
g.logf("%s: %v (issue 1940)", delivery.PlacementBucket, err)
}
if !g.known {
return false
}
if was != g.placed.Seeded() {
// The first placement came, or the last went: every report's standing changed.
return g.all()
}
return g.again(node, changed)
}
// Settled says the placement's history is read: from now on only the reports of placed nodes pass, once a node's
// placement is held. It answers whether what it passed on changed.
func (g *ToolchainGate) Settled() bool {
g.mu.Lock()
defer g.mu.Unlock()
g.known = true
return g.all()
}
// all passes on every report again. The caller holds mu.
func (g *ToolchainGate) all() bool {
changed := false
for _, module := range slices.Sorted(maps.Keys(g.raw)) {
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
changed = g.pass(module, k) || changed
}
}
return changed
}
// again passes on every report of node by a build module among modules. The caller holds mu.
func (g *ToolchainGate) again(node string, modules []string) bool {
changed := false
for _, module := range modules {
if !g.modules[module] {
continue
}
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
if n, _, err := delivery.ParseToolchainKey(k); err == nil && n == node {
changed = g.pass(module, k) || changed
}
}
}
return changed
}
// pass hands one key on as it stands: its value when its node is placed and wrote it since (or the placement is not
// known), deleted otherwise. The caller holds mu.
func (g *ToolchainGate) pass(module, key string) bool {
value, held := g.raw[module][key]
if !held {
return g.forward(module, key, nil, true)
}
node, _, err := delivery.ParseToolchainKey(key)
why := ""
if err == nil && g.known && g.placed.Seeded() {
why = g.leftOut(module, node, value)
}
if why == "" {
delete(g.left, module+"/"+key)
return g.forward(module, key, value, false)
}
if g.left[module+"/"+key] != why {
g.left[module+"/"+key] = why
g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why)
}
return g.forward(module, key, nil, true)
}
// leftOut is why a node's report is no builder's, or empty when it counts: the catalogue does not place the module on its
// node, or the report was written before it did. A value that is no toolchain is passed on for its reader to refuse.
// The caller holds mu.
func (g *ToolchainGate) leftOut(module, node string, value []byte) string {
since, placed := g.placed.Since(node, module)
if !placed {
return fmt.Sprintf("the catalogue does not place %s on %s, so it is no builder", module, node)
}
tc, err := delivery.DecodeToolchain(value)
if err == nil && tc.At.Before(since) {
return fmt.Sprintf("it was written at %s, before %s was placed on %s at %s: it counts once %s writes it again",
tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node)
}
return ""
}
// FollowPlacement reads the seat mesh-placement's placement with its history into gate, says it settled, and follows it; changed
// is told whenever what the gate passes on changed. Where the state cannot be read or watched (not there, or this part
// may not read it) it says so and the gate passes every report on: not fatal, since an indexer, builder or checker that
// stops for it stops every delivery (issue 1945). An error of the watch after that ends on the channel.
func FollowPlacement(ctx context.Context, js process.BucketOpener, gate *ToolchainGate, changed func(context.Context), logf func(string, ...any)) (<-chan error, error) {
errs := make(chan error, 1)
kv, err := js.KeyValue(ctx, delivery.PlacementBucket)
if err != nil {
logf("the state %s cannot be read, so every node's toolchain report counts, a build node's the catalogue does not place too (issue 1940): %v", delivery.PlacementBucket, err)
return errs, nil
}
w, err := kv.WatchAll(ctx, jetstream.IncludeHistory())
if err != nil {
logf("the state %s cannot be watched, so every node's toolchain report counts, a build node's the catalogue does not place too (issue 1940): %v", delivery.PlacementBucket, err)
return errs, nil
}
put := func(e jetstream.KeyValueEntry) bool {
return gate.Placement(e.Key(), e.Value(), e.Operation() != jetstream.KeyValuePut, e.Created())
}
for settled := false; !settled; {
select {
case <-ctx.Done():
_ = w.Stop()
return errs, ctx.Err()
case e, ok := <-w.Updates():
if !ok {
return errs, errors.New("the watch of " + delivery.PlacementBucket + " ended before it settled")
}
if e == nil {
settled = true
continue
}
put(e)
}
}
if gate.Settled() {
changed(ctx)
}
go func() {
defer func() { _ = w.Stop() }()
for {
select {
case <-ctx.Done():
errs <- ctx.Err()
return
case e, ok := <-w.Updates():
if !ok {
errs <- errors.New("the watch of " + delivery.PlacementBucket + " ended")
return
}
if e != nil && put(e) {
changed(ctx)
}
}
}
}()
return errs, nil
}
modules/mesh-delivery-builder/buildrun/standing_test.go
+263
View File
@@ -0,0 +1,263 @@
package buildrun
import (
"bytes"
"context"
"errors"
"fmt"
"log"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
func gateReport(t *testing.T, node, version string, at time.Time) (string, []byte) {
t.Helper()
key, err := delivery.ToolchainKey(node, "typescript")
if err != nil {
t.Fatal(err)
}
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
if err != nil {
t.Fatal(err)
}
return key, value
}
// placing is a value of the state `placement` placing modules on node.
func placing(t *testing.T, node string, modules ...string) (string, []byte) {
t.Helper()
p := delivery.Placement{Node: node}
for _, m := range modules {
p.Modules = append(p.Modules, delivery.PlacedModule{Module: m})
}
values, err := delivery.PlacementValues(p)
if err != nil {
t.Fatal(err)
}
key, _ := delivery.PlacementNodeKey(node)
return key, values[key]
}
// placingLegacy is placing in the form before, `placement.<node>` (research 072 row 37).
func placingLegacy(t *testing.T, node string, modules ...string) (string, []byte) {
t.Helper()
p := delivery.Placement{Node: node}
for _, m := range modules {
p.Modules = append(p.Modules, delivery.PlacedModule{Module: m})
}
key, _ := delivery.LegacyPlacementKey(node)
value, err := delivery.EncodePlacement(p)
if err != nil {
t.Fatal(err)
}
return key, value
}
func agreedOf(set *ToolchainSet) (string, string) {
tc, none := delivery.AgreedToolchain(set.Entries("typescript"))
return tc.Version, none
}
// Issue 1940, the gate over the catalogue's placement (issue 774, ADR 0549 §3): a build node's report counts only while
// the placement places its build module on its node, and only once written since; before the placement is settled, and
// while no node's placement is held, every report counts. The live cases: mesh-build-node taken off ace left
// ace.typescript at v26.12.0 against v26.12.1 on the three others; put on ace again, its report of 2026-10-23 06:29 came
// back before the module ran there.
func TestToolchainGate(t *testing.T) {
old := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
placed := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node placed on ace again
three := []string{"g14", "novox", "shanks"}
start := func(t *testing.T, seeded bool) (*ToolchainGate, *ToolchainSet, *[]string) {
set := &ToolchainSet{}
said := &[]string{}
gate := NewToolchainGate([]string{GoBuildModule, NodeBuildModule}, set.Put, func(f string, a ...any) { *said = append(*said, fmt.Sprintf(f, a...)) })
for _, node := range three {
if seeded {
key, value := placing(t, node, NodeBuildModule)
gate.Placement(key, value, false, old.Add(-time.Hour))
}
key, value := gateReport(t, node, "v26.12.1", placed.Add(time.Minute))
gate.Toolchain(NodeBuildModule, key, value, false)
}
return gate, set, said
}
t.Run("not placed on ace: its report is no builder's", func(t *testing.T) {
gate, set, said := start(t, true)
key, value := gateReport(t, "ace", "v26.12.0", old)
gate.Toolchain(NodeBuildModule, key, value, false)
if a := delivery.ToolchainAgreement(set.Entries("typescript")); len(a.SittingOut) != 1 || a.SittingOut[0].Node != "ace" { // ADR 0584: the majority is agreed, ace's report counts as the one that sits out
t.Fatalf("before the placement is settled every report counts; got %+v", a)
}
k, v := placing(t, "ace", GoBuildModule) // another build module on ace: no matter
gate.Placement(k, v, false, old)
if !gate.Settled() {
t.Fatal("settling left ace's report out, so what was passed on changed")
}
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
t.Fatalf("the placed nodes agree on v26.12.1; got %q, %q", v, none)
}
if len(*said) != 1 || !strings.Contains((*said)[0], "does not place mesh-build-node on ace") {
t.Fatalf("ace's report is said left out once; said %q", *said)
}
if gate.Toolchain(NodeBuildModule, key, value, false) {
t.Fatal("a report of a node not placed changed what was passed on")
}
// Placed on ace with a report written since: it counts, and the builders really disagree.
k, v = placing(t, "ace", GoBuildModule, NodeBuildModule)
gate.Placement(k, v, false, old.Add(-time.Minute))
if a := delivery.ToolchainAgreement(set.Entries("typescript")); a.Toolchain.Version != "v26.12.1" || len(a.SittingOut) != 1 || a.SittingOut[0].Version != "v26.12.0" {
t.Fatalf("builders that run different versions: the majority is agreed and ace sits out; got %+v", a)
}
// Taken off ace again (its key deleted): its report goes again.
if !gate.Placement(k, nil, true, placed) {
t.Fatal("taking ace's placement away passed nothing on")
}
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
t.Fatalf("got %q, %q", v, none)
}
// A report deleted from the bucket goes whether its node is placed or not.
key, _ = gateReport(t, "g14", "v26.12.1", placed)
if !gate.Toolchain(NodeBuildModule, key, nil, true) || len(set.Entries("typescript")) != 2 {
t.Fatalf("deleting g14's report: %d entries left, want novox and shanks", len(set.Entries("typescript")))
}
})
for _, c := range []struct {
name string
at time.Time // when ace wrote its report
counts bool
}{
{"a report left before the placement", old, false},
{"a report written since the placement", placed.Add(30 * time.Second), true},
} {
t.Run(c.name, func(t *testing.T) {
gate, set, said := start(t, true)
k, v := placing(t, "ace") // ace's key, first heard placing nothing
gate.Placement(k, v, false, old.Add(-time.Hour))
gate.Settled()
k, v = placing(t, "ace", NodeBuildModule)
if gate.Placement(k, v, false, placed) {
t.Fatal("a placement with no report passed something on")
}
key, value := gateReport(t, "ace", "v26.12.0", c.at)
if got := gate.Toolchain(NodeBuildModule, key, value, false); got != c.counts {
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
}
// ace's placement written again with another module beside: its standing does not move, nothing is said again.
k, v = placing(t, "ace", GoBuildModule, NodeBuildModule)
if gate.Placement(k, v, false, placed.Add(time.Minute)) {
t.Fatal("the same placement of mesh-build-node changed what was passed on")
}
if c.counts {
return
}
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
t.Fatalf("ace's left report is no builder's; got %q, %q", v, none)
}
if len(*said) != 1 || !strings.Contains((*said)[0], "before mesh-build-node was placed on ace at 2026-10-25T19:56:45Z") {
t.Fatalf("said %q, want once why ace's report is left out", *said)
}
key, value = gateReport(t, "ace", "v26.12.1", placed.Add(time.Minute))
if !gate.Toolchain(NodeBuildModule, key, value, false) || len(set.Entries("typescript")) != 4 {
t.Fatal("ace's new report was not passed on")
}
})
}
t.Run("no placement held: every report counts", func(t *testing.T) {
gate, set, _ := start(t, false)
key, value := gateReport(t, "ace", "v26.12.0", old)
gate.Toolchain(NodeBuildModule, key, value, false)
gate.Settled()
if n := len(set.Entries("typescript")); n != 4 {
t.Fatalf("before the first placement file every report counts; got %d", n)
}
k, v := placing(t, "g14", NodeBuildModule)
if !gate.Placement(k, v, false, old.Add(-time.Hour)) || len(set.Entries("typescript")) != 1 {
t.Fatalf("the first placement leaves every node it does not place out; got %d", len(set.Entries("typescript")))
}
if !gate.Placement(k, nil, true, placed) || len(set.Entries("typescript")) != 4 {
t.Fatal("the last placement gone: every report counts again")
}
})
t.Run("the history: a module in the first revision heard of a key stands since before it", func(t *testing.T) {
p := delivery.NewPlaced()
// ace in the form before, then converted: its module stands since before, the legacy key read no more.
k, v := placingLegacy(t, "ace", NodeBuildModule)
p.Put(k, v, false, old)
k, v = placing(t, "ace", NodeBuildModule)
if _, changed, err := p.Put(k, v, false, placed); len(changed) != 0 || err != nil {
t.Errorf("the conversion of ace's key changed %v, %v", changed, err)
}
if _, changed, _ := p.Put("ace."+NodeBuildModule, []byte("{}"), false, placed); len(changed) != 0 {
t.Errorf("a module key says which modules stand: %v", changed)
}
if k, v := placingLegacy(t, "ace"); true {
if _, changed, _ := p.Put(k, v, false, placed); len(changed) != 0 {
t.Errorf("the legacy key is read after the node key: %v", changed)
}
}
k2, v2 := placing(t, "g14")
p.Put(k2, v2, false, old)
k2, v2 = placing(t, "g14", NodeBuildModule)
if _, changed, _ := p.Put(k2, v2, false, placed); strings.Join(changed, ",") != NodeBuildModule {
t.Errorf("a module placed says changed %v", changed)
}
if _, _, err := p.Put("placement.novox", []byte("not json"), false, old); err == nil {
t.Error("a value that does not read was not said")
}
if s, ok := p.Since("ace", NodeBuildModule); !ok || !s.IsZero() {
t.Errorf("ace's module stands since %v, %v; want the zero time", s, ok)
}
if s, ok := p.Since("g14", NodeBuildModule); !ok || !s.Equal(placed) {
t.Errorf("g14's module stands since %v, %v; want the revision that placed it", s, ok)
}
if got := p.PlacedNodes(NodeBuildModule); strings.Join(got, ",") != "ace,g14" || len(p.ByNode()) != 2 || !p.Seeded() {
t.Errorf("PlacedNodes = %v, ByNode = %v", got, p.ByNode())
}
})
t.Run("a placement the bus refuses is said, and the part goes on", func(t *testing.T) {
var logs bytes.Buffer
logger := log.New(&logs, "", 0)
refused := errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-delivery-assigner_placement")
gate, _, _ := start(t, false)
for _, bus := range []refusingBus{{err: refused}, {refuseOpen: true, err: refused}} {
if _, err := FollowPlacement(context.Background(), bus, gate, func(context.Context) {}, logger.Printf); err != nil {
t.Errorf("a refused placement stopped the start: %v", err)
}
}
if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") {
t.Errorf("the refusals were not said: %q", logs.String())
}
})
}
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
type refusingBus struct {
refuseOpen bool
err error
}
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
if r.refuseOpen {
return nil, r.err
}
return refusedWatch{err: r.err}, nil
}
type refusedWatch struct {
jetstream.KeyValue
err error
}
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
return nil, r.err
}
modules/mesh-delivery-builder/buildrun/steps.go
+67
View File
@@ -0,0 +1,67 @@
package buildrun
import (
"context"
"encoding/json"
"fmt"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// Caller calls a tool of a module on one named machine (mesh-sdk process.Conn.CallOn; ADR 0469 clause 3).
type Caller interface {
CallOn(node, key string, body any) (json.RawMessage, error)
}
// Steps waits for the end of a build module's long step: the value of its key `<node>.<id>` in the module's steps state
// (ADR 0469 clause 2) once its state is not running. It is a watch of that key, not a poll; it returns when the step
// ends or ctx does.
type Steps interface {
Wait(ctx context.Context, module, key string) (delivery.Step, error)
}
// Runner runs a build module's long steps: the call answers at once with the step's key, and the result is read from that
// key when the step ends (ADR 0469 clause 2).
type Runner struct {
Caller Caller
Steps Steps
// Bound is how long a step may run before the builder stops waiting for it.
Bound time.Duration
}
// Run calls tool of the build module on node with args, waits for the step's end and answers its result. A step that
// failed or was refused is an error naming the node, the state and the reason.
func (r Runner) Run(ctx context.Context, module, node, tool string, args map[string]any) (json.RawMessage, error) {
raw, err := r.Caller.CallOn(node, module+"."+tool, args)
if err != nil {
return nil, fmt.Errorf("%s on %s: %w", tool, node, err)
}
var answer struct {
Step string `json:"step"`
Key string `json:"key"`
State string `json:"state"`
}
if err := json.Unmarshal(raw, &answer); err != nil {
return nil, fmt.Errorf("%s on %s answered what is not a step: %w", tool, node, err)
}
if answer.Key == "" {
return nil, fmt.Errorf("%s on %s answered no step key: %s", tool, node, raw)
}
if n, _, err := delivery.ParseStepKey(answer.Key); err != nil || n != node {
return nil, fmt.Errorf("%s on %s answered the step key %q, which is not a key of that node", tool, node, answer.Key)
}
wait, cancel := context.WithTimeoutCause(ctx, r.Bound, fmt.Errorf("the step %s did not end within %s", answer.Key, r.Bound))
defer cancel()
step, err := r.Steps.Wait(wait, module, answer.Key)
if err != nil {
if cause := context.Cause(wait); cause != nil && ctx.Err() == nil {
err = cause
}
return nil, fmt.Errorf("%s on %s: %w", tool, node, err)
}
if step.State != delivery.StepPassed {
return nil, fmt.Errorf("%s on %s: the step %s %s: %s", tool, node, answer.Key, step.State, step.Reason)
}
return step.Result, nil
}
modules/mesh-delivery-builder/buildrun/steps_test.go
+125
View File
@@ -0,0 +1,125 @@
package buildrun
import (
"context"
"encoding/json"
"errors"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
type call struct {
node, key string
body any
}
// fakeCaller answers a long tool as the build modules do: at once, with the step and its key.
type fakeCaller struct {
calls []call
answer string
err error
}
func (f *fakeCaller) CallOn(node, key string, body any) (json.RawMessage, error) {
f.calls = append(f.calls, call{node, key, body})
if f.err != nil {
return nil, f.err
}
return json.RawMessage(f.answer), nil
}
// fakeSteps holds the steps values by `<module>/<key>`; Wait on a key it does not hold blocks until the context ends.
type fakeSteps struct {
mu sync.Mutex
steps map[string]delivery.Step
// during, when set, is called as a wait starts and the function it answers as the wait ends.
during func(key string) func()
}
func (f *fakeSteps) Wait(ctx context.Context, module, key string) (delivery.Step, error) {
f.mu.Lock()
s, ok := f.steps[module+"/"+key]
during := f.during
f.mu.Unlock()
if ok {
if during != nil {
defer during(key)()
}
return s, nil
}
<-ctx.Done()
return delivery.Step{}, context.Cause(ctx)
}
func stepAt(node, id, state, reason string, result string) delivery.Step {
s := delivery.Step{ID: id, Tool: "build_go_fetch", Node: node, State: state, Reason: reason}
if result != "" {
s.Result = json.RawMessage(result)
}
return s
}
func runCallsTheToolOnTheNodeAndReadsTheStepsKey(t *testing.T) { // 1607/AC5
caller := &fakeCaller{answer: `{"step":"abc1","key":"ace.abc1","state":"running"}`}
steps := &fakeSteps{steps: map[string]delivery.Step{"mesh-build-go/ace.abc1": stepAt("ace", "abc1", delivery.StepPassed, "", `{"workspace":"w1","commit":"c"}`)}}
res, err := Runner{Caller: caller, Steps: steps, Bound: time.Second}.Run(context.Background(), "mesh-build-go", "ace", "build_go_fetch", map[string]any{"repository": "novox/r", "commit": "c"})
if err != nil {
t.Fatal(err)
}
if len(caller.calls) != 1 || caller.calls[0].node != "ace" || caller.calls[0].key != "mesh-build-go.build_go_fetch" {
t.Errorf("calls = %+v, want one on ace for mesh-build-go.build_go_fetch", caller.calls)
}
var fr struct{ Workspace string }
if err := json.Unmarshal(res, &fr); err != nil || fr.Workspace != "w1" {
t.Errorf("result = %s (%v), want the step's result", res, err)
}
}
func runFailsOnAFailedOrRefusedStepNamingItsReason(t *testing.T) { // 1607/AC5
for _, state := range []string{delivery.StepFailed, delivery.StepRefused} {
caller := &fakeCaller{answer: `{"step":"abc1","key":"ace.abc1","state":"running"}`}
steps := &fakeSteps{steps: map[string]delivery.Step{"mesh-build-go/ace.abc1": stepAt("ace", "abc1", state, "go is go1.26.0, the recipe builds with go1.27.1", "")}}
_, err := Runner{Caller: caller, Steps: steps, Bound: time.Second}.Run(context.Background(), "mesh-build-go", "ace", "build_go_build", nil)
if err == nil || !strings.Contains(err.Error(), "go1.27.1") || !strings.Contains(err.Error(), state) || !strings.Contains(err.Error(), "ace") {
t.Errorf("%s: error = %v, want the node, the state and the reason", state, err)
}
}
}
func runStopsWaitingAtTheBound(t *testing.T) { // 1607/AC5
caller := &fakeCaller{answer: `{"step":"abc1","key":"ace.abc1","state":"running"}`}
_, err := Runner{Caller: caller, Steps: &fakeSteps{}, Bound: 20 * time.Millisecond}.Run(context.Background(), "mesh-build-go", "ace", "build_go_build", nil)
if err == nil || !strings.Contains(err.Error(), "ace.abc1") || !strings.Contains(err.Error(), "did not end") {
t.Errorf("error = %v, want the step key and that it did not end within its bound", err)
}
}
func runRefusesAnAnswerWithNoKeyAndAKeyOfAnotherNode(t *testing.T) { // 1607/AC5
for _, answer := range []string{`{"state":"running"}`, `{"step":"abc1","key":"novox.abc1","state":"running"}`, `not json`} {
_, err := Runner{Caller: &fakeCaller{answer: answer}, Steps: &fakeSteps{}, Bound: time.Second}.Run(context.Background(), "mesh-build-go", "ace", "build_go_fetch", nil)
if err == nil {
t.Errorf("answer %s: no error", answer)
}
}
}
func runReturnsTheCallsOwnError(t *testing.T) { // 1607/AC5
boom := errors.New("no responders")
_, err := Runner{Caller: &fakeCaller{err: boom}, Steps: &fakeSteps{}, Bound: time.Second}.Run(context.Background(), "mesh-build-go", "ace", "build_go_fetch", nil)
if !errors.Is(err, boom) || !strings.Contains(err.Error(), "ace") {
t.Errorf("error = %v, want the call's error naming the node", err)
}
}
// A step run on its node and read from its steps key, one table of cases (the test budget).
func TestRunningAStepOnANode(t *testing.T) {
t.Run("run calls the tool on the node and reads the steps key", runCallsTheToolOnTheNodeAndReadsTheStepsKey)
t.Run("run fails on a failed or refused step naming its reason", runFailsOnAFailedOrRefusedStepNamingItsReason)
t.Run("run stops waiting at the bound", runStopsWaitingAtTheBound)
t.Run("run refuses an answer with no key and a key of another node", runRefusesAnAnswerWithNoKeyAndAKeyOfAnotherNode)
t.Run("run returns the calls own error", runReturnsTheCallsOwnError)
}
modules/mesh-delivery-builder/buildrun/toolchains.go
+64
View File
@@ -0,0 +1,64 @@
package buildrun
import (
"sync"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// LanguageOf is the language each build module's toolchains state holds (the indexer's ToolchainModules, reversed).
var LanguageOf = map[string]string{GoBuildModule: "go", NodeBuildModule: "typescript"}
// ToolchainSet holds what the build modules' toolchains buckets hold: each node's entry of a language, as the watches
// put them in. Safe for use from the watches and from a build at once.
type ToolchainSet struct {
mu sync.Mutex
by map[string]map[string]delivery.Toolchain // language -> node -> entry
}
// Put takes a change of a build module's toolchains bucket in: a put of a node's entry (key <node>.<language>) or its
// delete. It answers whether the language's entries changed (a node's version or recipe, or a node appearing or going).
// A key or a value that is not an entry of the module's language is left out.
func (s *ToolchainSet) Put(module, key string, value []byte, deleted bool) bool {
language, ok := LanguageOf[module]
if !ok {
return false
}
node, keyLanguage, err := delivery.ParseToolchainKey(key)
if err != nil || keyLanguage != language {
return false
}
s.mu.Lock()
defer s.mu.Unlock()
if deleted {
if _, had := s.by[language][node]; !had {
return false
}
delete(s.by[language], node)
return true
}
tc, err := delivery.DecodeToolchain(value)
if err != nil || tc.Language != language {
return false
}
if s.by == nil {
s.by = map[string]map[string]delivery.Toolchain{}
}
if s.by[language] == nil {
s.by[language] = map[string]delivery.Toolchain{}
}
cur, had := s.by[language][node]
s.by[language][node] = tc
return !had || cur.Version != tc.Version || cur.Recipe != tc.Recipe
}
// Entries are the nodes' entries of a language, in no particular order.
func (s *ToolchainSet) Entries(language string) []delivery.NodeToolchain {
s.mu.Lock()
defer s.mu.Unlock()
var out []delivery.NodeToolchain
for node, tc := range s.by[language] {
out = append(out, delivery.NodeToolchain{Node: node, Toolchain: tc})
}
return out
}
modules/mesh-delivery-builder/buildrun/toolchains_test.go
+49
View File
@@ -0,0 +1,49 @@
package buildrun
import (
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
func tcValue(t *testing.T, language, version string) []byte {
t.Helper()
b, err := delivery.EncodeToolchain(delivery.Toolchain{Language: language, Version: version, Recipe: "1", At: time.Now()})
if err != nil {
t.Fatal(err)
}
return b
}
func TestToolchainSetKeepsEachNodesEntryByLanguageAndSaysWhenOneChanged(t *testing.T) { // 1607/AC6
var s ToolchainSet
if !s.Put(GoBuildModule, "novox.go", tcValue(t, "go", "go1.27.1"), false) {
t.Error("a first entry did not count as a change")
}
if s.Put(GoBuildModule, "novox.go", tcValue(t, "go", "go1.27.1"), false) {
t.Error("the same entry again counted as a change")
}
s.Put(GoBuildModule, "ace.go", tcValue(t, "go", "go1.27.1"), false)
s.Put(NodeBuildModule, "ace.typescript", tcValue(t, "typescript", "tsc5"), false)
if got := s.Entries("go"); len(got) != 2 {
t.Errorf("go entries = %+v, want two", got)
}
if got := s.Entries("typescript"); len(got) != 1 || got[0].Node != "ace" || got[0].Version != "tsc5" {
t.Errorf("typescript entries = %+v", got)
}
if !s.Put(GoBuildModule, "ace.go", nil, true) || len(s.Entries("go")) != 1 {
t.Error("a delete did not remove the node's entry")
}
// What is not a toolchain of the module's language is left out.
s = ToolchainSet{}
if s.Put(GoBuildModule, "novox.typescript", tcValue(t, "typescript", "x"), false) {
t.Error("a typescript toolchain in the go build module's bucket counted")
}
if s.Put(GoBuildModule, "novox.go", []byte("not json"), false) || s.Put("other-module", "novox.go", tcValue(t, "go", "x"), false) {
t.Error("garbage or an unknown module counted")
}
if len(s.Entries("go")) != 0 {
t.Error("an entry was kept")
}
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go
+1 -1
View File
@@ -14,9 +14,9 @@ import (
"sync"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"mesh-delivery-builder/internal/build"
)
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go
+11 -8
View File
@@ -22,6 +22,9 @@ import (
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
// The build runner the build modules pull and run asks with: mesh-build-go's own package (novox/hq ADR 0593 §1),
// another repository's, by its published version, so the builder is tested against the real pulling code.
runner "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
@@ -53,7 +56,7 @@ const (
)
// fakeBuildModules are the build modules on the nodes as the real ones run since R4g (ADR 0525; issue 1887 A4): each pulls
// its build seat's asks on its nodes (buildstep.Puller over the seat's workers) and runs an ask's job with step bodies that
// its build seat's asks on its nodes (runner.Puller over the seat's workers) and runs an ask's job with step bodies that
// answer as the real tools do, ending the step on <node>.<ask id> of its steps bucket.
type fakeBuildModules struct {
js jetstream.JetStream
@@ -68,14 +71,14 @@ type fakeBuildModules struct {
// closed, while set, keeps every node's capacity at 0: no ask is pulled until open is called (a test of the asks
// waiting in their streams).
closed atomic.Bool
pullers []*buildstep.Puller
pullers []*runner.Puller
}
// open lets the nodes pull again and wakes their pullers.
func (f *fakeBuildModules) open() {
f.closed.Store(false)
f.mu.Lock()
ps := append([]*buildstep.Puller(nil), f.pullers...)
ps := append([]*runner.Puller(nil), f.pullers...)
f.mu.Unlock()
for _, p := range ps {
p.Wake()
@@ -139,26 +142,26 @@ func (f *fakeBuildModules) serve(t *testing.T, module string, nodes ...string) {
t.Fatal(err)
}
language := strings.TrimPrefix(module, "mesh-build-")
if err := buildstep.DeclareAskWorkers(f.ctx, f.js, language, 30*time.Second); err != nil {
if err := runner.DeclareAskWorkers(f.ctx, f.js, language, 30*time.Second); err != nil {
t.Fatal(err)
}
put := func(key string, value any) (uint64, error) {
raw, _ := value.(json.RawMessage)
return kv.Put(f.ctx, key, raw)
}
call := func(name string) buildstep.StepFunc {
call := func(name string) runner.StepFunc {
return func(ctx context.Context, args map[string]any) (any, error) { return f.step(module, name, args) }
}
steps := buildstep.AskSteps{Fetch: call("fetch"), Build: call("build"), Copy: call("copy")}
steps := runner.AskSteps{Fetch: call("fetch"), Build: call("build"), Copy: call("copy")}
for _, node := range nodes {
p := &buildstep.Puller{Source: &buildstep.JetStreamAsks{JS: f.js, Language: language}, Language: language, Node: node, Put: put,
p := &runner.Puller{Source: &runner.JetStreamAsks{JS: f.js, Language: language}, Language: language, Node: node, Put: put,
Capacity: func() int {
if f.closed.Load() {
return 0
}
return 2
}, PullWait: 200 * time.Millisecond,
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { return buildstep.RunAsk(ctx, steps, a) }}
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { return runner.RunAsk(ctx, steps, a) }}
f.mu.Lock()
f.pullers = append(f.pullers, p)
f.mu.Unlock()
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go
+1 -1
View File
@@ -18,7 +18,7 @@ import (
"syscall"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go
+1 -1
View File
@@ -12,8 +12,8 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// 1945 (a part exits at start for a bucket it works without): a bus that refuses every toolchains bucket leaves the
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go
+1 -1
View File
@@ -8,7 +8,7 @@ import (
"strings"
"sync"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
modules/mesh-delivery-builder/go.mod
+6 -1
View File
@@ -3,12 +3,15 @@ module mesh-delivery-builder
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
require git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a
require (
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun v0.0.0-00010101000000-000000000000
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
github.com/google/go-tpm v0.9.8 // indirect
github.com/klauspost/compress v1.20.0 // indirect
@@ -20,3 +23,5 @@ require (
golang.org/x/sys v0.48.0 // indirect
golang.org/x/time v0.15.0 // indirect
)
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun => ./buildrun
modules/mesh-delivery-builder/go.sum
+4 -2
View File
@@ -1,5 +1,7 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a h1:oxgksCiq272/afXbUION/dxtAS5MPGnmyq6+AZQxmXg=
git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a/go.mod h1:kpsX4Pgss5EeuBvel1zWa2ogj7MkR3jZso6oa/J0YUQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
modules/mesh-delivery-builder/internal/build/askagain.go
+1 -1
View File
@@ -11,9 +11,9 @@ import (
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// **An ask that can no longer be answered is asked again** (novox/hq issue 1945). A build seat's stream gives an ask up
modules/mesh-delivery-builder/internal/build/askfake_test.go
+94
View File
@@ -0,0 +1,94 @@
package build
import (
"context"
"encoding/json"
"fmt"
"maps"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The build runner a build node runs an ask with is mesh-build-go's (novox/hq ADR 0593 §1), another repository's,
// which the builder does not import. These tests run an ask as the builder needs it run: fetch once at the job's
// fetch arguments with its package versions, build or copy each artifact, answer each one's digest and reference. They
// test the builder, not the runner.
// stepFunc is one step of a build module, as its tool answers it.
type stepFunc func(ctx context.Context, args map[string]any) (any, error)
// askSteps are a build module's steps an ask is run with.
type askSteps struct{ Fetch, Build, Copy stepFunc }
// runAsk runs a build ask with steps and answers its delivery.BuildAnswer, or the error that fails the step.
func runAsk(ctx context.Context, steps askSteps, ask delivery.StepAsk) (any, error) {
if ask.Build == nil {
return nil, fmt.Errorf("the ask %s holds no build job", ask.ID)
}
job := *ask.Build
fetch := job.Fetch
if len(ask.Packages) > 0 && fetch != nil {
fetch = maps.Clone(fetch)
fetch["packages"] = maps.Clone(ask.Packages)
}
answer := delivery.BuildAnswer{}
workspace := ""
for _, a := range job.Artifacts {
var out any
var err error
if a.From != "" {
if steps.Copy == nil {
return nil, fmt.Errorf("this build module copies no artifact (%s)", a.Artifact)
}
out, err = steps.Copy(ctx, map[string]any{"module": job.Module, "artifact": a.Artifact, "from": a.From})
} else {
if workspace == "" {
raw, err := asRaw(steps.Fetch(ctx, fetch))
if err != nil {
return nil, err
}
var f struct {
Workspace string `json:"workspace"`
Merge string `json:"merge"`
Conflicts []string `json:"conflicts"`
}
if err := json.Unmarshal(raw, &f); err != nil || f.Workspace == "" {
return nil, fmt.Errorf("the fetch answered no workspace: %s", raw)
}
if _, candidate := fetch["base"]; candidate && (len(f.Conflicts) > 0 || f.Merge == "") {
return nil, fmt.Errorf("the head does not merge into the trunk: %s", strings.Join(f.Conflicts, ", "))
}
workspace = f.Workspace
}
args := map[string]any{"workspace": workspace, "module": job.Path, "artifact": a.Artifact}
if job.Publish {
args["publish"] = true
}
out, err = steps.Build(ctx, args)
}
raw, err := asRaw(out, err)
if err != nil {
return nil, fmt.Errorf("%s: %w", a.Artifact, err)
}
var built struct {
Digest string `json:"digest"`
Reference string `json:"reference"`
}
if err := json.Unmarshal(raw, &built); err != nil || built.Digest == "" {
return nil, fmt.Errorf("the build of %s answered no digest: %s", a.Artifact, raw)
}
answer.Artifacts = append(answer.Artifacts, delivery.AskBuilt{Artifact: a.Artifact, Digest: built.Digest, Reference: built.Reference})
}
return answer, nil
}
func asRaw(v any, err error) (json.RawMessage, error) {
if err != nil {
return nil, err
}
if r, ok := v.(json.RawMessage); ok {
return r, nil
}
return json.Marshal(v)
}
modules/mesh-delivery-builder/internal/build/buildagain.go
+1 -1
View File
@@ -7,8 +7,8 @@ import (
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// **A failed key is built again by hand** (novox/hq issue 1126 note 72; ADR 0429 clause 18: "a fingerprint is built again
modules/mesh-delivery-builder/internal/build/candidates.go
+1 -1
View File
@@ -5,8 +5,8 @@ import (
"fmt"
"sort"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// How a candidate's fingerprint is built (novox/hq ADR 0484 clause 3, issue 1684): the indexer writes, for each module a
modules/mesh-delivery-builder/internal/build/claim.go
+1 -1
View File
@@ -8,9 +8,9 @@ import (
"sort"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// ErrConflict is what a Store answers when a Create finds the key or an Update finds another revision than the one given.
modules/mesh-delivery-builder/internal/build/fixtures_test.go
+3 -3
View File
@@ -29,7 +29,7 @@ func captured(t *testing.T, name string) json.RawMessage {
return step.Result
}
// The captured image steps, run as a pulled ask runs them (buildstep.RunAsk), answer what the builder reads: the trunk's
// The captured image steps, run as a pulled ask runs them (runAsk), answer what the builder reads: the trunk's
// build its digest and reference, a candidate's its digest and no reference (issue 1887 A4a).
func TestCapturedImageStepsDecodeWithTheBuildersOwnTypes(t *testing.T) { // review r2, mesh-catalog 2; ADR 0487 clause 1
for _, tc := range []struct {
@@ -37,7 +37,7 @@ func TestCapturedImageStepsDecodeWithTheBuildersOwnTypes(t *testing.T) { // revi
publish bool
ref bool
}{{"build-trunk.json", true, true}, {"build-candidate.json", false, false}} {
steps := buildstep.AskSteps{
steps := askSteps{
Fetch: func(context.Context, map[string]any) (any, error) {
return json.RawMessage(captured(t, "fetch.json")), nil
},
@@ -47,7 +47,7 @@ func TestCapturedImageStepsDecodeWithTheBuildersOwnTypes(t *testing.T) { // revi
At: time.Now(), Build: &delivery.BuildJob{Module: "m", Path: "modules/m", Publish: tc.publish,
Fetch: map[string]any{"repository": "novox/mesh-catalog", "commit": strings.Repeat("a", 40)},
Artifacts: []delivery.AskArtifact{{Artifact: "server"}}}}
out, err := buildstep.RunAsk(context.Background(), steps, ask)
out, err := runAsk(context.Background(), steps, ask)
if err != nil {
t.Fatalf("%s: %v", tc.build, err)
}
modules/mesh-delivery-builder/internal/build/imports_test.go
+36
View File
@@ -0,0 +1,36 @@
package build
import (
"encoding/json"
"strings"
"sync"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// ADR 0593 §2 (F3): the imports a Go build module answers at a build's end reach the done key and the log's end entry,
// where the indexer keeps them on the module.
func importsABuildAnswersReachTheDoneKeyAndTheLog(t *testing.T) {
r := newRig(t, goBundle)
imports := []string{"git.novox.be/novox/mesh-sdk/go", "git.novox.be/novox/mesh-sdk/go/announce"}
r.asks.Answer = func(a delivery.StepAsk) delivery.Step {
step := r.answer(a)
var answer map[string]any
if err := json.Unmarshal(step.Result, &answer); err == nil && a.Language == "go" {
answer["imports"] = imports
step.Result, _ = json.Marshal(answer)
}
return step
}
var logged []delivery.BuildLogEntry
var mu sync.Mutex
r.b.Node, r.b.Log = "novox", func(e delivery.BuildLogEntry) { mu.Lock(); logged = append(logged, e); mu.Unlock() }
v := r.consider(t, "modules/m")
if v.State != delivery.BuildDone || strings.Join(v.Imports, ",") != strings.Join(imports, ",") {
t.Fatalf("done key %+v, want the imports", v)
}
if len(logged) != 2 || strings.Join(logged[1].Imports, ",") != strings.Join(imports, ",") || len(logged[0].Imports) != 0 {
t.Fatalf("log %+v, want the imports on the end alone", logged)
}
}
modules/mesh-delivery-builder/internal/build/plan.go
+1 -1
View File
@@ -8,9 +8,9 @@ import (
"sort"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// ImageBuildModule is the build module that builds an image artifact (novox/hq ADR 0487): buildah, rootless, on the nodes that
modules/mesh-delivery-builder/internal/build/plan_test.go
+1 -1
View File
@@ -3,8 +3,8 @@ package build
import (
"errors"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"strings"
"testing"
modules/mesh-delivery-builder/internal/build/r4e_test.go
+3 -4
View File
@@ -305,10 +305,9 @@ func TestTheFetchOfAModuleNamingItsPathsCarriesItsDirectoryAndThosePaths(t *test
r := newRig(t, tsPackage)
r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15"
paths := map[string]string{"src": hex40('1'), "package.json": hex40('2')}
tree, err := delivery.NamedTree(manifestD, paths)
if err != nil {
t.Fatal(err)
}
// The named tree the indexer computes from those paths (its own, ADR 0593 §2), given as data: the builder reads it,
// never computes it.
const tree = delivery.NamedTreePrefix + "5b5e0f3b6f3c1f2a9d8e7c6b5a4938271605f4e3d2c1b0a9f8e7d6c5b4a39281"
data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-sdk", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", FP2: fp1, Manifest: manifestD, Path: ".",
Toolchains: agreedToolchains, Inputs2: &delivery.KeptInputs2{Tree: tree, Paths: paths}})
if err != nil {
modules/mesh-delivery-builder/internal/build/round4_test.go
+1 -1
View File
@@ -2,7 +2,7 @@ package build
import (
"context"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
modules/mesh-delivery-builder/internal/build/run.go
+23 -2
View File
@@ -6,16 +6,17 @@ import (
"errors"
"fmt"
"maps"
"slices"
"sort"
"strconv"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// ToolchainEntries are the nodes' toolchain entries of a language (buildrun.ToolchainSet).
@@ -533,6 +534,7 @@ type lane struct {
toolchain *delivery.ToolchainUsed
digests map[string]string // artifact -> digest
references map[string]string // artifact -> the reference an image was pushed to (a trunk build only)
imports []string // the mesh packages the lane's programs import, as its build module answered them
err error
}
@@ -666,7 +668,8 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
if len(used) == 0 {
used = nil
}
done := delivery.BuilderBuild{State: delivery.BuildDone, By: b.Claimer.By, Manifest: m.Manifest, Step: step, Artifacts: artifacts, Toolchains: used}
done := delivery.BuilderBuild{State: delivery.BuildDone, By: b.Claimer.By, Manifest: m.Manifest, Step: step, Artifacts: artifacts, Toolchains: used,
Imports: importsOf(lanes)}
if _, err := delivery.EncodeBuilderBuild(done); err != nil {
// A reference the build module answered that does not name the artifact and its digest is no reference.
return failed("%s%v", delivery.NotStored, err)
@@ -866,6 +869,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del
l.err = fmt.Errorf("the ask of %s on %s answered what is not a build: %s", l.module, step.Node, step.Result)
return
}
l.imports = answer.Imports
built := map[string]delivery.AskBuilt{}
for _, a := range answer.Artifacts {
built[a.Artifact] = a
@@ -950,9 +954,26 @@ func LogEnded(module string, m delivery.Module, ref, by string, at time.Time, re
e.Against = append([]string(nil), m.Inputs2.Bases...)
}
e.Made = append([]delivery.BuiltArtifact(nil), result.Artifacts...)
e.Imports = append([]string(nil), result.Imports...) // the indexer keeps them on the module (novox/hq ADR 0593 §2)
return e
}
// importsOf are the mesh packages the lanes' programs import, as their build modules answered them (novox/hq ADR 0593
// §2): sorted and each once; nil when none answered any. A lane kept from a prior build answered none, so a build that
// kept one reports only what it built.
func importsOf(lanes []*lane) []string {
seen := map[string]bool{}
for _, l := range lanes {
for _, p := range l.imports {
seen[p] = true
}
}
if len(seen) == 0 {
return nil
}
return slices.Sorted(maps.Keys(seen))
}
// SittingOutWords says, per node that is not on the agreed toolchain of a language, that it sits out and which version
// it has against the agreed one: "ace sits out: typescript 26.12.1, agreed 26.13.0" (novox/hq ADR 0584 clause 2).
func SittingOutWords(language string, a delivery.Agreement) []string {
modules/mesh-delivery-builder/internal/build/run_test.go
+6 -6
View File
@@ -13,11 +13,10 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// buildModules simulates the two build modules on the nodes: a long tool answers a step and key at once and writes the
@@ -140,7 +139,7 @@ type rig struct {
// toolPrefix is the start of each build module's tool names, as the fake build modules answer them.
var toolPrefix = map[string]string{buildrun.GoBuildModule: "build_go_", buildrun.NodeBuildModule: "build_node_", ImageBuildModule: "build_image_"}
// answer runs an ask as the build module that pulls it does (buildstep.RunAsk), with the fake build modules' tools on the
// answer runs an ask as the build module that pulls it does (runAsk), with the fake build modules' tools on the
// rig's pulling node, and answers the step that node writes.
func (r *rig) answer(a delivery.StepAsk) delivery.Step {
node := r.puller
@@ -149,7 +148,7 @@ func (r *rig) answer(a delivery.StepAsk) delivery.Step {
}
module, _ := delivery.BuildModuleOf(a.Language)
runner := buildrun.Runner{Caller: r.bm, Steps: r.bm.steps, Bound: time.Second}
tool := func(name string) buildstep.StepFunc {
tool := func(name string) stepFunc {
return func(ctx context.Context, args map[string]any) (any, error) {
raw, err := runner.Run(ctx, module, node, toolPrefix[module]+name, args)
if err != nil {
@@ -158,8 +157,8 @@ func (r *rig) answer(a delivery.StepAsk) delivery.Step {
return raw, nil
}
}
steps := buildstep.AskSteps{Fetch: tool("fetch"), Build: tool("build"), Copy: tool("copy")}
out, err := buildstep.RunAsk(context.Background(), steps, a)
steps := askSteps{Fetch: tool("fetch"), Build: tool("build"), Copy: tool("copy")}
out, err := runAsk(context.Background(), steps, a)
step := delivery.Step{ID: a.ID, Tool: a.Kind + " ask", Node: node, State: delivery.StepPassed}
if err != nil {
step.State, step.Reason = delivery.StepFailed, err.Error()
@@ -222,6 +221,7 @@ var (
)
func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7, ADR 0478 clauses 1, 4
t.Run("the imports a build answers reach the done key and the log", importsABuildAnswersReachTheDoneKeyAndTheLog)
r := newRig(t, goBundle, archiveA, tsBundle)
// ADR 0584: a third node agrees with novox and ace's typescript is the odd one; the build is done and its step says ace sits out.
r.tools.Put(buildrun.NodeBuildModule, "g14.typescript", tcValue(t, "typescript", "tsc5"), false)
modules/mesh-delivery-builder/internal/build/stuck.go
+1 -1
View File
@@ -11,8 +11,8 @@ import (
"sync/atomic"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// The builder's tools (novox/hq issue 1945; the instruction "tools are how a module is spoken to"): builder_stuck reads
modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go
+3 -3
View File
@@ -10,9 +10,9 @@ import (
"sync/atomic"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"mesh-delivery-checker/internal/check"
)
@@ -189,7 +189,7 @@ func SeatTryTools(tr Trying) []stdio.Tool {
func Tools(r check.Reader) []stdio.Tool {
return []stdio.Tool{
{Name: check.VerdictsReadTool,
Description: "The verdicts the checker holds, each as stored under verdicts.<group>.<heads>: its state, what failed first, the tests run, kept and skipped per level, the time of each part, the trunk heads it judged against and the composition, with the full " +
Description: "The verdicts the checker holds, each as stored under verdicts.<group>.<heads>: its state, every failure it names one line each (every member's failed tests, refused steps and conflicts, the manifests' refusals and the composition), the tests run, kept and skipped per level, the time of each part, the trunk heads it judged against and the composition, with the full " +
"report of the status each pull request carries (the line the forge shows is cut at 140 characters, this is not). With a group, that group's only. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"group": map[string]any{"type": "string", "description": "the delivery group's branch name"}}},
@@ -199,7 +199,7 @@ func Tools(r check.Reader) []stdio.Tool {
return cutAnswer(map[string]any{"verdicts": vs}, cut), err
}},
{Name: check.PullReadTool,
Description: "The verdict that judges one pull request and the full report of its status mesh/check. Read-only.",
Description: "The verdict that judges one pull request, every failure it names one line each (all of them, not only the first the status line shows), and the full report of its status mesh/check. Read-only.",
Input: map[string]any{"type": "object", "required": []string{"repository", "number"}, "properties": map[string]any{
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
"number": map[string]any{"type": "integer", "description": "the pull request's number"}}},
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/composition_line_integration_test.go
-64
View File
@@ -1,64 +0,0 @@
//go:build integration
package main
import (
"strings"
"testing"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// assignerWrote raises the assigner's compositions bucket, the assigner online in the watchman's presence, and its verdict
// under the check's key, as the assigner leaves them on the bus.
func (b *bed) assignerWrote(t *testing.T, key string, v delivery.CompositionVerdict) {
t.Helper()
kv, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: delivery.AssignerCompositionsBucket, History: delivery.AssignerCompositionsHistory})
if err != nil {
t.Fatal(err)
}
pkey, _ := delivery.PresencePartKey(delivery.AssignerModule, "novox")
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
if _, err := b.buckets[delivery.PresenceBucket].Put(b.ctx, pkey, on); err != nil {
t.Fatal(err)
}
data, err := delivery.EncodeCompositionVerdict(v)
if err != nil {
t.Fatal(err)
}
if _, err := kv.Put(b.ctx, key, data); err != nil {
t.Fatal(err)
}
}
// 1297/AC3: started whole with the assigner's verdict on the bus, the checker's line says the composition's verdict, not
// "not yet run": a pass names its machine, a fail fails the check with the assigner's words.
func TestTheCheckersLineSaysTheAssignersComposition(t *testing.T) {
t.Run("the read awaits the assigner's composition", checkerAwaitsTheAssignersComposition)
for _, c := range []struct {
state, owed, words string
}{
{delivery.CompositionPass, "success", "composition passed"},
{delivery.CompositionFail, "failure", "g14 runs web, which the change removes from its source"},
} {
b := startBed(t)
key := b.candidate(t, "feat/x", pull(repo, 7, head1))
group, heads, _ := delivery.ParseCheckerVerdictKey(key)
v := delivery.CompositionVerdict{Group: group, Heads: heads, State: c.state, SDK: "v0.1.14-x", Machines: []string{"g14"}}
if c.state == delivery.CompositionFail {
v.Machine, v.Words = "g14", c.words
}
b.assignerWrote(t, key, v)
b.start(t, nil)
got := b.verdict(t, key, ended)
if got.Composition == nil || got.Composition.State != c.state {
t.Fatalf("%s: the verdict's composition is %+v", c.state, got.Composition)
}
o := b.owed(t, 7, head1, c.owed)
if !strings.Contains(o.Text, c.words) || strings.Contains(o.Text, "not yet run") {
t.Errorf("%s: the line says %q", c.state, o.Text)
}
}
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/compositions.go
-131
View File
@@ -1,131 +0,0 @@
package main
import (
"context"
"errors"
"sync"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-checker/internal/check"
)
// compositionWatch reads the assigner's composition of a check (ADR 0528 clause 5) by watching its key and the
// assigner's presence: no poll and no timer. With no assigner online it answers check.ErrAssignerNotRunning at once.
type compositionWatch struct {
kv jetstream.KeyValue
// presence is the watchman's presence bucket; nil (not there) counts as no assigner online.
presence jetstream.KeyValue
}
// Await answers the composition under key once the assigner wrote one that does not wait (check.AwaitComposition).
func (c compositionWatch) Await(ctx context.Context, key string) (delivery.CompositionVerdict, error) {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
comps := make(chan check.CompositionEvent)
presence := make(chan check.PresenceEvent)
cw, err := c.kv.Watch(ctx, key)
if err != nil {
return delivery.CompositionVerdict{}, err
}
defer cw.Stop()
go func() {
for e := range cw.Updates() {
ev := check.CompositionEvent{Settled: e == nil}
if e != nil && e.Operation() == jetstream.KeyValuePut {
ev.Value = e.Value()
}
select {
case comps <- ev:
case <-ctx.Done():
return
}
}
}()
if c.presence == nil {
go func() {
select {
case presence <- check.PresenceEvent{Settled: true}:
case <-ctx.Done():
}
}()
} else {
pw, err := c.presence.Watch(ctx, "part."+delivery.AssignerModule+".*")
if err != nil {
return delivery.CompositionVerdict{}, err
}
defer pw.Stop()
go func() {
for e := range pw.Updates() {
ev := check.PresenceEvent{Settled: e == nil}
if e != nil {
ev.Key = e.Key()
if e.Operation() == jetstream.KeyValuePut {
if p, err := delivery.DecodePresence(e.Value()); err == nil {
ev.Online = p.Online
}
}
}
select {
case presence <- ev:
case <-ctx.Done():
return
}
}
}()
}
return check.AwaitComposition(ctx, presence, comps)
}
// followCompositions hands the checker each composition the assigner writes from now on, so a verdict that said its
// composition was not judged takes it in when the assigner starts (event-driven).
func followCompositions(ctx context.Context, kv jetstream.KeyValue, ch *check.Checker) error {
w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly())
if err != nil {
return err
}
go func() {
defer w.Stop()
for e := range w.Updates() {
if e == nil || e.Operation() != jetstream.KeyValuePut {
continue
}
if comp, err := delivery.DecodeCompositionVerdict(e.Value()); err == nil {
ch.Composed(ctx, comp)
}
}
}()
return nil
}
// lazyCompositions opens the assigner's compositions when a check first asks for one and the bucket is there: until it
// is, no assigner ever ran, so the check says at once that its composition is not judged (check.ErrAssignerNotRunning),
// and the next check looks again. No timer: each look is a check asking.
type lazyCompositions struct {
open func(ctx context.Context) (check.CompositionReader, error)
mu sync.Mutex
reader check.CompositionReader
}
// Await is the opened reader's Await, opening it first when it is not yet.
func (l *lazyCompositions) Await(ctx context.Context, key string) (delivery.CompositionVerdict, error) {
l.mu.Lock()
r := l.reader
if r == nil {
opened, err := l.open(ctx)
switch {
case errors.Is(err, jetstream.ErrBucketNotFound):
l.mu.Unlock()
return delivery.CompositionVerdict{}, check.ErrAssignerNotRunning
case err != nil:
l.mu.Unlock()
return delivery.CompositionVerdict{}, err
}
l.reader, r = opened, opened
}
l.mu.Unlock()
return r.Await(ctx, key)
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/compositions_integration_test.go
-107
View File
@@ -1,107 +0,0 @@
//go:build integration
package main
import (
"context"
"errors"
"os"
"path/filepath"
"testing"
"time"
"github.com/nats-io/nats-server/v2/server"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-checker/internal/check"
)
// 1297/AC3 (its bus half): on the real bus, the checker's read of the assigner's composition waits while the key is
// absent and while the assigner says it waits, and answers the verdict the assigner then writes under the check's key.
// A case of TestTheCheckersLineSaysTheAssignersComposition (merged into it for the test budget, issue 774).
func checkerAwaitsTheAssignersComposition(t *testing.T) {
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "comp")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(dir) })
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: filepath.Join(dir, "js"), NoLog: true, NoSigs: true})
if err != nil {
t.Fatal(err)
}
go srv.Start()
if !srv.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
t.Cleanup(srv.Shutdown)
nc, err := nats.Connect(srv.ClientURL())
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, _ := jetstream.New(nc)
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.AssignerCompositionsBucket, History: delivery.AssignerCompositionsHistory})
if err != nil {
t.Fatal(err)
}
pkv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.PresenceBucket, History: 5})
if err != nil {
t.Fatal(err)
}
pkey, _ := delivery.PresencePartKey(delivery.AssignerModule, "novox")
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
if _, err := pkv.Put(ctx, pkey, on); err != nil {
t.Fatal(err)
}
// With no assigner online the wait answers at once (a second key, before the assigner of this test is online).
off, _ := delivery.EncodePresence(delivery.Presence{Online: false})
if _, err := pkv.Put(ctx, pkey, off); err != nil {
t.Fatal(err)
}
other, _ := delivery.CompositionKey("feat/y", "0123456789ab")
if _, err := (compositionWatch{kv: kv, presence: pkv}).Await(ctx, other); !errors.Is(err, check.ErrAssignerNotRunning) {
t.Fatalf("with no assigner online the wait answered %v", err)
}
if _, err := pkv.Put(ctx, pkey, on); err != nil {
t.Fatal(err)
}
key, _ := delivery.CompositionKey("feat/x", "0123456789ab")
got := make(chan delivery.CompositionVerdict, 1)
go func() {
v, err := compositionWatch{kv: kv, presence: pkv}.Await(ctx, key)
if err != nil {
t.Error(err)
}
got <- v
}()
put := func(v delivery.CompositionVerdict) {
data, err := delivery.EncodeCompositionVerdict(v)
if err != nil {
t.Fatal(err)
}
if _, err := kv.Put(ctx, key, data); err != nil {
t.Fatal(err)
}
}
time.Sleep(200 * time.Millisecond)
put(delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionWaits, Words: "the indexer holds no candidates yet"})
select {
case v := <-got:
t.Fatalf("a waiting composition was answered: %+v", v)
case <-time.After(300 * time.Millisecond):
}
put(delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionPass, SDK: "x", Machines: []string{"g14"}})
select {
case v := <-got:
if v.State != delivery.CompositionPass || v.Machines[0] != "g14" {
t.Errorf("answered %+v", v)
}
case <-time.After(5 * time.Second):
t.Fatal("the composition written was not answered")
}
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/compositions_test.go
-46
View File
@@ -1,46 +0,0 @@
package main
import (
"context"
"errors"
"testing"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-checker/internal/check"
)
type answering struct{ v delivery.CompositionVerdict }
func (a answering) Await(context.Context, string) (delivery.CompositionVerdict, error) {
return a.v, nil
}
// 1297 review item 3: the compositions bucket missing when the checker starts is looked up again at the next check: while
// it is missing a check says at once the assigner is not running; once it is there the check reads its composition, and
// it is opened once.
func TestTheCompositionsBucketIsLookedUpAtEachCheck(t *testing.T) {
opens, there := 0, false
l := &lazyCompositions{open: func(context.Context) (check.CompositionReader, error) {
opens++
if !there {
return nil, jetstream.ErrBucketNotFound
}
return answering{delivery.CompositionVerdict{State: delivery.CompositionPass, SDK: "x"}}, nil
}}
if _, err := l.Await(context.Background(), "k"); !errors.Is(err, check.ErrAssignerNotRunning) {
t.Fatalf("without the bucket: %v", err)
}
there = true
for range 2 {
v, err := l.Await(context.Background(), "k")
if err != nil || v.State != delivery.CompositionPass {
t.Fatalf("with the bucket: %+v, %v", v, err)
}
}
if opens != 2 {
t.Errorf("opened %d times, want twice (missing, then once)", opens)
}
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/engines.go
+142
View File
@@ -0,0 +1,142 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/composer"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The checker's composition of a check on the bus (novox/hq ADR 0593 §7, §9; check.EngineCompositions): each placed
// node's engine asked `node-engine.compose-candidate` on that node, and, for a group that changes the node-engine's
// composition code, the composer built from its head run on the Go build seat.
// composeCandidateKey is the seat verb's call key, asked of one node (process.Conn.CallOn).
var composeCandidateKey = "seat:" + catalogue.EngineSeat + "." + delivery.EngineComposeCandidateVerb
// engineAsks asks one node's engine over the bus, its answer waited for at most ctx's deadline (the answer bound).
type engineAsks struct {
call func(node, key string, body any) (json.RawMessage, error)
}
func (e engineAsks) ComposeCandidate(ctx context.Context, node string, ask delivery.ComposeCandidateAsk) (delivery.ComposeCandidateAnswer, error) {
type answer struct {
raw json.RawMessage
err error
}
got := make(chan answer, 1)
go func() {
raw, err := e.call(node, composeCandidateKey, ask)
got <- answer{raw, err}
}()
select {
case <-ctx.Done():
return delivery.ComposeCandidateAnswer{}, fmt.Errorf("no answer within its bound: %w", ctx.Err())
case a := <-got:
if a.err != nil {
return delivery.ComposeCandidateAnswer{}, a.err
}
return delivery.DecodeComposeCandidateAnswer(a.raw)
}
}
// placedNodes names every node the seat mesh-placement places now, read from its state when a check composes (no
// watch: a check reads it once). A state that cannot be read is an error, so the check says its composition was not
// made rather than passing with no node asked (review finding 4).
type placedNodes struct {
kv func(ctx context.Context) (jetstream.KeyValue, error)
}
func (p placedNodes) Placed() ([]string, error) {
ctx, cancel := context.WithTimeout(context.Background(), delivery.ComposeAnswerBound)
defer cancel()
kv, err := p.kv(ctx)
if err != nil {
return nil, fmt.Errorf("the state %s cannot be read: %w", delivery.PlacementBucket, err)
}
lister, err := kv.ListKeys(ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
return nil, nil // a placement that places nothing
}
if err != nil {
return nil, fmt.Errorf("the state %s cannot be listed: %w", delivery.PlacementBucket, err)
}
seen := map[string]bool{}
for key := range lister.Keys() {
if node, _, _, err := delivery.ParsePlacementKey(key); err == nil {
seen[node] = true
}
}
out := make([]string, 0, len(seen))
for n := range seen {
out = append(out, n)
}
sort.Strings(out)
return out, nil
}
// seatSandbox runs a compose job on the Go build seat's check stream, as the checker's own ask, and answers the
// composer's verdict from the step's result (delivery.ComposeAnswer).
type seatSandbox struct{ asker buildrun.Asker }
func (s seatSandbox) Compose(ctx context.Context, job delivery.ComposeJob) (composer.Verdict, error) {
ask := delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: delivery.CheckerModule,
At: time.Now().UTC(), Compose: &job}
step, err := s.asker.Ask(ctx, ask)
if err != nil {
return composer.Verdict{}, err
}
if step.State != delivery.StepPassed {
return composer.Verdict{}, fmt.Errorf("the compose ask %s ended %s: %s", ask.ID, step.State, step.Reason)
}
var a delivery.ComposeAnswer
if err := json.Unmarshal(step.Result, &a); err != nil {
return composer.Verdict{}, fmt.Errorf("the compose ask %s answered no compose answer: %w", ask.ID, err)
}
var v composer.Verdict
if err := json.Unmarshal(a.Verdict, &v); err != nil || v.V != composer.Version {
return composer.Verdict{}, fmt.Errorf("the compose ask %s answered no composer verdict of version %d (%v)", ask.ID, composer.Version, err)
}
return v, nil
}
// manifestVerdicts are the indexer's refusals of the trunk's manifests, by module, from its modules' values
// (delivery.Module.VerdictOf, ADR 0578 §3): what the sandbox's composer holds a refused module by.
type manifestVerdicts struct {
mu sync.Mutex
by map[string]delivery.ManifestVerdict
}
func (m *manifestVerdicts) put(module string, v *delivery.ManifestVerdict) {
m.mu.Lock()
defer m.mu.Unlock()
if m.by == nil {
m.by = map[string]delivery.ManifestVerdict{}
}
if v == nil {
delete(m.by, module)
return
}
m.by[module] = *v
}
func (m *manifestVerdicts) all() map[string]delivery.ManifestVerdict {
m.mu.Lock()
defer m.mu.Unlock()
out := make(map[string]delivery.ManifestVerdict, len(m.by))
for k, v := range m.by {
out[k] = v
}
return out
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go
+68 -9
View File
@@ -10,6 +10,7 @@ import (
"io"
"os"
"path/filepath"
"slices"
"sort"
"strings"
"sync"
@@ -21,10 +22,10 @@ import (
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
gobuild "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-sdk/go/announce"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-checker/internal/check"
@@ -87,7 +88,7 @@ type fakeBuildModules struct {
calls map[string]int // by `<module>.<tool>`
testTool *testing.T
// beside is, by workspace, the siblings the fetch placed in it as `<owner/repo> <commit>`, sorted: a digest echoes them
// (1929/AC2; mesh-sdk buildstep.besideOf), or the level runs whole and keeps nothing.
// (1929/AC2; mesh-build-go buildstep.besideOf), or the level runs whole and keeps nothing.
beside map[string][]string
// names are the nodes' names the last check ask carried, for the names check (ADR 0567 §12).
names []string
@@ -155,11 +156,11 @@ func (f *fakeBuildModules) serve(t *testing.T, module string, kept jetstream.Key
t.Fatal(err)
}
language := strings.TrimPrefix(module, "mesh-build-")
if err := buildstep.DeclareAskWorkers(f.ctx, f.js, language, 30*time.Second); err != nil {
if err := gobuild.DeclareAskWorkers(f.ctx, f.js, language, 30*time.Second); err != nil {
t.Fatal(err)
}
prefix := map[string]string{"mesh-build-go": "build_go_", "mesh-build-node": "build_node_"}[module]
call := func(name string) buildstep.StepFunc {
call := func(name string) gobuild.StepFunc {
return func(ctx context.Context, args map[string]any) (any, error) {
tool := prefix + name
f.mu.Lock()
@@ -168,7 +169,7 @@ func (f *fakeBuildModules) serve(t *testing.T, module string, kept jetstream.Key
return f.result(t, tool, args), nil
}
}
steps := buildstep.AskSteps{Fetch: call("fetch"), Test: call("test"),
steps := gobuild.AskSteps{Fetch: call("fetch"), Test: call("test"),
Kept: func(ctx context.Context, level, digest string) (int, bool, error) {
key, err := delivery.KeptKey(level, digest)
if err != nil {
@@ -191,7 +192,7 @@ func (f *fakeBuildModules) serve(t *testing.T, module string, kept jetstream.Key
raw, _ := value.(json.RawMessage)
return kv.Put(f.ctx, key, raw)
}
p := &buildstep.Puller{Source: &buildstep.JetStreamAsks{JS: f.js, Language: language}, Language: language, Node: "n1", Put: put,
p := &gobuild.Puller{Source: &gobuild.JetStreamAsks{JS: f.js, Language: language}, Language: language, Node: "n1", Put: put,
Capacity: func() int { return 2 }, PullWait: 200 * time.Millisecond,
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) {
if a.Check != nil {
@@ -199,7 +200,7 @@ func (f *fakeBuildModules) serve(t *testing.T, module string, kept jetstream.Key
f.names = a.Check.NodeNames
f.mu.Unlock()
}
return buildstep.RunAsk(ctx, steps, a)
return gobuild.RunAsk(ctx, steps, a)
}}
go func() { _ = p.Pull(f.ctx) }()
}
@@ -262,6 +263,58 @@ type bed struct {
mu sync.Mutex
}
// engine places each node and serves compose-candidate for the first one, as its node-engine does on the bus (answered on
// the seat verb's caller-named and plain subjects); the others never answer. It answers what it was asked.
func (b *bed) engine(t *testing.T, nodes ...string) func() []delivery.ComposeCandidateAsk {
t.Helper()
kv, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: delivery.PlacementBucket, History: 1})
if err != nil {
t.Fatal(err)
}
for _, n := range nodes {
pk, _ := delivery.PlacementNodeKey(n)
raw, _ := json.Marshal(delivery.Placement{Node: n})
if _, err := kv.Put(b.ctx, pk, raw); err != nil {
t.Fatal(err)
}
}
nc, err := nats.Connect(b.srvURL, nats.UserInfo("ctl", "pw"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
var mu sync.Mutex
var asked []delivery.ComposeCandidateAsk
answer := func(m *nats.Msg) {
ask, err := delivery.DecodeComposeCandidateAsk(m.Data)
if err != nil {
_ = m.Respond([]byte(`{"error":"` + err.Error() + `"}`))
return
}
mu.Lock()
asked = append(asked, ask)
mu.Unlock()
a := delivery.ComposeCandidateAnswer{Node: nodes[0], Candidate: ask.Candidate, Verdict: delivery.ComposeCandidateComposed,
Own: delivery.ModuleComposed{Module: "node", State: delivery.ModuleComposedState, Digest: "sha256:" + strings.Repeat("b", 64)}}
raw, err := delivery.EncodeComposeCandidateAnswer(a)
if err != nil {
t.Error(err)
}
out, _ := json.Marshal(map[string]any{"result": json.RawMessage(raw), "node": nodes[0]})
_ = m.Respond(out)
}
for _, subject := range []string{"mesh.seat.node-engine.call.compose-candidate." + nodes[0] + ".>", "mesh.seat.node-engine.tool.compose-candidate." + nodes[0]} {
if _, err := nc.Subscribe(subject, answer); err != nil {
t.Fatal(err)
}
}
return func() []delivery.ComposeCandidateAsk {
mu.Lock()
defer mu.Unlock()
return slices.Clone(asked)
}
}
// startBed starts the bus, the buckets the controller makes from the checker's and the other parts' manifests, the build
// modules' streams and the checker's durable consumers on them, one node's toolchains, the gate's source naming the trunk, and
// the fake build modules. It runs no checker.
@@ -509,14 +562,20 @@ func (b *bed) owed(t *testing.T, number int, head, state string) delivery.Owed {
func TestCheckerJudgesACandidateAndOwesItsStatus(t *testing.T) { // N6/AC5, 1300/AC5, 1445/AC1, 1929/AC1
b := startBed(t)
key := b.candidate(t, "feat/x", pull(repo, 7, head1))
// ADR 0593 §7, §9 (D12): n1 and n2 are placed; n1's engine answers compose-candidate on the bus, n2's never does.
asked := b.engine(t, "n1", "n2")
b.start(t, nil)
v := b.verdict(t, key, ended)
if v.State != delivery.VerdictPass || v.Levels[delivery.LevelUnit].Run != 2 || v.Levels[delivery.LevelIntegration].Run != 2 || v.Trunk[repo] != trunk {
t.Fatalf("verdict = %+v", v)
}
if v.Composition == nil || v.Composition.State != delivery.CompositionNotRun || len(v.Machines) != 1 || v.Machines[0] != "n1" {
if c := v.Composition; c == nil || c.State != delivery.CompositionPass || !slices.Equal(c.Machines, []string{"n1"}) ||
!strings.Contains(c.Words, "n2: "+delivery.ComposeNoAnswer) || len(v.Machines) != 1 || v.Machines[0] != "n1" {
t.Errorf("composition %+v machines %v", v.Composition, v.Machines)
}
if got := asked(); len(got) == 0 || got[0].Candidate != key {
t.Errorf("n1's engine was asked %+v, want the check's candidate %s", got, key)
}
b.fake.mu.Lock()
names := b.fake.names
b.fake.mu.Unlock()
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go
+16 -35
View File
@@ -11,8 +11,8 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-checker/internal/check"
@@ -26,10 +26,12 @@ type judgeDeps struct {
kept check.KV
previews check.KV
repos *repoModules
forge check.Forge
announce check.Announcer
builds check.BuildStates
// compositions is the assigner's bucket of compositions; nil leaves every composition not run.
// published is the indexer's packages state as it is followed (packages.go).
published *publishedPackages
forge check.Forge
announce check.Announcer
builds check.BuildStates
// compositions makes a check's composition from the engines' answers (ADR 0593 §7); nil leaves it not run.
compositions check.CompositionReader
// nodes names every node with a record; nil hands no names.
nodes *nodeNames
@@ -95,7 +97,7 @@ func newJudge(d judgeDeps) check.Judge {
nodes = d.nodes
}
return &check.Examiner{Nodes: nodes, Compositions: d.compositions, Asks: d.asks, Toolchains: d.tools,
Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Packages: d.repos, Published: d.repos, Forge: d.forge, Runs: d.repos, Builds: d.builds, Announce: d.announce, Trunks: d.trunks, Manifests: d.repos, Predefined: d.predefined, Logf: d.logf}
Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Packages: d.repos, Published: d.published, Forge: d.forge, Runs: d.repos, Builds: d.builds, Announce: d.announce, Trunks: d.trunks, Manifests: d.repos, Predefined: d.predefined, Logf: d.logf}
}
// manifestKV reads a manifest by its digest.
@@ -109,6 +111,8 @@ type manifestKV interface {
type repoModules struct {
manifests manifestKV
logf func(format string, args ...any)
// verdicts keeps the indexer's verdict on each module's manifest, for the sandbox's composer (engines.go); nil keeps none.
verdicts *manifestVerdicts
mu sync.Mutex
langs map[string]map[string]bool // by manifest digest: the languages of its artifacts
@@ -118,7 +122,6 @@ type repoModules struct {
repoOf map[string]string // by module: its repository
commit map[string]string // by module: the commit its record was computed at (the commit the mesh runs)
typed map[string]string // by module: its manifest digest
uses map[string][]string // by module: the mesh packages it uses (Inputs2.Packages, ADR 0573 §2)
}
// Put takes a change of the indexer's modules bucket in: a module's repository and the languages its manifest names.
@@ -127,15 +130,16 @@ func (r *repoModules) Put(ctx context.Context, key string, value []byte, deleted
if r.repoOf == nil {
r.repoOf, r.typed, r.langs, r.commit = map[string]string{}, map[string]string{}, map[string]map[string]bool{}, map[string]string{}
r.tsPkgs, r.path, r.named = map[string][]string{}, map[string]string{}, map[string]map[string][]string{}
r.uses = map[string][]string{}
}
if deleted {
if r.verdicts != nil {
r.verdicts.put(key, nil)
}
delete(r.repoOf, key)
delete(r.typed, key)
delete(r.commit, key)
delete(r.path, key)
delete(r.named, key)
delete(r.uses, key)
r.mu.Unlock()
return
}
@@ -144,16 +148,15 @@ func (r *repoModules) Put(ctx context.Context, key string, value []byte, deleted
if err != nil {
return
}
if r.verdicts != nil {
r.verdicts.put(key, m.VerdictOf(key))
}
r.mu.Lock()
r.repoOf[key] = m.Repository
r.typed[key] = m.Manifest
r.commit[key] = m.Commit
r.path[key] = m.Path
r.named[key] = m.ArtifactPaths
r.uses[key] = nil
if m.Inputs2 != nil {
r.uses[key] = m.Inputs2.Packages
}
_, known := r.langs[m.Manifest]
r.mu.Unlock()
if m.Manifest == "" || known {
@@ -185,28 +188,6 @@ func (r *repoModules) Put(ctx context.Context, key string, value []byte, deleted
r.mu.Unlock()
}
// LatestPackages is the latest published version of each mesh package a module of the mesh uses, and of mesh-sdk's Go
// module always (novox/hq ADR 0573 §2): the trunk merge of the module whose directory holds it, as the indexer's modules
// say. A package no module holds is left out: a check builds it at the version go.mod names.
func (r *repoModules) LatestPackages() delivery.PackageVersions {
r.mu.Lock()
defer r.mu.Unlock()
used := map[string]bool{check.SDKModulePath: true}
held := map[string]delivery.Module{}
for module, repo := range r.repoOf {
for _, p := range r.uses[module] {
used[p] = true
}
held[module] = delivery.Module{Repository: repo, Path: r.path[module], Commit: r.commit[module]}
}
pkgs := make([]string, 0, len(used))
for p := range used {
pkgs = append(pkgs, p)
}
versions, _ := delivery.PackageVersionsOf(pkgs, held)
return versions
}
// PackagePaths are the paths, relative to the repository, that the TypeScript package artifacts of a repository's modules
// are built from, sorted: the named paths the indexer keeps in each module's value (delivery.Module.ArtifactPaths; only the
// indexer reads them from a manifest, the lead's decision L8), under the module's directory. None for a module whose value
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go
+28 -13
View File
@@ -8,9 +8,10 @@ import (
"reflect"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
type memManifests map[string][]byte
@@ -113,19 +114,33 @@ func TestTheTypeScriptPackagesPathsComeFromTheIndexersModulesNotTheManifest(t *t
if got := r.PackagePaths("novox/mesh-delivery"); len(got) != 0 {
t.Errorf("another repository: %v", got)
}
// Issue 1585 (ADR 0573 §2): the latest published version of mesh-sdk's Go module and of every package a module uses is
// the commit of the module that holds it; a package no module holds is left out.
put(".", nil)
const contract = "git.novox.be/novox/mesh-controller/modules/route-proxy/contract"
user, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: strings.Repeat("b", 40), FP: "src1:" + strings.Repeat("b", 64),
Manifest: delivery.ManifestDigest([]byte(sdkManifest)), Path: "modules/x",
Inputs2: &delivery.KeptInputs2{Tree: "t", Packages: []string{contract, "git.novox.be/novox/mesh-sdk/go"}}})
if err != nil {
t.Fatal(err)
}
// ADR 0593 §2 (D8): a Go check builds every mesh package at the version the indexer's packages state publishes, read
// there and computed nowhere here: a key's module at its commit, a deleted key gone, an unreadable value left out.
func TestThePublishedVersionsAreTheIndexersPackagesState(t *testing.T) {
var p publishedPackages
put := func(module, repository, dir, commit string) {
key, _ := delivery.PackagesKey(module)
raw, err := delivery.EncodePublishedPackages(delivery.PublishedPackages{Module: module, Repository: repository, Dir: dir,
Commit: commit, Requires: strings.Repeat("e", 64),
Packages: []delivery.PackageDigest{{Package: module, Digest: strings.Repeat("d", 64)}}, At: time.Now().UTC()})
if err != nil {
t.Fatal(err)
}
p.Put(key, raw, false)
}
r.Put(context.Background(), "x", user, false)
if got := r.LatestPackages(); !reflect.DeepEqual(got, delivery.PackageVersions{"git.novox.be/novox/mesh-sdk/go": strings.Repeat("a", 40)}) {
t.Errorf("latest packages = %v, want mesh-sdk at its module's commit and no unheld contract", got)
const sdk, build = "git.novox.be/novox/mesh-sdk/go", "git.novox.be/novox/mesh-catalog/modules/mesh-build-go"
put(sdk, "novox/mesh-sdk", "go", strings.Repeat("a", 40))
put(build, "novox/mesh-catalog", "modules/mesh-build-go", strings.Repeat("b", 40))
p.Put("broken", []byte("{"), false)
if got := p.LatestPackages(); !reflect.DeepEqual(got, delivery.PackageVersions{sdk: strings.Repeat("a", 40), build: strings.Repeat("b", 40)}) {
t.Errorf("latest packages = %v", got)
}
key, _ := delivery.PackagesKey(build)
p.Put(key, nil, true)
if got := p.LatestPackages(); len(got) != 1 || got[sdk] == "" {
t.Errorf("after a delete: %v", got)
}
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go
+32 -20
View File
@@ -20,9 +20,9 @@ import (
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-sdk/go/announce"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-checker/internal/check"
@@ -169,23 +169,19 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if kv := process.OptionalBucket(ctx, js, delivery.BuilderBuildsBucket, "the line says nothing of the builds until the checker starts again", logger.Printf); kv != nil {
builds = buildReader{kvStore{kv}}
}
// The assigner's compositions (ADR 0528 clause 5, issue 1297): their bucket is looked up at each check until it is
// there, so an assigner delivered after the checker started is read at the next check, and followed from then on.
compositions := &lazyCompositions{open: func(at context.Context) (check.CompositionReader, error) {
kv, err := js.KeyValue(at, delivery.AssignerCompositionsBucket)
if err != nil {
return nil, err
}
cw := compositionWatch{kv: kv}
if pkv, err := js.KeyValue(at, delivery.PresenceBucket); err == nil {
cw.presence = pkv
}
if err := followCompositions(ctx, kv, ch); err != nil {
return nil, fmt.Errorf("the bucket %s cannot be watched: %w", delivery.AssignerCompositionsBucket, err)
}
logger.Printf("the assigner's compositions are read from %s", delivery.AssignerCompositionsBucket)
return cw, nil
}}
// The check's composition is the checker's, from what each placed node's engine answers (ADR 0593 §7, §9).
answerBound, err := composeAnswerBound(getenv("MESH_CHECKER_SETTINGS"))
if err != nil {
return err
}
verdicts := &manifestVerdicts{}
compositions := check.EngineCompositions{
Asker: engineAsks{call: caller.CallOn},
Placed: placedNodes{kv: func(at context.Context) (jetstream.KeyValue, error) { return js.KeyValue(at, delivery.PlacementBucket) }}.Placed,
Bound: answerBound,
Sandbox: seatSandbox{asker: buildrun.Asker{Publisher: asks, Waiter: waiter, Bound: stepBound}},
Verdicts: verdicts.all,
}
// The tries (ADR 0501 §5) are judged by the same judge, held by the same deliveries.
tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, NotTaken: stepBound, Logf: logger.Printf}
announce := func(ctx context.Context, event, key string) {
@@ -194,12 +190,28 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
logger.Printf("%s of %s was written and the part that reads it could not be told: %v", event, key, err)
}
}
repos := &repoModules{manifests: kvStore{buckets[delivery.ManifestsBucket]}, logf: logger.Printf}
repos := &repoModules{manifests: kvStore{buckets[delivery.ManifestsBucket]}, logf: logger.Printf, verdicts: verdicts}
compositions.Composer = func() string { return repos.CommitOf(check.NodeEngineModule) }
compositions.Manifests = repos
// The indexer's packages state (ADR 0593 §2): followed once the bus has it, so a Go check builds every mesh package at
// its published version; until then a check builds each at the version its go.mod names.
published := &publishedPackages{}
process.AwaitBucket(ctx, js, conn, delivery.PackagesBucket, "a Go check builds each mesh package at the version its go.mod names",
logger.Printf).Ready(func(kv jetstream.KeyValue) {
w, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { published.Put(e.Key(), e.Value(), !opOf(e)) })
if err != nil {
logger.Printf("%s cannot be watched, so a Go check builds each mesh package at the version its go.mod names: %v", delivery.PackagesBucket, err)
return
}
go func() {
_ = follow(ctx, w, func(e jetstream.KeyValueEntry) { published.Put(e.Key(), e.Value(), !opOf(e)) })
}()
})
ch.Runs = repos // the node-engine a check reads beside at its running commit (ADR 0548 §6)
ch.Judge = newJudge(judgeDeps{
asks: buildrun.Asker{Publisher: asks, Waiter: waiter, Bound: stepBound}, tools: tools, trunks: trunks,
kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]},
repos: repos, builds: builds, forge: forgeReads{call: caller.Call}, logf: logger.Printf,
repos: repos, published: published, builds: builds, forge: forgeReads{call: caller.Call}, logf: logger.Printf,
announce: announce, compositions: compositions,
nodes: openNodeNames(ctx, js, logger.Printf),
predefined: keptPredefined{process.AwaitBucket(ctx, js, conn, delivery.IndexerModule+"_"+delivery.IndexerPredefinedState,
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/packages.go
+53
View File
@@ -0,0 +1,53 @@
package main
import (
"sync"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// publishedPackages is the indexer's state `packages` as the checker follows it (novox/hq ADR 0593 §2): by mesh Go
// module, the latest published commit of its repository the indexer read its packages at. The indexer alone computes a
// package's version; the checker reads it, so a Go check builds every mesh package at the version the mesh publishes
// (check.Examiner.Published) and computes none.
type publishedPackages struct {
mu sync.Mutex
byKey map[string]delivery.PublishedPackages
unread map[string]string
}
// Put takes one key of the state in: its value, or its deletion. A value that does not decode is kept out and remembered
// with why, never read as an older one.
func (p *publishedPackages) Put(key string, raw []byte, deleted bool) {
p.mu.Lock()
defer p.mu.Unlock()
if p.byKey == nil {
p.byKey, p.unread = map[string]delivery.PublishedPackages{}, map[string]string{}
}
delete(p.byKey, key)
delete(p.unread, key)
if deleted {
return
}
v, err := delivery.DecodePublishedPackages(raw)
if err != nil {
p.unread[key] = err.Error()
return
}
p.byKey[key] = v
}
// LatestPackages is the latest published version of every mesh Go module the indexer holds, by module path: the commit
// its repository was published at. A module the state does not hold is left out: a check builds it at the version
// go.mod names.
func (p *publishedPackages) LatestPackages() delivery.PackageVersions {
p.mu.Lock()
defer p.mu.Unlock()
out := delivery.PackageVersions{}
for _, v := range p.byKey {
if v.Module != "" && v.Commit != "" {
out[v.Module] = v.Commit
}
}
return out
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/refused_test.go
+1 -1
View File
@@ -12,7 +12,7 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
)
// 1945 (a part exits at start for a bucket it works without): a bus that refuses every toolchains bucket leaves the
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/settings.go
+44
View File
@@ -0,0 +1,44 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The checker's own settings (its config, written by the node-engine into ${dir:state}/settings.json, named to the
// process by MESH_CHECKER_SETTINGS): compose-answer-bound, the seconds a check waits for a node's engine to answer
// compose-candidate (novox/hq ADR 0593 §7, §9: 60 by default; the checker owns it).
// composeAnswerBound reads compose-answer-bound from the settings file at path: delivery.ComposeAnswerBound where no
// file or no value is there; a value that is no number of seconds from 1 to 600 is refused, naming it.
func composeAnswerBound(path string) (time.Duration, error) {
if path == "" {
return delivery.ComposeAnswerBound, nil
}
raw, err := os.ReadFile(path)
if errors.Is(err, fs.ErrNotExist) {
return delivery.ComposeAnswerBound, nil
}
if err != nil {
return 0, err
}
var s struct {
Bound *float64 `json:"compose-answer-bound"`
}
if err := json.Unmarshal(raw, &s); err != nil {
return 0, fmt.Errorf("%s is no settings file: %w", path, err)
}
if s.Bound == nil {
return delivery.ComposeAnswerBound, nil
}
if *s.Bound < 1 || *s.Bound > 600 || *s.Bound != float64(int(*s.Bound)) {
return 0, fmt.Errorf("compose-answer-bound is %v: a whole number of seconds from 1 to 600", *s.Bound)
}
return time.Duration(*s.Bound) * time.Second, nil
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/settings_test.go
+35
View File
@@ -0,0 +1,35 @@
package main
import (
"os"
"path/filepath"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// ADR 0593 §7: the answer bound is the checker's setting compose-answer-bound, 60 s when the file or the value is
// absent; a value that is no whole number of seconds from 1 to 600 stops the start, naming it.
func TestTheAnswerBoundIsTheCheckersSetting(t *testing.T) {
dir := t.TempDir()
for _, row := range []struct {
body string
want time.Duration
err bool
}{{"", delivery.ComposeAnswerBound, false}, {`{}`, delivery.ComposeAnswerBound, false},
{`{"compose-answer-bound": 90}`, 90 * time.Second, false}, {`{"compose-answer-bound": 0}`, 0, true},
{`{"compose-answer-bound": 1.5}`, 0, true}} {
path := filepath.Join(dir, "settings.json")
os.Remove(path)
if row.body != "" {
if err := os.WriteFile(path, []byte(row.body), 0o600); err != nil {
t.Fatal(err)
}
}
got, err := composeAnswerBound(path)
if (err != nil) != row.err || got != row.want {
t.Errorf("%q: %s, %v", row.body, got, err)
}
}
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/toolchains.go
+1 -1
View File
@@ -14,7 +14,7 @@ import (
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
)
// toolchainStreams are the stream each build module carries its toolchain.reported event on, as the bus names it
modules/mesh-delivery-checker/files/settings.json
+3
View File
@@ -0,0 +1,3 @@
{
"compose-answer-bound": 60
}
modules/mesh-delivery-checker/go.mod
+6 -1
View File
@@ -3,12 +3,15 @@ module mesh-delivery-checker
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
require git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a
require (
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun v0.0.0-00010101000000-000000000000
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
github.com/google/go-tpm v0.9.8 // indirect
github.com/klauspost/compress v1.20.0 // indirect
@@ -20,3 +23,5 @@ require (
golang.org/x/sys v0.48.0 // indirect
golang.org/x/time v0.15.0 // indirect
)
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun => ../mesh-delivery-builder/buildrun
modules/mesh-delivery-checker/go.sum
+4 -2
View File
@@ -1,5 +1,7 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a h1:oxgksCiq272/afXbUION/dxtAS5MPGnmyq6+AZQxmXg=
git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a/go.mod h1:kpsX4Pgss5EeuBvel1zWa2ogj7MkR3jZso6oa/J0YUQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
modules/mesh-delivery-checker/internal/check/awaiting.go
-163
View File
@@ -1,163 +0,0 @@
package check
import (
"context"
"errors"
"fmt"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// ErrAssignerNotRunning is what a wait for a composition answers when no instance of the assigner is online: nobody is
// making it, so the check does not wait for it (issue 1297). The verdict says the composition is not judged, and when the
// assigner starts and writes it, the checker takes it in then (Checker.Composed).
var ErrAssignerNotRunning = errors.New(delivery.AssignerNotRunning)
// PresenceEvent is a change of the watchman's presence bucket, or Settled once what stands was handed.
type PresenceEvent struct {
Key string
Online bool
Settled bool
}
// CompositionEvent is a change of the composition's key (its value, nil for a delete), or Settled once what stands was
// handed.
type CompositionEvent struct {
Value []byte
Settled bool
}
// AwaitComposition answers the assigner's composition once one is written that does not wait, while an instance of the
// assigner is online by the watchman's presence (`part.mesh-delivery-assigner.<instance>`). Once both watches handed
// what stands, no instance online is ErrAssignerNotRunning at once, and so is the last instance going offline during the
// wait. No timer: every answer follows an event; ctx ending is ctx's error.
func AwaitComposition(ctx context.Context, presence <-chan PresenceEvent, comps <-chan CompositionEvent) (delivery.CompositionVerdict, error) {
online := map[string]bool{}
presenceSettled, compSettled := false, false
for {
select {
case <-ctx.Done():
return delivery.CompositionVerdict{}, ctx.Err()
case e, ok := <-presence:
if !ok {
return delivery.CompositionVerdict{}, errors.New("the watch of the assigner's presence ended")
}
if e.Settled {
presenceSettled = true
break
}
if _, part, instance, err := delivery.ParsePresenceKey(e.Key); err == nil && part == delivery.AssignerModule {
if e.Online {
online[instance] = true
} else {
delete(online, instance)
}
}
case e, ok := <-comps:
if !ok {
return delivery.CompositionVerdict{}, errors.New("the watch of the composition ended")
}
if e.Settled {
compSettled = true
break
}
if e.Value == nil {
break
}
v, err := delivery.DecodeCompositionVerdict(e.Value)
if err != nil {
return delivery.CompositionVerdict{}, fmt.Errorf("the composition cannot be read: %w", err)
}
if v.State != delivery.CompositionWaits {
return v, nil
}
}
if presenceSettled && compSettled && len(online) == 0 {
return delivery.CompositionVerdict{}, ErrAssignerNotRunning
}
}
}
// withComposition is v with the assigner's composition taken in: a failed one fails the check, naming its machine.
func withComposition(v *delivery.Verdict, comp delivery.CompositionVerdict) {
v.Composition = comp.Composition()
v.Took.Composition = comp.Took
if comp.State == delivery.CompositionFail {
machine := comp.Machine
if machine == "" {
machine = "the composition"
}
v.State, v.Failed = delivery.VerdictFail, delivery.FailedFirst{Machine: machine, Words: comp.Words}
}
}
// notJudged says whether a verdict's composition was not judged because the assigner was not running.
func notJudged(v delivery.Verdict) bool {
return v.Composition != nil && v.Composition.State == delivery.CompositionNotRun && v.Composition.Words == delivery.AssignerNotRunning
}
// unanswered says whether a verdict failed only on its composition, and that only because nodes the change places did
// not answer its placement in time (delivery.CompositionUnanswered, ADR 0559 §5): its tests passed (a composition is read
// only into a passing verdict), so a later composition of the same heads is taken in on its event, with no new check
// (ADR 0548 §1: a part of the mesh not heard is judged again when it is heard).
func unanswered(v delivery.Verdict) bool {
return v.State == delivery.VerdictFail && v.Composition != nil && v.Composition.State == delivery.CompositionFail &&
strings.Contains(v.Composition.Words, delivery.CompositionUnanswered) && v.Failed.Words == v.Composition.Words
}
// Composed takes in a composition the assigner wrote after the check's verdict said it was not judged because the
// assigner was not running (the assigner started since), or after it failed only because placed nodes had not answered
// (unanswered): the verdict and its status are written again with it. A composition of heads no candidate holds now, or
// of a verdict that judged its composition otherwise, is left.
func (ch *Checker) Composed(ctx context.Context, comp delivery.CompositionVerdict) {
ch.init()
if comp.State == delivery.CompositionWaits {
return
}
ckey, err := delivery.CandidatesKey(comp.Group)
if err != nil {
return
}
ch.mu.Lock()
c, ok := ch.known[ckey]
ch.mu.Unlock()
if !ok {
return
}
cs := CaseOf(c, ch.heads())
if cs == nil || cs.HeadsHash != comp.Heads {
return
}
cur, rev, found, err := ch.Verdicts.Get(ctx, cs.Key)
if err != nil || !found {
return
}
v, err := delivery.DecodeVerdict(cur)
if err != nil {
return
}
switch {
case v.State == delivery.VerdictPass && notJudged(v):
case unanswered(v) && comp.Words != v.Composition.Words:
// The nodes answered since (or another failure stands now): the verdict is its tests' pass with this composition.
v.State, v.Failed = delivery.VerdictPass, delivery.FailedFirst{}
default:
return
}
withComposition(&v, comp)
if v.Composition != nil && v.Composition.State != delivery.CompositionNotRun {
v.Composition.FPSet = cs.FPSet
}
data, err := delivery.EncodeVerdict(v)
if err != nil {
ch.logf("%s: the verdict with its composition cannot be written: %v", cs.Key, err)
return
}
if _, err := ch.Verdicts.Update(ctx, cs.Key, data, rev); err != nil {
ch.logf("%s: the verdict with its composition could not be written: %v", cs.Key, err)
return
}
ch.writeOwed(ctx, cs, v, Outcome{Verdict: v})
ch.logf("%s: the composition came after the verdict: %s", cs.Key, comp.State)
}
modules/mesh-delivery-checker/internal/check/awaiting_test.go
-170
View File
@@ -1,170 +0,0 @@
package check
import (
"context"
"errors"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
func assignerOn(instance string, online bool) PresenceEvent {
key, _ := delivery.PresencePartKey(delivery.AssignerModule, instance)
return PresenceEvent{Key: key, Online: online}
}
func compEvent(t *testing.T, v delivery.CompositionVerdict) CompositionEvent {
t.Helper()
data, err := delivery.EncodeCompositionVerdict(v)
if err != nil {
t.Fatal(err)
}
return CompositionEvent{Value: data}
}
var passed = delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionPass, SDK: "x", Machines: []string{"g14"}}
// 1297/B3d: with no assigner online, the wait answers ErrAssignerNotRunning as soon as both watches handed what stands.
func TestNoAssignerAnswersAtOnce(t *testing.T) {
presence, comps := make(chan PresenceEvent, 4), make(chan CompositionEvent, 4)
presence <- assignerOn("novox", false)
presence <- PresenceEvent{Settled: true}
comps <- CompositionEvent{Settled: true}
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
if _, err := AwaitComposition(ctx, presence, comps); !errors.Is(err, ErrAssignerNotRunning) {
t.Fatalf("answered %v, want the assigner not running", err)
}
}
// 1297/B3d: with the assigner online it waits for the key, past a composition that waits, and answers the verdict.
func TestAnOnlineAssignerIsWaitedFor(t *testing.T) {
presence, comps := make(chan PresenceEvent, 4), make(chan CompositionEvent, 4)
presence <- assignerOn("novox", true)
presence <- PresenceEvent{Settled: true}
comps <- CompositionEvent{Settled: true}
done := make(chan error, 1)
var got delivery.CompositionVerdict
go func() {
var err error
got, err = AwaitComposition(context.Background(), presence, comps)
done <- err
}()
comps <- compEvent(t, delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionWaits, Words: "w"})
select {
case err := <-done:
t.Fatalf("answered before the composition was written: %v", err)
case <-time.After(50 * time.Millisecond):
}
comps <- compEvent(t, passed)
if err := <-done; err != nil || got.State != delivery.CompositionPass {
t.Fatalf("answered %+v, %v", got, err)
}
// And the last instance going offline during a wait ends it as not running.
presence2, comps2 := make(chan PresenceEvent, 4), make(chan CompositionEvent, 4)
presence2 <- assignerOn("novox", true)
presence2 <- PresenceEvent{Settled: true}
comps2 <- CompositionEvent{Settled: true}
presence2 <- assignerOn("novox", false)
if _, err := AwaitComposition(context.Background(), presence2, comps2); !errors.Is(err, ErrAssignerNotRunning) {
t.Errorf("the assigner going offline answered %v", err)
}
}
// 1297/B3d: a passing check with no assigner says at once, loud, that its composition is not judged; never a pass of it.
func TestAPassingCheckWithNoAssignerSaysTheCompositionIsNotJudged(t *testing.T) {
b := newBench(t)
b.ex.Compositions = &fakeCompositions{err: ErrAssignerNotRunning}
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
if v.Composition == nil || v.Composition.State != delivery.CompositionNotRun || v.Composition.Words != delivery.AssignerNotRunning {
t.Fatalf("composition = %+v", v.Composition)
}
if words := compositionWords(v.Composition, 0); words != "composition not judged: the assigner is not running" {
t.Errorf("the line says %q", words)
}
}
// 1297/B3d: the assigner starting writes the composition later, and the checker takes it into the verdict that said it
// was not judged, writing the status again; a failed one fails the check.
func TestTheAssignerStartingBringsTheComposition(t *testing.T) {
for _, comp := range []delivery.CompositionVerdict{passed,
{Group: "feat/x", Heads: "", State: delivery.CompositionFail, SDK: "x", Machine: "g14", Words: "g14 runs web, which the change removes"}} {
ch, cs := checkerWithNotJudged(t)
comp.Heads = cs.HeadsHash
ch.Composed(context.Background(), comp)
raw, _, _, _ := ch.Verdicts.Get(context.Background(), cs.Key)
v, err := delivery.DecodeVerdict(raw)
if err != nil || v.Composition == nil || v.Composition.State != comp.State {
t.Fatalf("after the composition came, the verdict is %+v, %v", v, err)
}
if comp.State == delivery.CompositionFail && (v.State != delivery.VerdictFail || v.Failed.Machine != "g14") {
t.Errorf("a failed composition left the verdict %+v", v)
}
owed := ch.Owed.(*memKV)
keys, _ := owed.Keys(context.Background())
if len(keys) == 0 {
t.Fatal("no status was written again")
}
data, _, _, _ := owed.Get(context.Background(), keys[0])
o, _ := delivery.DecodeCheckerOwed(data)
if !strings.Contains(o.Text, "composition passed") && !strings.Contains(o.Text, "g14 runs web") {
t.Errorf("the status's line says %q", o.Text)
}
}
// ADR 0559 §5: a verdict that failed only because a placed node had not answered takes the same heads' later
// composition in on its event, a pass making it pass with no new check; a composition failing the same way is left.
ch, cs := checkerWithNotJudged(t)
silent := delivery.CompositionVerdict{Group: cs.Group, Heads: cs.HeadsHash, State: delivery.CompositionFail, SDK: "x", Machine: "g14",
Words: "g14 (offline) " + delivery.CompositionUnanswered}
ch.Composed(context.Background(), silent)
for _, row := range []struct {
name string
comp delivery.CompositionVerdict
state string
}{
{"the same failure again", silent, delivery.VerdictFail},
{"the node's answer", delivery.CompositionVerdict{Group: cs.Group, Heads: cs.HeadsHash, State: delivery.CompositionPass, SDK: "x"}, delivery.VerdictPass},
} {
ch.Composed(context.Background(), row.comp)
raw, _, _, _ := ch.Verdicts.Get(context.Background(), cs.Key)
if v, err := delivery.DecodeVerdict(raw); err != nil || v.State != row.state || v.Composition == nil || v.Composition.State != row.comp.State {
t.Errorf("%s: the verdict is %+v, %v", row.name, v, err)
}
}
}
// checkerWithNotJudged is a checker holding a candidate whose passing verdict said its composition was not judged.
func checkerWithNotJudged(t *testing.T) (*Checker, *Case) {
t.Helper()
cs := caseOf(t, pullOf(repoA, 7, head1))
raw := candidateOf(t, cs.Group, pullOf(repoA, 7, head1))
ch := &Checker{Verdicts: newMemKV(), Owed: newMemKV(), Trunks: fixedTrunks{cs.Trunk}}
ckey, _ := delivery.CandidatesKey(cs.Group)
ch.init()
c, err := delivery.DecodeCandidate(raw)
if err != nil {
t.Fatal(err)
}
ch.known[ckey] = c
if got := CaseOf(c, ch.heads()); got == nil || got.Key != cs.Key {
t.Fatalf("the case is %+v, want %s", got, cs.Key)
}
v := delivery.Verdict{Group: cs.Group, Heads: cs.Heads, State: delivery.VerdictPass,
Levels: map[string]delivery.LevelResult{delivery.LevelUnit: {Run: 1}, delivery.LevelIntegration: {Run: 1}},
Composition: &delivery.Composition{State: delivery.CompositionNotRun, Words: delivery.AssignerNotRunning}}
data, err := delivery.EncodeVerdict(v)
if err != nil {
t.Fatal(err)
}
if _, err := ch.Verdicts.Create(context.Background(), cs.Key, data); err != nil {
t.Fatal(err)
}
return ch, cs
}
type fixedTrunks struct{ heads map[string]string }
func (f fixedTrunks) Heads() map[string]string { return f.heads }
modules/mesh-delivery-checker/internal/check/beside.go
+8 -5
View File
@@ -27,18 +27,21 @@ const (
catalogueRepository = "novox/mesh-catalog"
nodeEngineRepo = "novox/mesh-host"
nodeEngineModule = "mesh-host"
controllerRepo = "novox/mesh-controller"
// networkingRepository holds route-proxy, whose manifest the node-engine's proofs of a composition read.
networkingRepository = "novox/mesh-networking"
)
// besideRules are the repositories each core repository's tests read, as the tests state it (mesh-sdk go/internal/beside
// beside.go): the catalogue, the node-engine, the SDK, and the delivery's pinned forge dependencies (mesh-catalog
// checks/deps). Which commit of each is besideCommit's rule. No repository's tests read the controller, which ended by its
// unassignment (novox/hq ADR 0590, issue 1126); its own repository keeps its siblings while it holds a module.
// unassignment (novox/hq ADR 0590, issue 1126), and its repository is emptied and archived (ADR 0593 §11), so it has no
// siblings placed either. The catalogue's tests read the node-engine since its composer command moved there (ADR 0593
// §7): mesh-build-go builds it for a compose ask. The node-engine's read mesh-networking since route-proxy's proofs of a
// composition moved into them (ADR 0593).
var besideRules = map[string][]string{
controllerRepo: {catalogueRepository, nodeEngineRepo, SDKRepository},
nodeEngineRepo: {catalogueRepository, SDKRepository},
nodeEngineRepo: {catalogueRepository, SDKRepository, networkingRepository},
SDKRepository: {catalogueRepository, nodeEngineRepo},
catalogueRepository: {SDKRepository},
catalogueRepository: {SDKRepository, nodeEngineRepo},
"novox/mesh-delivery": {SDKRepository, catalogueRepository},
}
modules/mesh-delivery-checker/internal/check/composition.go
+28
View File
@@ -0,0 +1,28 @@
package check
import (
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// withComposition is v with the check's composition taken in: a failed one fails the check, naming its machine.
func withComposition(v *delivery.Verdict, comp delivery.CompositionVerdict) {
v.Composition = comp.Composition()
v.Took.Composition = comp.Took
if comp.State == delivery.CompositionPass && len(comp.Warnings) > 0 && v.Composition != nil {
v.Composition.Words = strings.Join(comp.Warnings, "; ") // passed with a warning (ADR 0593 §9)
}
if comp.State == delivery.CompositionFail {
machine := comp.Machine
if machine == "" {
machine = "the composition"
}
f := delivery.FailedFirst{Machine: machine, Words: comp.Words}
if v.State == delivery.VerdictFail {
addFailure(v, f) // the tests failed too: the composition is named after them (the operator, 2026-11-02)
return
}
v.State, v.Failed = delivery.VerdictFail, f
}
}
modules/mesh-delivery-checker/internal/check/composition_test.go
+14 -3
View File
@@ -9,15 +9,21 @@ import (
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// fakeCompositions answers the assigner's composition of a check from memory, recording the keys asked.
// fakeCompositions answers a check's composition from memory, recording the keys asked.
type fakeCompositions struct {
v delivery.CompositionVerdict
err error
asked []string
// waits answers only when its context ends.
waits bool
}
func (f *fakeCompositions) Await(_ context.Context, key string) (delivery.CompositionVerdict, error) {
f.asked = append(f.asked, key)
func (f *fakeCompositions) Compose(ctx context.Context, c *Case) (delivery.CompositionVerdict, error) {
f.asked = append(f.asked, c.Key)
if f.waits {
<-ctx.Done()
return delivery.CompositionVerdict{}, ctx.Err()
}
return f.v, f.err
}
@@ -88,3 +94,8 @@ func TestACompositionNotHadIsNotRun(t *testing.T) {
t.Errorf("a try asked for a composition: %v", f2.asked)
}
}
// fixedTrunks are trunk heads that never move.
type fixedTrunks struct{ heads map[string]string }
func (f fixedTrunks) Heads() map[string]string { return f.heads }
modules/mesh-delivery-checker/internal/check/engines.go
+238
View File
@@ -0,0 +1,238 @@
package check
import (
"context"
"encoding/json"
"fmt"
"sort"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/composer"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// **A check's composition is the checker's, made from what each node's engine answers** (novox/hq ADR 0593 §7, §9).
// Only a node knows whether a change composes there: the checker asks every placed node's engine
// `node-engine.compose-candidate` in parallel, with the check's candidate, and waits at most its answer bound
// (compose-answer-bound, 60 s by default). A node that refuses fails the check, naming the node and each module with the
// engine's words; a node that does not answer within the bound, answers unready or cannot be asked is named, "not composed
// on that node: no answer", and the check passes with that warning: its own composition judges again when it applies,
// and the canary rule holds. The checker decides by the answer's verdict, never by its words.
//
// **A group that changes the node-engine's composition code** (a member of the node-engine's repository) is not judged by
// the running engines, which cannot judge their own next code: each node's engine answers the composition input it
// composed from (ComposeCandidateAsk.Input), and the composer program built from that member's head composes it in a
// build module's sandbox (Sandbox), with every manifest of the catalogue as the case leaves it and the indexer's verdicts.
// EngineAsker asks one node's engine to compose a check's candidate (the verb node-engine.compose-candidate on that node).
type EngineAsker interface {
ComposeCandidate(ctx context.Context, node string, ask delivery.ComposeCandidateAsk) (delivery.ComposeCandidateAnswer, error)
}
// Sandbox runs a compose job on the Go build seat and answers the composer's verdict.
type Sandbox interface {
Compose(ctx context.Context, job delivery.ComposeJob) (composer.Verdict, error)
}
// EngineCompositions is the checker's composition of a check (Examiner.Compositions).
type EngineCompositions struct {
Asker EngineAsker
// Placed names every node in placement now (the seat mesh-placement's state), or why it cannot be read: then no node
// can be asked, and the composition is not made (an error), never passed with nobody asked.
Placed func() ([]string, error)
// Bound is the answer bound (the setting compose-answer-bound); zero is delivery.ComposeAnswerBound.
Bound time.Duration
// Composer is the commit of the node-engine the mesh runs, named as what composed (ADR 0548 §6's running commit).
Composer func() string
// Sandbox runs the composer of a node-engine member's head; nil leaves such a group judged by the running engines,
// said in a warning.
Sandbox Sandbox
// Manifests are the indexer's trunk manifest digests by module, and Verdicts its refusals of them: the catalogue the
// sandbox composes with, before the case's own manifests are laid over.
Manifests TrunkManifests
Verdicts func() map[string]delivery.ManifestVerdict
}
// composerHead is the node-engine member's head of a case, or empty.
func composerHead(c *Case) string {
if m := c.member(nodeEngineRepo); m != nil {
return m.Head
}
return ""
}
// Compose asks every placed node and judges their answers (JudgeEngines). An error is only a case it cannot ask at all.
func (e EngineCompositions) Compose(ctx context.Context, c *Case) (delivery.CompositionVerdict, error) {
began := time.Now()
key, err := delivery.CompositionKey(c.Group, c.HeadsHash)
if err != nil {
return delivery.CompositionVerdict{}, err
}
var nodes []string
if e.Placed != nil {
if nodes, err = e.Placed(); err != nil {
return delivery.CompositionVerdict{}, fmt.Errorf("the placement cannot be read, so no node can be asked: %w", err)
}
}
bound := e.Bound
if bound <= 0 {
bound = delivery.ComposeAnswerBound
}
head := composerHead(c)
sandboxed := head != "" && e.Sandbox != nil
answers := map[string]delivery.ComposeCandidateAnswer{}
silent := map[string]string{}
var mu sync.Mutex
var wg sync.WaitGroup
for _, node := range nodes {
wg.Add(1)
go func() {
defer wg.Done()
at, cancel := context.WithTimeout(ctx, bound)
defer cancel()
a, err := e.Asker.ComposeCandidate(at, node, delivery.ComposeCandidateAsk{Candidate: key, Input: sandboxed})
if err == nil && sandboxed && a.Verdict != delivery.ComposeCandidateUnready {
// The composer's build and run are the check's to bound, not the engine's answer bound. A composer that
// cannot run is no silent node (ADR 0593 §9 lets only that pass): the node's composition fails, naming
// the head.
if composed, serr := e.inSandbox(ctx, c, head, node, a); serr != nil {
a = delivery.ComposeCandidateAnswer{V: a.V, Node: node, Candidate: a.Candidate, Verdict: delivery.ComposeCandidateRefused,
Refused: "the composer at the node-engine's head " + head + " could not compose it: " + serr.Error()}
} else {
a = composed
}
}
mu.Lock()
defer mu.Unlock()
switch {
case err != nil:
silent[node] = err.Error()
case a.Verdict == delivery.ComposeCandidateUnready:
silent[node] = "unready"
default:
answers[node] = a
}
}()
}
wg.Wait()
// What composed, named by the node-engine's commit: the member's head in the sandbox, else the one the mesh runs.
composerName := "node-engine"
if sandboxed {
composerName = head
} else if e.Composer != nil && e.Composer() != "" {
composerName = e.Composer()
}
v := JudgeEngines(c.Group, c.HeadsHash, composerName, nodes, answers, silent)
if head != "" && e.Sandbox == nil {
v.Warnings = append(v.Warnings, "the group changes the node-engine's composition code and was composed by the running engines")
}
v.Took = time.Since(began).Milliseconds()
return v, nil
}
// JudgeEngines is the check's composition verdict from the engines' answers (ADR 0593 §7, §9): failed when any node's
// answer is refused, naming each node and each module refused with its words (the first refusing node is the machine);
// passed otherwise. Every node in nodes without an answer (silent, by node with why, for the journal only) is named
// with delivery.ComposeNoAnswer as a warning. composer names what composed, for the line. Pure.
func JudgeEngines(group, heads, composerName string, nodes []string, answers map[string]delivery.ComposeCandidateAnswer, silent map[string]string) delivery.CompositionVerdict {
v := delivery.CompositionVerdict{Group: group, Heads: heads, State: delivery.CompositionPass, SDK: composerName}
sorted := append([]string(nil), nodes...)
sort.Strings(sorted)
var fails []string
for _, node := range sorted {
a, answered := answers[node]
if !answered {
v.Warnings = append(v.Warnings, node+": "+delivery.ComposeNoAnswer)
continue
}
v.Machines = append(v.Machines, node)
if a.Verdict != delivery.ComposeCandidateRefused {
continue
}
if v.Machine == "" {
v.Machine = node
}
var words []string
if a.Refused != "" {
words = append(words, a.Refused)
}
for _, m := range append(append([]delivery.ModuleComposed(nil), a.Modules...), a.Own) {
if m.State == delivery.ModuleRefusedState {
words = append(words, m.Module+": "+m.Words)
}
}
if len(words) == 0 {
words = append(words, "refused, naming no module")
}
fails = append(fails, "not composed on "+node+": "+strings.Join(words, "; "))
}
if len(fails) > 0 {
v.State, v.Words = delivery.CompositionFail, strings.Join(fails, "; ")
}
if len(sorted) == 0 {
v.Warnings = append(v.Warnings, "no node is placed, so the change was composed on none")
}
return v
}
// inSandbox composes a node's answer again with the composer built from the node-engine member's head, over the input the
// node's engine composed from: the node's answer becomes the sandbox's, refused with the composer's words where the
// composer refuses the node.
func (e EngineCompositions) inSandbox(ctx context.Context, c *Case, head, node string, a delivery.ComposeCandidateAnswer) (delivery.ComposeCandidateAnswer, error) {
if len(a.Input) == 0 {
return a, fmt.Errorf("the engine of %s answered no composition input, so the node-engine's head cannot compose it", node)
}
manifests := map[string]composer.Manifest{}
if e.Manifests != nil {
for name, digest := range e.Manifests.ManifestDigests() {
manifests[name] = composer.Manifest{Digest: digest}
}
}
for _, p := range c.Pulls {
for name, m := range p.Fps {
if m.Manifest != "" {
manifests[name] = composer.Manifest{Digest: m.Manifest}
}
}
}
verdicts := map[string]delivery.ManifestVerdict{}
if e.Verdicts != nil {
for name, v := range e.Verdicts() {
verdicts[name] = v
}
}
for _, p := range c.Pulls {
for name := range p.Fps {
delete(verdicts, name) // a manifest the change brings is the checker's own judge's (ADR 0578 §2)
}
}
req := composer.Request{V: composer.Version, Input: a.Input, Depth: composer.DepthCompose, Machines: []string{node},
Cases: []composer.Case{{Name: "change", Manifests: manifests, Verdicts: verdicts}}}
raw, err := json.Marshal(req)
if err != nil {
return a, err
}
job := delivery.ComposeJob{Fetch: map[string]any{"repository": nodeEngineRepo, "commit": head}, Dir: delivery.RootPath,
Package: composer.Package, Request: raw, Reader: "the node-engine head of " + c.Group}
verdict, err := e.Sandbox.Compose(ctx, job)
if err != nil {
return a, fmt.Errorf("the composer at the node-engine's head %s did not answer for %s: %w", head, node, err)
}
out := delivery.ComposeCandidateAnswer{V: a.V, Node: node, Candidate: a.Candidate, Verdict: delivery.ComposeCandidateComposed}
for _, cv := range verdict.Cases {
if m, ok := cv.Machines[node]; ok && !m.Composes {
out.Verdict = delivery.ComposeCandidateRefused
out.Refused = strings.Join(append(append([]string{m.Reason}, m.Problems...), m.Refusals...), "; ")
}
for name, read := range cv.Manifests {
if !read.Accepted {
out.Verdict = delivery.ComposeCandidateRefused
out.Modules = append(out.Modules, delivery.ModuleComposed{Module: name, State: delivery.ModuleRefusedState, Words: read.Refused})
}
}
}
sort.Slice(out.Modules, func(i, j int) bool { return out.Modules[i].Module < out.Modules[j].Module })
return out, nil
}
modules/mesh-delivery-checker/internal/check/engines_test.go
+194
View File
@@ -0,0 +1,194 @@
package check
import (
"context"
"encoding/json"
"errors"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/composer"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// fakeEngines answers compose-candidate by node from memory; a node it holds no answer for never answers.
type fakeEngines struct {
mu sync.Mutex
answers map[string]delivery.ComposeCandidateAnswer
errs map[string]error
asked map[string]delivery.ComposeCandidateAsk
}
func (f *fakeEngines) ComposeCandidate(ctx context.Context, node string, ask delivery.ComposeCandidateAsk) (delivery.ComposeCandidateAnswer, error) {
f.mu.Lock()
if f.asked == nil {
f.asked = map[string]delivery.ComposeCandidateAsk{}
}
f.asked[node] = ask
a, ok := f.answers[node]
err := f.errs[node]
f.mu.Unlock()
if err != nil {
return delivery.ComposeCandidateAnswer{}, err
}
if !ok {
<-ctx.Done()
return delivery.ComposeCandidateAnswer{}, ctx.Err()
}
return a, nil
}
// fakeSandbox answers a compose job with a verdict, recording the job.
type fakeSandbox struct {
mu sync.Mutex
jobs []delivery.ComposeJob
v func(node string) composer.Verdict
err error
}
func (f *fakeSandbox) Compose(_ context.Context, job delivery.ComposeJob) (composer.Verdict, error) {
var req composer.Request
if err := json.Unmarshal(job.Request, &req); err != nil {
return composer.Verdict{}, err
}
f.mu.Lock()
f.jobs = append(f.jobs, job)
f.mu.Unlock()
if f.err != nil {
return composer.Verdict{}, f.err
}
return f.v(req.Machines[0]), nil
}
type digests map[string]string
func (d digests) ManifestDigests() map[string]string { return d }
// ADR 0593 §7, §9 (D12, G1, G2): the checker asks every placed node's engine in parallel and makes the composition from
// their answers: a refusal fails it naming the node and each module; a node that does not answer within the bound,
// answers unready or cannot be asked is named "not composed on that node: no answer" and the check passes with that
// warning, decided by the verdict and never by its words; a group with a node-engine member is composed by the composer
// built at that member's head, over the input each node's engine composed from.
func TestTheCompositionIsMadeFromTheEnginesAnswers(t *testing.T) {
composed := func(node string) delivery.ComposeCandidateAnswer {
return delivery.ComposeCandidateAnswer{Node: node, Verdict: delivery.ComposeCandidateComposed,
Modules: []delivery.ModuleComposed{{Module: "web", State: delivery.ModuleComposedState, Digest: "sha256:" + strings.Repeat("a", 64)}},
Own: delivery.ModuleComposed{Module: "node", State: delivery.ModuleComposedState, Digest: "sha256:" + strings.Repeat("b", 64)}}
}
cs := caseOf(t, pullOf(repoA, 7, head1))
key, _ := delivery.CompositionKey(cs.Group, cs.HeadsHash)
compose := func(e *fakeEngines, nodes ...string) delivery.CompositionVerdict {
t.Helper()
ec := EngineCompositions{Asker: e, Placed: func() ([]string, error) { return nodes, nil }, Bound: 50 * time.Millisecond,
Composer: func() string { return strings.Repeat("7", 40) }}
v, err := ec.Compose(context.Background(), cs)
if err != nil {
t.Fatal(err)
}
return v
}
t.Run("one node refusing fails the composition, naming the node and the module", func(t *testing.T) {
refusing := composed("laptop")
refusing.Verdict = delivery.ComposeCandidateRefused
refusing.Modules = []delivery.ModuleComposed{{Module: "web", State: delivery.ModuleRefusedState, Words: "a required value is unset"},
{Module: "mail", State: delivery.ModuleHeldState, Words: "held as today"}}
e := &fakeEngines{answers: map[string]delivery.ComposeCandidateAnswer{"anchor": composed("anchor"), "laptop": refusing}}
v := compose(e, "anchor", "laptop")
if v.State != delivery.CompositionFail || v.Machine != "laptop" || !strings.Contains(v.Words, "not composed on laptop: web: a required value is unset") ||
strings.Contains(v.Words, "mail") || len(v.Machines) != 2 {
t.Errorf("verdict = %+v", v)
}
if e.asked["anchor"].Candidate != key || e.asked["laptop"].Input {
t.Errorf("asked %+v, want the check's candidate %s and no input", e.asked, key)
}
var ver delivery.Verdict
withComposition(&ver, v)
if ver.State != delivery.VerdictFail || ver.Failed.Machine != "laptop" {
t.Errorf("the check's verdict = %+v", ver)
}
})
t.Run("a silent, unready or unreachable node is no answer, and the check passes naming it", func(t *testing.T) {
unready := delivery.ComposeCandidateAnswer{Node: "g14", Verdict: delivery.ComposeCandidateUnready, Refused: "a module refused it"}
e := &fakeEngines{answers: map[string]delivery.ComposeCandidateAnswer{"anchor": composed("anchor"), "g14": unready},
errs: map[string]error{"ace": errors.New("no responders")}}
began := time.Now()
v := compose(e, "anchor", "laptop", "g14", "ace")
if took := time.Since(began); took > 2*time.Second {
t.Errorf("the nodes were not asked in parallel within the bound: %s", took)
}
if v.State != delivery.CompositionPass || v.SDK != strings.Repeat("7", 40) || len(v.Machines) != 1 {
t.Fatalf("verdict = %+v", v)
}
for _, node := range []string{"ace", "g14", "laptop"} {
if !strings.Contains(strings.Join(v.Warnings, "; "), node+": "+delivery.ComposeNoAnswer) {
t.Errorf("%s is not named: %v", node, v.Warnings)
}
}
var ver delivery.Verdict
ver.State = delivery.VerdictPass
withComposition(&ver, v)
if ver.State != delivery.VerdictPass || !strings.Contains(composition(ver.Composition, 0, false), "laptop: "+delivery.ComposeNoAnswer) {
t.Errorf("the line = %q", composition(ver.Composition, 0, false))
}
})
t.Run("a node-engine member is composed by the composer at its head over each node's input", func(t *testing.T) {
c, err := delivery.DecodeCandidate(candidateOf(t, "feat/x", pullOf(nodeEngineRepo, 3, head2), pullOf(repoA, 7, head1)))
if err != nil {
t.Fatal(err)
}
host := CaseOf(c, map[string]string{repoA: trunkA, nodeEngineRepo: trunkA})
withInput := composed("anchor")
withInput.Input = json.RawMessage(`{"nodes":[{"name":"anchor"}]}`)
e := &fakeEngines{answers: map[string]delivery.ComposeCandidateAnswer{"anchor": withInput}}
sb := &fakeSandbox{v: func(node string) composer.Verdict {
return composer.Verdict{V: composer.Version, Cases: []composer.CaseVerdict{{Name: "change",
Machines: map[string]composer.Machine{node: {Composes: false, Reason: "the new composer refuses it"}}}}}
}}
ec := EngineCompositions{Asker: e, Placed: func() ([]string, error) { return []string{"anchor"}, nil }, Bound: time.Second,
Sandbox: sb, Manifests: digests{"web": "sha256:" + strings.Repeat("c", 64)}}
v, err := ec.Compose(context.Background(), host)
if err != nil {
t.Fatal(err)
}
if !e.asked["anchor"].Input || len(sb.jobs) != 1 {
t.Fatalf("asked %+v, jobs %+v: want the input asked and one job", e.asked, sb.jobs)
}
job := sb.jobs[0]
var req composer.Request
_ = json.Unmarshal(job.Request, &req)
if job.Fetch["repository"] != nodeEngineRepo || job.Fetch["commit"] != head2 || job.Package != composer.Package ||
string(req.Input) != string(withInput.Input) || req.Cases[0].Manifests["web"].Digest == "" {
t.Errorf("job = %+v, request %+v", job, req)
}
if v.State != delivery.CompositionFail || v.SDK != head2 || !strings.Contains(v.Words, "the new composer refuses it") {
t.Errorf("verdict = %+v", v)
}
// Review finding 3 (ADR 0593 §9 lets only a node that does not answer pass): a composer at the head that cannot
// run fails the composition, naming the head, never passing as a silent node.
sb.err = errors.New("the build of mesh-compose failed")
v, err = ec.Compose(context.Background(), host)
if err != nil || v.State != delivery.CompositionFail || !strings.Contains(v.Words, head2) ||
!strings.Contains(v.Words, "the build of mesh-compose failed") || len(v.Warnings) != 0 {
t.Errorf("a composer that cannot run: %+v, %v", v, err)
}
})
// Review finding 4: a placement that cannot be read is no mesh with no node: the composition is not made, said.
t.Run("an unreadable placement makes no composition", func(t *testing.T) {
ec := EngineCompositions{Asker: &fakeEngines{}, Placed: func() ([]string, error) { return nil, errors.New("no such bucket") }}
if _, err := ec.Compose(context.Background(), cs); err == nil || !strings.Contains(err.Error(), "no such bucket") {
t.Errorf("an unreadable placement: %v", err)
}
b := newBench(t)
b.ex.Compositions = ec
v := b.judge(t, cs).Verdict
if v.Composition == nil || v.Composition.State != delivery.CompositionNotRun || !strings.Contains(v.Composition.Words, "no such bucket") {
t.Errorf("the verdict's composition = %+v", v.Composition)
}
})
}
modules/mesh-delivery-checker/internal/check/every_failure_test.go
+13 -9
View File
@@ -23,7 +23,7 @@ func judgeLevel(t *testing.T, tests ...json.RawMessage) (*delivery.Verdict, *sta
t.Helper()
st := &state{j: &Examiner{}, c: &Case{}, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{}}
v, _ := st.level(t.Context(), &target{pull: Pull{Repository: "novox/x"}}, delivery.LevelAnswer{Level: delivery.LevelIntegration, Tests: tests})
return v, st
return settled(joinFailures(v)), st
}
// Issue 1921: two failing tests in two packages are two entries of the verdict's Failures and two lines of a try's
@@ -58,7 +58,7 @@ func TestTwoFailingTestsInTwoPackagesAreTwoEntries(t *testing.T) {
}
// Issue 1921: past delivery.MaxFailures the list names the first fifty and how many more, the status line says how many
// failed, and each entry's line is cut to MaxFailureLine.
// failed and names the first, cut so "and N more" stays whole, and each entry's line is cut to MaxFailureLine.
func TestFailuresAreCappedWithHowManyMore(t *testing.T) {
var fs []Failure
for i := range 60 {
@@ -75,16 +75,20 @@ func TestFailuresAreCappedWithHowManyMore(t *testing.T) {
t.Errorf("failedCount = %d, want 60", n)
}
cs := &Case{Pulls: []Pull{{Repository: "novox/x", Head: "0123456789abcdef", Trunk: "fedcba9876543210"}}}
if line := Line(*v, cs, cs.Pulls[0], nil, nil, nil); !strings.HasPrefix(line, "failed (60 tests): integration: x/p0.TestT") {
if line := Line(*v, cs, cs.Pulls[0], nil, nil, nil); !strings.HasPrefix(line, "failed (60): integration: x/p0.TestT: yyy") ||
!strings.Contains(line, "…; and 59 more | 0123456789ab on fedcba987654") {
t.Errorf("line %q", line)
}
}
// A composition failing after them is counted in the last entry (the operator, 2026-11-02).
withComposition(v, delivery.CompositionVerdict{State: delivery.CompositionFail, Machine: "g14", Words: "route-proxy is defined twice"})
if n := failedCount(*v); n != 61 || v.Failed.Test != "integration: x/p0.TestT" {
t.Errorf("after the composition failedCount = %d, failed %+v", n, v.Failed)
}
// A verdict that names no list (a conflict, a refused step, a verdict written before issue 1921) is one failed line, as
// before.
func TestAVerdictWithoutFailuresIsOneLine(t *testing.T) {
v := delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "g14", Words: "files in conflict: go.mod\nmore"}}
if got := failedLines(v); len(got) != 1 || got[0] != "g14: files in conflict: go.mod" {
// A verdict that names no list (a conflict, a refused step, a verdict written before issue 1921) is one failed line, as
// before.
one := delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "g14", Words: "files in conflict: go.mod\nmore"}}
if got := failedLines(one); len(got) != 1 || got[0] != "g14: files in conflict: go.mod" {
t.Fatalf("%q", got)
}
}
modules/mesh-delivery-checker/internal/check/examiner_test.go
+90 -32
View File
@@ -10,16 +10,18 @@ import (
"os"
"path/filepath"
"reflect"
"slices"
"sort"
"strings"
"sync"
"testing"
"time"
"unicode/utf8"
"git.novox.be/novox/mesh-sdk/go/buildstep"
gobuild "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-sdk/go/composer"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// captured reads a result the build module's own code marshalled (testdata/captured/README).
@@ -134,12 +136,12 @@ func refsOf(v any) []BesideRef {
// toolPrefix is the start of each build module's tool names, as the fake build modules answer them.
var toolPrefix = map[string]string{buildrun.GoBuildModule: "build_go_", buildrun.NodeBuildModule: "build_node_"}
// answer runs a check ask as the build module that pulls it does (buildstep.RunAsk), with the fake tools on shanks and the
// answer runs a check ask as the build module that pulls it does (gobuild.RunAsk), with the fake tools on shanks and the
// bench's kept verdicts, and answers the step it writes.
func (b *bench) answer(a delivery.StepAsk) delivery.Step {
module, _ := delivery.BuildModuleOf(a.Language)
const node = "shanks"
tool := func(name string) buildstep.StepFunc {
tool := func(name string) gobuild.StepFunc {
return func(ctx context.Context, args map[string]any) (any, error) {
raw, err := b.steps.Run(ctx, module, node, toolPrefix[module]+name, args)
if err != nil {
@@ -148,7 +150,7 @@ func (b *bench) answer(a delivery.StepAsk) delivery.Step {
return raw, nil
}
}
steps := buildstep.AskSteps{Fetch: tool("fetch"), Test: tool("test"),
steps := gobuild.AskSteps{Fetch: tool("fetch"), Test: tool("test"),
Kept: func(ctx context.Context, level, digest string) (int, bool, error) {
k, found, err := (Kept{Store: b.kept}).Lookup(ctx, level, digest)
return k.Tests, found, err
@@ -166,7 +168,7 @@ func (b *bench) answer(a delivery.StepAsk) delivery.Step {
}
steps.Has = func(string) bool { return true }
}
out, err := buildstep.RunAsk(context.Background(), steps, a)
out, err := gobuild.RunAsk(context.Background(), steps, a)
step := delivery.Step{ID: a.ID, Tool: a.Kind + " ask", Node: node, State: delivery.StepPassed, Started: began, Ended: began.Add(2 * time.Second)}
if err != nil {
step.State, step.Reason = delivery.StepFailed, err.Error()
@@ -215,7 +217,8 @@ func caseOf(t testing.TB, pulls ...delivery.CandidatePull) *Case {
if err != nil {
t.Fatal(err)
}
cs := CaseOf(c, map[string]string{repoA: trunkA, repoSDK: trunkS, "novox/mesh-controller": trunkA, "novox/mesh-catalog": trunkA})
cs := CaseOf(c, map[string]string{repoA: trunkA, repoSDK: trunkS, "novox/mesh-controller": trunkA, "novox/mesh-catalog": trunkA,
"novox/mesh-host": trunkA, "novox/mesh-networking": trunkA})
if cs == nil {
t.Fatal("no case")
}
@@ -431,32 +434,87 @@ func TestANodePackageIsNeverKeptAndRunsEveryCheck(t *testing.T) { // 1288/AC4, 1
}
}
// 1289/AC1, AC2 (R4c, mesh-sdk buildrun at 0b12b62): the change's own tests run first, and when one fails the level ends
// there: no other package of the level and no later level runs, so the rest of the suite is not paid for.
func TestTheChangesOwnFailingTestEndsTheLevelThere(t *testing.T) {
b := newBench(t)
b.fetch.Changed = []string{"modules/m/internal/b/b_test.go"}
b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"m/internal/a": digA, "m/internal/b": digB}}
failing := captured[Tested](t, "go-test-failing")
b.test = func(c stepCall) (any, error) {
if got := packagesOf(c); len(got) == 1 && got[0] == "m/internal/b" {
return failing, nil
// The operator, 2026-11-02 ("A failed check shows only its first fault"): a check runs every member, every level and the
// composition to its end and its verdict names every failure in the order judged, Failed keeping the first whole; the
// change's own tests still run first (issue 1289), and the rest of the level after them. A member that only failed its
// tests still gets its preview, so the composition is read and its failure named after the tests'. A member whose head
// does not merge has no preview of a merge, so the assigner's composition would only wait: the check does not wait for it
// and says why it was not run (review of 650a1d0).
func TestEveryFailureOfEveryMemberLevelAndTheCompositionIsNamed(t *testing.T) {
for _, conflicts := range []bool{false, true} {
b := newBench(t)
comps := &fakeCompositions{v: delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionFail, SDK: "x",
Machine: "g14", Words: "route-proxy is defined twice"}, waits: conflicts}
b.ex.Compositions = comps
b.fetch.Changed = []string{"modules/m/internal/b/b_test.go"}
b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"m/internal/a": digA, "m/internal/b": digB}}
conflict := captured[Fetched](t, "go-fetch-conflict")
b.steps.handlers["build_go_fetch"] = func(c stepCall) (any, error) {
if conflicts && c.args["repository"] == repoSDK {
return conflict, nil
}
return b.fetch, nil
}
failing := captured[Tested](t, "go-test-failing")
b.test = func(c stepCall) (any, error) {
if got := packagesOf(c); c.args["level"] == "integration" || (len(got) == 1 && got[0] == "m/internal/b") {
r := failing
r.Level = c.args["level"].(string)
return r, nil
}
return captured[Tested](t, "go-test-slow"), nil
}
cs := caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))
done := make(chan delivery.Verdict, 1)
go func() { done <- b.judge(t, cs).Verdict }()
var v delivery.Verdict
select {
case v = <-done:
case <-time.After(5 * time.Second):
t.Fatalf("conflicts %v: the check waits on a composition that only waits", conflicts)
}
var got []string
for _, f := range v.Failures {
got = append(got, firstFailed(delivery.Verdict{Failed: f}))
}
tests := []string{"unit: fixture/failing/bad.TestThreeIsFour", "integration: fixture/failing/bad.TestThreeIsFour"}
want := append(append(slices.Clone(tests), tests...), "g14: route-proxy is defined twice") // both members fail their tests
if conflicts {
want = append([]string{"shanks: novox/mesh-sdk#12 does not merge into the trunk: files in conflict: a.go"}, tests...)
}
if v.State != delivery.VerdictFail || len(got) != len(want) {
t.Fatalf("conflicts %v: verdict %+v: failures\n%s\nwant\n%s", conflicts, v, strings.Join(got, "\n"), strings.Join(want, "\n"))
}
for i := range want {
if !strings.HasPrefix(got[i], want[i]) {
t.Errorf("conflicts %v: failure %d is %q, want %q", conflicts, i, got[i], want[i])
}
}
keys, _ := b.prev.Keys(context.Background())
switch {
case !conflicts && (len(keys) != 2 || v.Composition == nil || v.Composition.State != delivery.CompositionFail):
t.Errorf("tests alone failed: previews %v, composition %+v, want both previews and the composition read", keys, v.Composition)
case conflicts && (len(comps.asked) != 0 || v.Composition == nil || v.Composition.State != delivery.CompositionNotRun ||
!strings.Contains(v.Composition.Words, "novox/mesh-sdk#12")):
t.Errorf("a member did not merge: asked %v, composition %+v, want not run naming novox/mesh-sdk#12", comps.asked, v.Composition)
}
var unit []string
for _, c := range b.steps.tool("test") {
if c.args["level"] == "unit" && c.args["workspace"] != nil {
unit = append(unit, strings.Join(packagesOf(c), ","))
}
}
if conflicts && (strings.Join(unit, " | ") != "m/internal/b | m/internal/a" || len(b.steps.tool("digest")) != 2) {
t.Errorf("the unit level ran %v and %d digests, want the changed test's package, then the rest, at both levels", unit, len(b.steps.tool("digest")))
}
v.Group, v.Heads = "feat/x", "0123456789ab"
if _, err := delivery.EncodeVerdict(v); err != nil {
t.Errorf("conflicts %v: the verdict cannot be written: %v", conflicts, err)
}
if line := Line(v, cs, cs.Pulls[0], nil, nil, nil); !strings.HasPrefix(line, fmt.Sprintf("failed (%d): %s", len(want), want[0][:20])) ||
!strings.Contains(line, fmt.Sprintf("; and %d more | ", len(want)-1)) || utf8.RuneCountInString(line) > MaxLine {
t.Errorf("conflicts %v: line %q", conflicts, line)
}
return captured[Tested](t, "go-test-slow"), nil
}
out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
if out.Verdict.State != delivery.VerdictFail || !strings.Contains(out.Verdict.Failed.Test, "TestThreeIsFour") {
t.Fatalf("verdict = %+v", out.Verdict)
}
tests := b.steps.tool("test")
if len(tests) != 1 || strings.Join(packagesOf(tests[0]), ",") != "m/internal/b" {
t.Fatalf("test calls = %+v, want m/internal/b alone: its failure ends the level", tests)
}
if n := len(b.steps.tool("digest")); n != 1 {
t.Errorf("%d digest calls, want the unit level's alone (the check stopped)", n)
}
if out.Verdict.Composition == nil || out.Verdict.Composition.State != delivery.CompositionNotRun {
t.Errorf("composition = %+v", out.Verdict.Composition)
}
}
modules/mesh-delivery-checker/internal/check/judge.go
+110 -44
View File
@@ -3,7 +3,6 @@ package check
import (
"context"
"encoding/json"
"errors"
"fmt"
"path"
"slices"
@@ -11,9 +10,9 @@ import (
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// goModule is the build module the Go repositories are judged by.
@@ -76,8 +75,9 @@ var askLanguage = map[string]string{buildrun.GoBuildModule: delivery.AskGo, buil
// Examiner judges a case through the build seats (ADR 0484 clauses 3, 4, 10, 13; ADR 0525 §1, §3): for each pull request
// and build module it puts one check ask whose job fetches the head merged onto its trunk head, then runs the unit level
// and the integration level, taking the kept verdicts of unchanged packages, running the change's own tests first and
// stopping at the first failure; it judges the step's answer as it judged the tools' answers.
// and the integration level, taking the kept verdicts of unchanged packages and running the change's own tests first;
// every level and member runs to its end also after a failure, so the verdict names every failure (the operator,
// 2026-11-02). It judges the step's answer as it judged the tools' answers.
type Examiner struct {
Asks Asks
// DeclineBound is how long a placed ask may go on being declined, from its first decline, before the check fails
@@ -116,9 +116,8 @@ type Examiner struct {
Manifests TrunkManifests
// Predefined reads the catalogue's predefined keys the indexer keeps (manifests_judge.go); nil reads none.
Predefined PredefinedList
// Compositions answers the assigner's composition check of a check by the check's key (delivery.CompositionKey),
// waiting until the assigner wrote a verdict that does not wait, or ctx ends (ADR 0528 clause 5, issue 1297); nil
// leaves the composition not run.
// Compositions makes a check's composition (ADR 0593 §7): the engines' answers (EngineCompositions); nil leaves the
// composition not run.
Compositions CompositionReader
// Nodes names every node with a record, read from each `<node>.node` of the node-engine's have (novox/hq ADR 0567
// §12): every check ask carries them (CheckJob.NodeNames) for the names check, so no repository lists them. A check
@@ -130,10 +129,9 @@ type Examiner struct {
Logf func(format string, args ...any)
}
// CompositionReader answers the assigner's composition of a check by the check's key, waiting until the assigner wrote
// a verdict that does not wait, or ctx ends.
// CompositionReader makes a check's composition verdict; an error is only a case that cannot be composed at all.
type CompositionReader interface {
Await(ctx context.Context, key string) (delivery.CompositionVerdict, error)
Compose(ctx context.Context, c *Case) (delivery.CompositionVerdict, error)
}
func (j *Examiner) now() time.Time {
@@ -231,6 +229,9 @@ type state struct {
judgeHead string
// predefinedHead is the catalogue member's head when its merge changes predefined.json: the list is read there.
predefinedHead string
// unpreviewed are the members (pullKey) with no preview of a merge: a head that does not merge, or a fetch that did not
// answer. The indexer holds no manifest of them, so the assigner's composition would only wait: it is not awaited.
unpreviewed []string
}
// Judge runs the case (it is the Judge of a Checker). A failure of the change, of a merge or of a step is a verdict; an error is only a case that cannot
@@ -238,30 +239,27 @@ type state struct {
func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) {
st := &state{j: j, c: c, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{},
beside: map[string]string{}}
if v := st.judgeAll(ctx); v != nil {
return st.finish(ctx, *v), nil
}
// Every manifest the change touches, judged by the indexer's judge (ADR 0578 §2): a refusal fails the check.
if v := st.judgeManifests(ctx); v != nil {
// Every member and level runs to its end, and every manifest the change touches is judged by the indexer's judge (ADR
// 0578 §2), also after a failure: the verdict names every failure, not only the first (the operator, 2026-11-02).
tests := st.judgeAll(ctx)
if v := settled(joinFailures(tests, st.judgeManifests(ctx))); v != nil {
return st.finish(ctx, *v), nil
}
return st.finish(ctx, delivery.Verdict{State: delivery.VerdictPass}), nil
}
// finish is the outcome of the verdict with the state of the builds the candidate's modules carry, and, for a check
// whose tests passed, the assigner's composition of it.
// finish is the outcome of the verdict with the state of the builds the candidate's modules carry and the assigner's
// composition of it, read also when the tests failed, so a failed composition is named beside them.
func (st *state) finish(ctx context.Context, v delivery.Verdict) Outcome {
out := st.outcome(v)
if out.Verdict.State == delivery.VerdictPass {
st.composed(ctx, &out.Verdict)
}
st.composed(ctx, &out.Verdict)
out.Builds = st.j.buildNotes(ctx, st.c)
return out
}
// composed reads the assigner's composition of the check into its verdict (ADR 0528 clause 5): a failed composition
// fails the check, naming the machine it failed at. A try is not composed (it has no candidate); a composition that
// cannot be had leaves it not run, said; with no assigner running it is not judged, said at once.
// composed makes the check's composition from the engines' answers into its verdict (ADR 0593 §7, §9): a failed
// composition fails the check, naming the machine it failed at; a node that did not answer is a warning. A try is not
// composed (it has no candidate); a composition that cannot be made leaves it not run, said.
func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
if st.c.Composition != nil && st.c.Try == nil {
// A carry whose candidate's fingerprints did not change keeps the composition it passed (ADR 0548 §5).
@@ -273,13 +271,17 @@ func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
if st.j.Compositions == nil || st.c.Try != nil {
return
}
comp, err := st.j.Compositions.Await(ctx, st.c.Key)
switch {
case errors.Is(err, ErrAssignerNotRunning):
v.Composition = &delivery.Composition{State: delivery.CompositionNotRun, Words: delivery.AssignerNotRunning}
if len(st.unpreviewed) > 0 {
// An engine would only say it cannot read a manifest the indexer will not have: the check does not ask it.
v.Composition = &delivery.Composition{State: delivery.CompositionNotRun, Words: "no preview of a merge of " +
strings.Join(st.unpreviewed, ", ") + " (its head does not merge or its fetch did not answer), so the change cannot be composed"}
return
case err != nil:
st.j.logf("%s: the composition could not be read, so it is said as not run: %v", st.c.Key, err)
}
comp, err := st.j.Compositions.Compose(ctx, st.c)
if err != nil {
// Said in the verdict, never left as a composition not yet run (review finding 4).
st.j.logf("%s: the composition could not be made, so it is said as not run: %v", st.c.Key, err)
v.Composition = &delivery.Composition{State: delivery.CompositionNotRun, Words: "not made: " + err.Error()}
return
}
withComposition(v, comp)
@@ -308,7 +310,7 @@ func (st *state) outcome(v delivery.Verdict) Outcome {
v.OverBound = append(v.OverBound, level)
}
}
// The composition is the assigner's check (issue 1297): until it delivers a verdict says it was not run (ADR 0484 clause 6).
// The composition is made from the engines' answers (ADR 0593 §7): until it is, a verdict says it was not run (ADR 0484 clause 6).
v.Composition = &delivery.Composition{State: delivery.CompositionNotRun}
var slow []string
for _, level := range []string{delivery.LevelUnit, delivery.LevelIntegration} {
@@ -354,8 +356,9 @@ func (st *state) levelsOf() []string {
}
// judgeAll puts one check ask per pull request and build module the repository needs, the mesh-sdk member first, and
// judges each answer. A head that does not merge, a pin that is not allowed or a level that fails ends the check at once
// (ADR 0484 clause 13).
// judges each answer. A head that does not merge, a pin that is not allowed or a level that fails is a failure of that
// member; every other member is still judged, and the verdict names every failure (the operator, 2026-11-02), unjoined
// to its first lines (settled does that). Only a check whose context ended stops.
func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
// The mesh-sdk member first: whether its merge changes the TypeScript package's paths decides whether the other members'
// Node fetches take the group's package (ADR 0500 §5).
@@ -370,9 +373,17 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
return 0
})
var groupPackage map[string]any
var failures []*delivery.Verdict
for _, p := range pulls {
if ctx.Err() != nil {
if len(failures) == 0 { // an ask that ended with the context said it already
failures = append(failures, fail(delivery.CheckerModule, ctx.Err().Error()))
}
break
}
var merged []Fetched
for _, bm := range st.j.Modules.BuildModules(p.Repository) {
modules := st.j.Modules.BuildModules(p.Repository)
for _, bm := range modules {
t := &target{pull: p, module: bm}
if tc, none := delivery.AgreedToolchain(st.j.Toolchains.Entries(buildrun.LanguageOf[bm])); none == "" {
t.toolchain = tc.Version
@@ -385,7 +396,8 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
if bm == buildrun.GoBuildModule {
beside, err := st.j.besideFor(ctx, st.c, p)
if err != nil {
return fail(forgeMachine(err, bm), err.Error()+unheardWords(err))
failures = append(failures, fail(forgeMachine(err, bm), err.Error()+unheardWords(err)))
continue
}
if len(beside) > 0 {
args["beside"] = beside
@@ -399,18 +411,24 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
t.fetchArgs = args
st.targets = append(st.targets, t)
if v := st.ask(ctx, t, &merged); v != nil {
return v
failures = append(failures, v)
}
}
if st.c.Try == nil { // a try owes the indexer no preview: no pull request holds its head
// Every build module's fetch merged (each is in merged once): the preview is written, also when the member's tests
// failed, so the indexer holds its manifests and the assigner can compose the change (review of 650a1d0).
switch {
case st.c.Try != nil: // a try owes the indexer no preview: no pull request holds its head
case len(merged) == len(modules):
st.j.writePreview(ctx, p, merged)
default:
st.unpreviewed = append(st.unpreviewed, pullKey(p))
}
st.judgedByMember(p, merged)
if p.Repository == SDKRepository && len(st.c.Pulls) > 1 && st.j.changesPackage(p, merged) {
groupPackage = map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number)}
}
}
return nil
return joinFailures(failures...)
}
// DefaultDeclineBound is an Examiner's DeclineBound when it names none: longer than a check runs, so a node with the
@@ -508,6 +526,7 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
st.j.logf("%s#%d: the ask %s was declined on %s, which lacks %s; asked again", p.Repository, p.Number, a.ID, step.Node, strings.Join(d.Lacks, ", "))
a.ID, a.At = buildstep.NewID(), now.UTC()
}
var failures []*delivery.Verdict
// The fetch and the digests are what of the ask's time its levels' running did not take (Took.Clone).
took := time.Duration(0)
if !step.Started.IsZero() && step.Ended.After(step.Started) {
@@ -539,7 +558,7 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
if !placed { // the placed ask fetched the same merge: its preview was read from the first
*merged = append(*merged, t.fetched)
if v := st.checkBinariesOf(t); v != nil {
return v
failures = append(failures, v) // the levels still run: the verdict names every failure
}
}
ran := 0.0
@@ -555,13 +574,12 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
v, elapsed := st.level(ctx, t, la)
ran += elapsed
if v != nil {
st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
return v
failures = append(failures, v) // the next level was run too (buildstep): its failures are named as well
}
}
st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
if unmet := delivery.UnmetNeeds(answer.Levels); len(unmet) > 0 && !placed {
// Every level passed, its passes kept, and some packages were left to a node with what their tests need: the levels
// Every level ran, its passes kept, and some packages were left to a node with what their tests need: the levels
// from the first that left one are asked of such a node, which takes the kept passes and tests those (issue 890).
var again, packages []string
seen := map[string]bool{}
@@ -577,9 +595,9 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
}
}
st.j.logf("%s#%d: %s on %s left untested; asked of a node with %s", p.Repository, p.Number, unmetOf(answer.Levels), step.Node, strings.Join(unmet, ", "))
return st.askNeeding(ctx, t, merged, again, unmet, packages)
failures = append(failures, st.askNeeding(ctx, t, merged, again, unmet, packages))
}
return nil
return joinFailures(failures...)
}
// unmetNames names the unmet packages and their needs.
@@ -865,10 +883,58 @@ func levelFailure(t *target, level string, results ...Tested) *delivery.Verdict
all = append(all, delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: "the " + level + " level did not pass"})
}
v.Failed = all[0]
v.Failures = failureLines(all)
v.Failures = all // whole: settled cuts them to their first lines once every failure of the check is joined
return &v
}
// joinFailures is one failed verdict of every failure of vs, in the order given (a nil one passed), unsettled: Failed is
// the first whole, and Failures every failure whole, or none when the one failure is no level's (a conflict, a refused
// step: one line, as before issue 1921). Nil when none failed.
func joinFailures(vs ...*delivery.Verdict) *delivery.Verdict {
var out *delivery.Verdict
var all []delivery.FailedFirst
listed := false
for _, v := range vs {
if v == nil {
continue
}
if out == nil {
out = &delivery.Verdict{State: delivery.VerdictFail, Failed: v.Failed}
}
if len(v.Failures) > 0 {
all, listed = append(all, v.Failures...), true
continue
}
all = append(all, v.Failed)
}
if out != nil && (listed || len(all) > 1) {
out.Failures = all
}
return out
}
// settled is a joined verdict as it is written: each failure of its list cut to its first line, at most
// delivery.MaxFailures and then how many more (failureLines).
func settled(v *delivery.Verdict) *delivery.Verdict {
if v != nil && len(v.Failures) > 0 {
v.Failures = failureLines(v.Failures)
}
return v
}
// addFailure is a settled list of failures with one more at its end: the first failure is put first when the list is
// empty (a verdict of one failure keeps none), and past delivery.MaxFailures the last entry counts one more.
func addFailure(v *delivery.Verdict, f delivery.FailedFirst) {
if len(v.Failures) == 0 {
v.Failures = failureLines([]delivery.FailedFirst{v.Failed})
}
if n := len(v.Failures); n > delivery.MaxFailures {
v.Failures[n-1].Test = fmt.Sprintf("and %d more", failedCount(*v)-delivery.MaxFailures+1)
return
}
v.Failures = append(v.Failures, failureLines([]delivery.FailedFirst{f})...)
}
// failureLines is every failure with the first line of its words, cut to MaxFailureLine, at most delivery.MaxFailures
// and then one entry saying how many more failed.
func failureLines(all []delivery.FailedFirst) []delivery.FailedFirst {
modules/mesh-delivery-checker/internal/check/line.go
+49 -9
View File
@@ -65,7 +65,35 @@ func failedLines(v delivery.Verdict) []string {
return lines
}
// failedCount is how many tests a verdict names as failed, the ones after delivery.MaxFailures counted too.
// failedHead is the head of a failed verdict's line: what failed first, and when more did, how many failed and "and N
// more" after it (the operator, 2026-11-02), so a line cut to MaxLine still says that more failed. The words of the first
// are cut to fit room characters when room is not negative; the count and "and N more" are never cut.
func failedHead(v delivery.Verdict, room int) string {
first := firstFailed(v)
n := failedCount(v)
if n <= 1 {
return clipRunes("failed: "+first, room)
}
prefix, more := fmt.Sprintf("failed (%d): ", n), fmt.Sprintf("; and %d more", n-1)
if room >= 0 {
first = clipRunes(first, room-utf8.RuneCountInString(prefix+more))
}
return prefix + first + more
}
// clipRunes cuts s to at most n characters, the last an ellipsis; a negative n leaves it whole.
func clipRunes(s string, n int) string {
r := []rune(s)
if n < 0 || len(r) <= n {
return s
}
if n < 1 {
return ""
}
return string(r[:n-1]) + "…"
}
// failedCount is how many failures a verdict names, the ones after delivery.MaxFailures counted too.
func failedCount(v delivery.Verdict) int {
n := len(v.Failures)
if n > delivery.MaxFailures {
@@ -128,8 +156,8 @@ func levelWords(name string, r delivery.LevelResult) string {
return s
}
// compositionWords says the composition: not yet run, or once the assigner's check reports it (issue 1297) passed or
// failed, with the time it took and the machines it composed (issue 1300).
// compositionWords says the composition: not yet run, or once the engines answered (ADR 0593 §7) passed, with the nodes
// that gave no answer, or failed, with the time it took and the machines it composed (issue 1300).
func compositionWords(c *delivery.Composition, took int64) string { return composition(c, took, false) }
// composition is compositionWords; short counts the machines and leaves the SDK commit to the report.
@@ -155,11 +183,15 @@ func composition(c *delivery.Composition, took int64, short bool) string {
on = fmt.Sprintf(" on %d machines: %s", len(machines), strings.Join(machines, ", "))
}
}
warned := ""
if c.State == delivery.CompositionPass && c.Words != "" {
warned = " (" + c.Words + ")" // a node that did not answer (ADR 0593 §9)
}
switch {
case c.State == delivery.CompositionPass && short:
return "composition passed" + in + on // the SDK it composed with is in the report
return "composition passed" + in + on + warned // what it composed with is in the report
case c.State == delivery.CompositionPass:
return "composition passed" + in + " with " + short12(c.SDK) + on
return "composition passed" + in + " with " + short12(c.SDK) + on + warned
}
return "composition failed" + in + on + ": " + c.Words
}
@@ -232,10 +264,7 @@ func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []s
pc.carried = "carried from " + w // the counts that follow say what the carry ran and kept
}
default:
pc.head = "failed: " + firstFailed(v)
if n := failedCount(v); n > 1 {
pc.head = fmt.Sprintf("failed (%d tests): %s", n, firstFailed(v))
}
pc.head = failedHead(v, -1)
pc.commits = commits
}
var counts []string
@@ -325,6 +354,11 @@ func Report(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []
// what follows, and is cut in its first part rather than lose it.
func Line(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string, judged map[string]string) string {
pc := parts(v, cs, p, builds, slowest, judged)
if v.State == delivery.VerdictFail {
// The head gives way first to the commits and what is never dropped: its count and "and N more" stay whole.
kept := joinSet(pc.commits, pc.carried, pc.warning, pc.judged)
pc.head = failedHead(v, MaxLine-utf8.RuneCountInString(" | "+kept))
}
for drop := 0; drop <= 4; drop++ {
if s := pc.join(drop); utf8.RuneCountInString(s) <= MaxLine {
return s
@@ -342,6 +376,12 @@ func Line(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []st
return string(head[:room]) + "…" + rest
}
}
if v.State == delivery.VerdictFail {
// A failure's head, sized above, and the commits fit: the counts and the composition are in the report.
if s := joinSet(pc.head, pc.commits, pc.carried, pc.warning, pc.judged); utf8.RuneCountInString(s) <= MaxLine {
return s
}
}
runes := []rune(pc.join(4))
return string(runes[:MaxLine-1]) + "…"
}
modules/mesh-delivery-checker/internal/check/line_test.go
+8
View File
@@ -1,6 +1,7 @@
package check
import (
"fmt"
"strings"
"testing"
"unicode/utf8"
@@ -43,6 +44,7 @@ func TestAPassOpensWithTheVerdictAndSaysTheCommitsTheCountsTheTimesAndThatTheCom
func TestAFailureNamesTheTestTheConflictOrTheRefusedStepFirst(t *testing.T) { // 1300/AC2
cs, p := lineCase(t, pullOf(repoA, 7, head1))
long := delivery.FailedFirst{Test: "integration: " + strings.Repeat("pkg/", 50) + "TestSomething", Words: strings.Repeat("because ", 40)}
for name, c := range map[string]struct {
v delivery.Verdict
want string
@@ -50,8 +52,14 @@ func TestAFailureNamesTheTestTheConflictOrTheRefusedStepFirst(t *testing.T) { //
"test": {delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Test: "unit: m/a.TestX", Words: "want 4"}}, "failed: unit: m/a.TestX: want 4"},
"conflict": {delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "shanks", Words: "novox/mesh-delivery#7 does not merge into the trunk: files in conflict: go.mod"}}, "failed: shanks: novox/mesh-delivery#7 does not merge into the trunk: files in conflict: go.mod"},
"refused": {delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "mesh-build-go", Words: "no go build node qualifies: builders disagree"}}, "failed: mesh-build-go: no go build node qualifies"},
// ADR 0591: a first failure longer than the forge's limit is shortened; "failed (N)" and "and N-1 more" are never cut.
"many, the first too long": {delivery.Verdict{State: delivery.VerdictFail, Failed: long, Failures: []delivery.FailedFirst{long, {Machine: "g14", Words: "route-proxy is defined twice"},
{Test: "unit: m/b.TestY", Words: "boom"}}}, "failed (3): integration: pkg/pkg/"},
} {
line := Line(c.v, cs, p, nil, nil, nil)
if more := len(c.v.Failures) - 1; more > 0 && (!strings.Contains(line, fmt.Sprintf("…; and %d more | 111111111111", more)) || utf8.RuneCountInString(line) > MaxLine) {
t.Errorf("%s: line = %q (%d), want the first's words shortened, the count and \"and %d more\" whole, within %d", name, line, utf8.RuneCountInString(line), more, MaxLine)
}
if !strings.HasPrefix(line, c.want) || !strings.Contains(line, "111111111111 on aaaaaaaaaaaa") {
t.Errorf("%s: line = %q, want it to open with %q and name the commits", name, line, c.want)
}
modules/mesh-delivery-checker/internal/check/lostask.go
+1 -1
View File
@@ -3,9 +3,9 @@ package check
import (
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// **An ask the build seat gave up on ends its step** (novox/hq issue 734): mesh-sdk buildrun.LostAskWaiter hands the
modules/mesh-delivery-checker/internal/check/lostask_test.go
+1 -1
View File
@@ -8,9 +8,9 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// losingAsks is the build seat over a bus that gives up the first `lose` asks put (DEAD_LETTERS keeps them, as on
modules/mesh-delivery-checker/internal/check/pins_test.go
+47 -52
View File
@@ -81,66 +81,54 @@ func besideOf(t *testing.T, b *bench, repository string) map[string]BesideRef {
}
func TestTheSiblingsAreWhereTheTestsStateThem(t *testing.T) { // review D2, ADR 0484 clause 4
const controller, catalogue, host = "novox/mesh-controller", "novox/mesh-catalog", "novox/mesh-host"
const catalogue, host, controller, networking = "novox/mesh-catalog", "novox/mesh-host", "novox/mesh-controller", "novox/mesh-networking"
hostRuns := strings.Repeat("7", 40)
b := newBench(t)
b.ex.Trunks = trunksOf(t, map[string]string{controller: trunkA, catalogue: trunkS, host: trunkA2, SDKRepository: "9999999999999999999999999999999999999999"})
b.ex.Trunks = trunksOf(t, map[string]string{SDKRepository: trunkS, catalogue: trunkA, host: trunkA2})
b.ex.Runs = runsAt{"mesh-host": hostRuns}
// A group of the controller and the catalogue: the controller's tests read the group's catalogue head, the heads the group
// merges (ADR 0482 §2, ADR 0573 §4; issue 1126: a judge in the group was judged against the catalogue's trunk).
out := b.judge(t, caseOf(t, pullOf(controller, 7, head1), pullOf(catalogue, 12, head2)))
// ADR 0548 §6, issue 568's own example: the verdict names the node-engine at the commit the mesh runs, and mesh-sdk at the
// trunk the catalogue's tests read it at; the catalogue, a member here, is not named.
if out.Beside[host] != hostRuns || out.Beside[catalogue] != "" || out.Beside[SDKRepository] != "9999999999999999999999999999999999999999" {
t.Errorf("the siblings named for the verdict = %v, want the node-engine at %s and mesh-sdk at its trunk", out.Beside, hostRuns[:12])
// A group of mesh-sdk and the catalogue: mesh-sdk's tests read the group's catalogue head, the heads the group merges
// (ADR 0482 §2, ADR 0573 §4; issue 1126: a judge in the group was judged against the catalogue's trunk).
out := b.judge(t, caseOf(t, pullOf(SDKRepository, 7, head1), pullOf(catalogue, 12, head2)))
// ADR 0548 §6, issue 568's own example: the verdict names the node-engine at the commit the mesh runs; the catalogue,
// a member here, is not named.
if out.Beside[host] != hostRuns || out.Beside[catalogue] != "" {
t.Errorf("the siblings named for the verdict = %v, want the node-engine at %s", out.Beside, hostRuns[:12])
}
got := besideOf(t, b, controller)
got := besideOf(t, b, SDKRepository)
if c := got[catalogue]; c.Commit != head2 || c.Ref != "refs/pull/12/head" {
t.Errorf("the catalogue beside the controller in a group = %+v, want the member's head %s", c, head2)
t.Errorf("the catalogue beside mesh-sdk in a group = %+v, want the member's head %s", c, head2)
}
if h := got[host]; h.Commit != hostRuns {
t.Errorf("the node-engine beside the controller = %+v, want the commit the mesh runs %s", h, hostRuns)
t.Errorf("the node-engine beside mesh-sdk = %+v, want the commit the mesh runs %s", h, hostRuns)
}
// ADR 0573 §2: the controller is built against mesh-sdk's latest published version, so its tests read that one too,
// not the commit its go.mod names.
if s := got[SDKRepository]; s.Commit != "9999999999999999999999999999999999999999" {
t.Errorf("mesh-sdk beside the controller = %+v, want its trunk", s)
// ADR 0593: route-proxy's proofs of a composition moved into the node-engine's tests, which read its manifest in
// mesh-networking: beside a node-engine change, at the group's head when the group holds one, else at its trunk.
b = newBench(t)
b.ex.Trunks = trunksOf(t, map[string]string{SDKRepository: trunkS, catalogue: trunkA, host: trunkA2, networking: trunkA2})
b.judge(t, caseOf(t, pullOf(host, 8, head1), pullOf(networking, 3, head2)))
if n := besideOf(t, b, host)[networking]; n.Commit != head2 || n.Ref != "refs/pull/3/head" {
t.Errorf("mesh-networking beside the node-engine in a group = %+v, want the member's head %s", n, head2)
}
// Alone, the controller's tests read the catalogue's trunk, and the verdict names it.
// Alone, mesh-sdk's tests read the catalogue's trunk, and the verdict names it.
b = newBench(t)
b.ex.Trunks = trunksOf(t, map[string]string{SDKRepository: trunkS, catalogue: trunkA, host: trunkA2})
if out := b.judge(t, caseOf(t, pullOf(SDKRepository, 7, head1))); out.Beside[catalogue] != trunkA {
t.Errorf("the siblings named for the verdict alone = %v, want the catalogue at its trunk %s", out.Beside, trunkA)
}
if c := besideOf(t, b, SDKRepository)[catalogue]; c.Commit != trunkA || c.Ref != "" {
t.Errorf("the catalogue beside mesh-sdk alone = %+v, want its trunk %s", c, trunkA)
}
// 1126/B8 (novox/hq ADR 0590, ADR 0593 §11): the controller's repository is emptied and archived, so no repository's
// tests read it and it has no siblings of its own: a pull request of it has only its own trunk beside it.
b = newBench(t)
b.ex.Trunks = trunksOf(t, map[string]string{controller: trunkA, catalogue: trunkS, host: trunkA2, SDKRepository: trunkS})
if out := b.judge(t, caseOf(t, pullOf(controller, 7, head1))); out.Beside[catalogue] != trunkS {
t.Errorf("the siblings named for the verdict alone = %v, want the catalogue at its trunk %s", out.Beside, trunkS)
}
if c := besideOf(t, b, controller)[catalogue]; c.Commit != trunkS || c.Ref != "" {
t.Errorf("the catalogue beside the controller alone = %+v, want its trunk %s", c, trunkS)
}
}
// 1126/B8 (novox/hq ADR 0590): the controller is gone, so no repository's tests read it: a mesh-host pull request has
// the catalogue and mesh-sdk beside it and no mesh-controller, even where its trunk is known and the group holds one.
func TestNoRepositorysTestsHaveTheControllerBesideThem(t *testing.T) {
const host = "novox/mesh-host"
b := newBench(t)
b.ex.Trunks = trunksOf(t, map[string]string{host: trunkA, controllerRepo: trunkA2, catalogueRepository: trunkS, SDKRepository: trunkS})
c, err := delivery.DecodeCandidate(candidateOf(t, "feat/x", pullOf(host, 5, head1), pullOf(controllerRepo, 9, head2)))
if err != nil {
t.Fatal(err)
}
if v := b.judge(t, CaseOf(c, map[string]string{host: trunkA, controllerRepo: trunkA2})).Verdict; v.State == delivery.VerdictFail &&
strings.Contains(v.Failed.Words, "mesh-controller") {
t.Fatalf("verdict = %+v", v)
}
got := besideOf(t, b, host)
if _, ok := got[controllerRepo]; ok {
t.Errorf("the controller is beside the node-engine: %+v", got)
}
if _, ok := got[catalogueRepository]; !ok {
t.Errorf("the catalogue is not beside the node-engine: %+v", got)
b.judge(t, caseOf(t, pullOf(controller, 9, head2)))
if got := besideOf(t, b, controller); len(got) != 1 || got[controller].Commit != trunkA {
t.Errorf("beside the controller's own change = %+v, want its own trunk alone", got)
}
for repo, siblings := range besideRules {
if repo != controllerRepo && slices.Contains(siblings, controllerRepo) {
t.Errorf("the tests of %s read the controller", repo)
if repo == controller || slices.Contains(siblings, controller) {
t.Errorf("the tests of %s read the controller, or it has siblings", repo)
}
}
}
@@ -157,14 +145,18 @@ func TestTheChangesOwnRepositoryIsBesideItAtItsTrunk(t *testing.T) {
}
// 1887/A12: a catalogue change's siblings hold the catalogue itself at the trunk it is merged onto, beside mesh-sdk, so
// the catalogue's shrink-only list tests read $MESH_CHECK_BESIDE/mesh-catalog.
// the catalogue's shrink-only list tests read $MESH_CHECK_BESIDE/mesh-catalog. And the node-engine at the commit the mesh
// runs (ADR 0593 §7): mesh-build-go's composer test builds the node-engine's composer command from it.
func TestACatalogueChangeHasTheCatalogueBesideItAtItsTrunk(t *testing.T) {
hostRuns := strings.Repeat("7", 40)
b := newBench(t)
b.ex.Trunks = trunksOf(t, map[string]string{catalogueRepository: trunkA, SDKRepository: trunkS, controllerRepo: trunkA2})
b.ex.Trunks = trunksOf(t, map[string]string{catalogueRepository: trunkA, SDKRepository: trunkS, "novox/mesh-controller": trunkA2, nodeEngineRepo: trunkA2})
b.ex.Runs = runsAt{nodeEngineModule: hostRuns}
out := b.judge(t, caseOf(t, pullOf(catalogueRepository, 12, head1)))
// ADR 0548 §6: the outcome names the siblings read at their trunk's head, never the repository it merges.
if out.Beside[SDKRepository] != trunkS || out.Beside[controllerRepo] != "" || out.Beside[catalogueRepository] != "" {
t.Errorf("the siblings named for the verdict = %v, want mesh-sdk at its trunk and neither the controller nor the catalogue", out.Beside)
// ADR 0548 §6: the outcome names the siblings read at their trunk's head or the commit the mesh runs, never the
// repository it merges.
if out.Beside[SDKRepository] != trunkS || out.Beside[nodeEngineRepo] != hostRuns || out.Beside["novox/mesh-controller"] != "" || out.Beside[catalogueRepository] != "" {
t.Errorf("the siblings named for the verdict = %v, want mesh-sdk at its trunk, the node-engine at the commit the mesh runs, and neither the controller nor the catalogue", out.Beside)
}
got := besideOf(t, b, catalogueRepository)
if c := got[catalogueRepository]; c.Commit != trunkA || c.Ref != "" {
@@ -173,6 +165,9 @@ func TestACatalogueChangeHasTheCatalogueBesideItAtItsTrunk(t *testing.T) {
if _, ok := got[SDKRepository]; !ok {
t.Errorf("mesh-sdk is no longer beside a catalogue change: %v", got)
}
if h := got[nodeEngineRepo]; h.Commit != hostRuns {
t.Errorf("the node-engine beside a catalogue change = %+v, want the commit the mesh runs %s", h, hostRuns)
}
}
// 1887/A12: a mesh-lab head gets a check: with its trunk known it is judged like any repository, its Go fetch finding
modules/mesh-delivery-checker/internal/check/read.go
+9 -5
View File
@@ -7,8 +7,8 @@ import (
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// The read tools (novox/hq ADR 0484 clauses 8 and 13, issue 1686): what the checker stored, as stored, with the line of the
@@ -52,11 +52,14 @@ type Reader struct {
Now func() time.Time
}
// VerdictRead is a verdict as stored with the full report of the status of each pull request it judged (Report), when the
// candidate still holds those heads; without it the report is that of the verdict alone.
// VerdictRead is a verdict as stored with every failure it names, one line each, and the full report of the status of each
// pull request it judged (Report), when the candidate still holds those heads; without it the report is that of the verdict alone.
type VerdictRead struct {
Key string `json:"key"`
Verdict json.RawMessage `json:"verdict"`
Key string `json:"key"`
Verdict json.RawMessage `json:"verdict"`
// Failed is everything the verdict names as failed, one line each, in the order judged (failedLines): every member's
// failed tests, refused steps and conflicts, the manifests' refusals and the composition (the operator, 2026-11-02).
Failed []string `json:"failed,omitempty"`
Reports map[string]string `json:"reports,omitempty"`
Note string `json:"note,omitempty"`
}
@@ -80,6 +83,7 @@ func (r Reader) read(key string, raw json.RawMessage) (VerdictRead, error) {
if err != nil {
return out, fmt.Errorf("%s holds what is not a verdict: %w", key, err)
}
out.Failed = failedLines(v)
c, ok := r.candidate(v.Group)
if !ok || (c.HeadsHash != v.Heads && GateHeads(c) != v.Heads) {
out.Note = "the candidate no longer holds these heads"

Some files were not shown because too many files have changed in this diff Show More