156 changed files with 1278 additions and 12057 deletions
-3
View File
@@ -1,4 +1 @@
vendor/
# A compiled program never enters a commit (issue 1577): `go build .` in a command directory leaves one there.
modules/*/cmd/*/mesh-*
!modules/*/cmd/*/mesh-*.go
@@ -9,7 +9,6 @@ import (
"io"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"time"
@@ -173,9 +172,6 @@ func TestTheAssignerStartsAndSaysWhatItKeeps(t *testing.T) {
// 1297/B2b: a bucket it reads that the bus does not hold stops the start, naming the bucket.
func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
for _, missing := range readBuckets {
if missing == delivery.PlacementBucket {
continue // it starts without it, below
}
b := startBed(t, missing)
select {
case err := <-b.start(t):
@@ -222,34 +218,6 @@ func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
t.Fatal("the put was not taken further once its bucket was made")
}
}
// Review of L2 (D1): without the indexer's placement state it starts all the same, and once the bucket is made (the
// indexer and the assigner come in one delivery) it is read with no restart: a module only the placement names is
// said as placed by one source only.
pb := startBed(t, delivery.PlacementBucket)
pdone := pb.start(t)
select {
case err := <-pdone:
t.Fatalf("without %s the assigner stopped: %v", delivery.PlacementBucket, err)
case <-time.After(2 * time.Second):
}
placement, err := pb.js.CreateKeyValue(pb.ctx, jetstream.KeyValueConfig{Bucket: delivery.PlacementBucket, History: 5})
if err != nil {
t.Fatal(err)
}
pl, _ := delivery.EncodePlacement(delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "web"}}})
plkey, _ := delivery.PlacementKey("n1")
if _, err := placement.Put(pb.ctx, plkey, pl); err != nil {
t.Fatal(err)
}
for deadline := time.Now().Add(15 * time.Second); ; time.Sleep(200 * time.Millisecond) {
if c, ok := pb.conditionsSaid(t)[delivery.AssignerModule+".placement.n1.web"]; ok && c.State == asks.StateRaised {
break
}
if time.Now().After(deadline) {
t.Fatal("the placement state made after the start was not read")
}
}
}
// putJSON writes a value into one of the bed's buckets.
@@ -297,10 +265,6 @@ func (b *bed) manifest(t *testing.T, module, body string) string {
// 1297/AC1: against the real bus, a fixture change touching one module on one machine is composed and answered within
// seconds under the checker's key of the check, naming the machine composed and the mesh-sdk it composed with.
func TestAChangeIsComposedOnTheBus(t *testing.T) {
// The placement answers' bound (ADR 0559 §5), shortened before the program starts and put back after it stops.
bound := assign.AnswerBound
t.Cleanup(func() { assign.AnswerBound = bound })
assign.AnswerBound = 1500 * time.Millisecond
b := startBed(t, "")
trunk := b.manifest(t, "web", `{"module":"web","version":"1"}`)
changed := b.manifest(t, "web", `{"module":"web","version":"2"}`)
@@ -352,63 +316,6 @@ func TestAChangeIsComposedOnTheBus(t *testing.T) {
} else if info, err := st.Info(b.ctx); err != nil || info.State.Msgs != 0 {
t.Errorf("a group without a mesh-sdk member put %d asks on the Go build seat (%v)", info.State.Msgs, err)
}
// ADR 0559 §5 (issue 1126): a catalogue pull request that places g14 for the first time (the seed's case) is composed
// with its candidate placement, touches g14, and waits for g14's answer at its head; g14 answering the same passes it.
// A node that does not answer within the bound fails the check, named, at the bound's end with no other event; its later
// answer at the same head passes the same heads' composition.
record, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "g14"}})
b.putJSON(t, delivery.HaveBucket, "g14.node", record)
placed := func(group string, number int, head string) (string, uint64) {
t.Helper()
value, err := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Group: group,
Pull: delivery.Head{Repository: delivery.CatalogueRepository, Number: number, Head: head},
Placement: &delivery.Placement{Node: "g14", Modules: []delivery.PlacedModule{{Module: "web"}}}})
if err != nil {
t.Fatal(err)
}
key, _ := delivery.PlacementCandidateKey(group, "g14")
rev, err := b.buckets[delivery.PlacementCandidatesBucket].Put(b.ctx, key, value)
if err != nil {
t.Fatal(err)
}
pull := []delivery.CandidatePull{{Head: delivery.Head{Repository: delivery.CatalogueRepository, Number: number, Head: head}, Modules: []string{}}}
hash, _ := delivery.HeadsHash([]delivery.Head{pull[0].Head})
cand, err := delivery.EncodeCandidate(delivery.Candidate{Group: group, HeadsHash: hash, Pulls: pull, FPSet: delivery.FingerprintSet(pull)})
if err != nil {
t.Fatal(err)
}
ckey, _ := delivery.CandidatesKey(group)
b.putJSON(t, delivery.CandidatesBucket, ckey, cand)
key, _ = delivery.CompositionKey(group, hash)
return key, rev
}
answer := func(group, head string, rev uint64) {
t.Helper()
have, err := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "g14"},
Candidates: map[string]delivery.CandidateAnswer{group: {Head: head, Revision: rev, Verdict: delivery.CandidateSame, Same: []string{"web"}}}})
if err != nil {
t.Fatal(err)
}
b.putJSON(t, delivery.HaveBucket, "g14.node", have)
}
seedHead, lateHead := strings.Repeat("5", 40), strings.Repeat("6", 40)
seed, seedRev := placed("feat/seed", 20, seedHead)
waiting := b.verdict(t, seed, func(v delivery.CompositionVerdict) bool { return v.State == delivery.CompositionWaits })
if !strings.Contains(waiting.Words, "g14") {
t.Errorf("the seed's composition waits saying %q, want g14 named", waiting.Words)
}
answer("feat/seed", seedHead, seedRev)
if v := b.verdict(t, seed, func(v delivery.CompositionVerdict) bool { return v.State == delivery.CompositionPass }); !slices.Contains(v.Machines, "g14") {
t.Errorf("the seed's composition = %+v, want g14 composed", v)
}
late, lateRev := placed("feat/late", 21, lateHead)
failed := b.verdict(t, late, func(v delivery.CompositionVerdict) bool { return v.State == delivery.CompositionFail })
if failed.Machine != "g14" || !strings.Contains(failed.Words, delivery.CompositionUnanswered) {
t.Errorf("a node silent past its bound: %+v", failed)
}
answer("feat/late", lateHead, lateRev)
b.verdict(t, late, func(v delivery.CompositionVerdict) bool { return v.State == delivery.CompositionPass })
}
// get waits for a key of one of the bed's buckets and answers its value.
@@ -499,32 +406,6 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
if !cleared {
t.Errorf("the condition of web on neither list on g14 was not cleared once g14 got its want")
}
// Issue 1126 (ADR 0547 §2): the catalogue's placement reaches the indexer's state naming web on every node the
// assignments name it on. Each node then composes its own: its want names the build alone (Self, no composition), and
// its own want says it composes its own, which the node agent reads (mesh-host's TestNats...ComposesAndAppliesItsOwn).
for _, node := range []string{"ace", "g14", "shanks"} {
pl, err := delivery.EncodePlacement(delivery.Placement{Node: node, Modules: []delivery.PlacedModule{{Module: "web"}}})
if err != nil {
t.Fatal(err)
}
pkey, _ := delivery.PlacementKey(node)
b.putJSON(t, delivery.PlacementBucket, pkey, pl)
}
if w := b.wantNow(t, "ace", "web", "ace's want once placed: the build alone, composed by the node", func(w delivery.Want) bool { return w.Self }); w.Composed != "" || w.FP != fNew {
t.Errorf("ace's want once placed = %+v", w)
}
nkey, _ := delivery.WantNodeKey("ace")
var self bool
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && !self; time.Sleep(100 * time.Millisecond) {
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, nkey); err == nil {
nw, err := delivery.DecodeNodeWant(e.Value())
self = err == nil && nw.Self
}
}
if !self {
t.Error("ace's own want does not say it composes its own once placed")
}
}
// 1297/B9 (the assigner's half, ADR 0528 clause 6): started whole, the assigner holds a fingerprint whose health check
@@ -8,12 +8,13 @@ package main
import (
"context"
"errors"
"fmt"
"io"
"log"
"os"
"os/signal"
"sync"
"strings"
"syscall"
"time"
@@ -29,7 +30,7 @@ import (
)
func main() {
if os.Getenv(process.BrokerFileEnv) == "" {
if os.Getenv("MESH_BROKER_FILE") == "" {
// Started by the tool runner, with no credential of its own: the put's tools (ADR 0546), served through the runtime,
// since a process never serves (ADR 0436). The process the manifest runs names its credential.
if err := serveTools(); err != nil {
@@ -58,8 +59,6 @@ var readBuckets = []string{
delivery.CandidatesBucket,
delivery.ManifestsBucket,
delivery.ModulesBucket,
delivery.PlacementBucket,
delivery.PlacementCandidatesBucket,
delivery.PresenceBucket,
delivery.HaveBucket,
delivery.StepsBucket(goBuildModule),
@@ -99,12 +98,6 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
logf("the bucket %s cannot be opened (%v): no put is taken further until the assigner starts again", name, err)
continue
}
if name == delivery.PlacementBucket && err != nil {
// The indexer's placement state comes with the second milestone (ADR 0549 §2): without it the controller's
// assignments alone name where modules run, as before any placement was read (ADR 0549 §5).
logf("the bucket %s cannot be opened (%v): the controller's assignments alone are read until the assigner starts again", name, err)
continue
}
if err != nil {
return nil, process.EssentialErr("assigner", name, "write a want from the whole mesh", err)
}
@@ -116,15 +109,33 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
// run is the program: it returns when ctx ends or the bus cannot be used.
func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
logger := log.New(logOut, "mesh-delivery-assigner: ", log.LstdFlags)
account, err := process.ReadBrokerAccount(getenv)
path := strings.TrimSpace(getenv("MESH_BROKER_FILE"))
if path == "" {
return errors.New("MESH_BROKER_FILE is not set, so there is no credential to read the bus with")
}
account, err := process.ReadAccount(path)
if err != nil {
return err
}
conn, js, err := account.Dial()
opts := []nats.Option{
nats.Name(account.Node + "." + account.Module),
nats.UserInfo(account.User, account.Password),
nats.CustomInboxPrefix(account.InboxPrefix()),
nats.MaxReconnects(-1),
nats.RetryOnFailedConnect(true),
}
if strings.TrimSpace(account.Fingerprint) != "" {
opts = append(opts, nats.Secure(process.PinnedToFingerprint(account.Fingerprint)))
}
conn, err := nats.Connect(account.URL, opts...)
if err != nil {
return err
return fmt.Errorf("connecting to the bus: %w", err)
}
defer conn.Close()
js, err := jetstream.New(conn)
if err != nil {
return err
}
kvs, err := open(ctx, js, logger.Printf)
if err != nil {
return err
@@ -140,7 +151,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
logger.Printf("what the assigner keeps could not be said on the bus, so its readers are not granted until it is: %v", err)
}
logger.Printf("started on the account %s, reading %d buckets", account.User, len(kvs))
return follow(ctx, js, conn, kvs, sayVia(ctx, js), emitting{said}, logger)
return follow(ctx, js, kvs, sayVia(ctx, js), emitting{said}, logger)
}
// emitting is the process connection as the puts' caller: its calls, and its events with no options.
@@ -158,8 +169,6 @@ var watched = []string{
delivery.BuilderBuildsBucket,
delivery.CandidatesBucket,
delivery.ModulesBucket,
delivery.PlacementBucket,
delivery.PlacementCandidatesBucket,
delivery.PresenceBucket,
delivery.HaveBucket,
delivery.AssignerAsksBucket,
@@ -170,7 +179,7 @@ var watched = []string{
// follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after
// each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends.
func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, kvs buckets, say func(asks.Condition) error, bus caller, logger *log.Logger) error {
func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(asks.Condition) error, bus caller, logger *log.Logger) error {
w := newWorld()
dirty := make(chan struct{}, 1)
wake := func() {
@@ -214,7 +223,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
if e.Operation() == jetstream.KeyValuePut {
value = e.Value()
}
w.putEntry(name, e.Key(), value, e.Revision(), e.Created())
w.put(name, e.Key(), value)
wake()
}
}
@@ -254,12 +263,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
}
logger.Printf("the bucket %s is open: puts are taken further", delivery.AssignerPutsBucket)
}
started := 0
for _, name := range watched {
if kvs[name] == nil && (name == delivery.PlacementBucket || name == delivery.PlacementCandidatesBucket) {
continue // not on the bus at this start (open said so)
}
started++
var opts []jetstream.WatchOpt
if name == delivery.HaveBucket {
opts = append(opts, jetstream.IncludeHistory()) // what each node ran healthy before, for a set-back
@@ -271,29 +275,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
if err := watch(delivery.ManifestsBucket, jetstream.MetaOnly()); err != nil {
return err
}
// The indexer's placement state arrives with the same delivery as this assigner (ADR 0549 §2): when it is not on the
// bus at this start it is bound once the bus says it was created, and watched from then on, with no restart (issue
// 1790's kind). Until then the controller's assignments alone name where modules run (ADR 0549 §5).
type appeared struct {
name string
kv jetstream.KeyValue
}
placementAppeared := make(chan appeared, 2)
if kvs[delivery.PlacementBucket] == nil && js != nil {
process.AwaitBucket(ctx, js, nc, delivery.PlacementBucket, "the controller's assignments alone are read",
logger.Printf).Ready(func(kv jetstream.KeyValue) { placementAppeared <- appeared{delivery.PlacementBucket, kv} })
}
// So are the placement candidates (ADR 0559 §3), with the same delivery as the indexer that writes them.
if kvs[delivery.PlacementCandidatesBucket] == nil && js != nil {
process.AwaitBucket(ctx, js, nc, delivery.PlacementCandidatesBucket, "no change's placement is composed",
logger.Printf).Ready(func(kv jetstream.KeyValue) { placementAppeared <- appeared{delivery.PlacementCandidatesBucket, kv} })
}
takePlacement := func(a appeared) error {
kvs[a.name] = a.kv
logger.Printf("the bucket %s is open and watched", a.name)
return watch(a.name)
}
for range started + 1 {
for range len(watched) + 1 {
select {
case <-ctx.Done():
return nil
@@ -303,7 +285,6 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
}
}
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, said: map[string]bool{}, answeredAt: map[string]uint64{}}
p.wakeAt = wakeAtVia(ctx, wake)
p.asks = newComposeAsks(ctx, buildrun.Asker{Publisher: buildrun.JetStreamAsks{JS: js}, Waiter: buildrun.JetStreamAsks{JS: js}}, wake, logger.Printf)
if bus != nil && kvs[delivery.AssignerPutsBucket] != nil {
p.acts = newPutActs(bus, wake, logger.Printf)
@@ -316,10 +297,6 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
case err := <-errs:
return err
case <-settled:
case kv := <-placementAppeared:
if err := takePlacement(kv); err != nil {
return err
}
case <-dirty:
reopenPuts(p)
p.pass(ctx)
@@ -327,31 +304,6 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
}
}
// wakeAtVia asks wake once at each time it is given, as a context's deadline (ADR 0437 §8: no timer), each time once.
func wakeAtVia(ctx context.Context, wake func()) func(time.Time) {
var mu sync.Mutex
asked := map[int64]bool{}
return func(at time.Time) {
mu.Lock()
defer mu.Unlock()
if asked[at.UnixNano()] {
return
}
asked[at.UnixNano()] = true
go func() {
bound, cancel := context.WithDeadline(ctx, at)
defer cancel()
<-bound.Done()
mu.Lock()
delete(asked, at.UnixNano())
mu.Unlock()
if ctx.Err() == nil {
wake()
}
}()
}
}
// PutsReopenEvery is how often, at most, a pass opens the puts' bucket again while it is not open.
var PutsReopenEvery = time.Minute
@@ -94,7 +94,7 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
kvs[delivery.ControllerComposedBucket].(*watchKV).entries = []kvEntry{{key: ckey, value: cd}}
kvs[delivery.AssignerPutsBucket].(*watchKV).closed = true
kvs[delivery.AssignerWantsBucket].(*watchKV).wrote = wrote
go follow(ctx, nil, nil, kvs, func(asks.Condition) error { return nil }, nil, log.New(io.Discard, "", 0))
go follow(ctx, nil, kvs, func(asks.Condition) error { return nil }, nil, log.New(io.Discard, "", 0))
select {
case key := <-wrote:
if key != "want.ace.node" {
@@ -170,5 +170,3 @@ type kvEntry struct {
func (e kvEntry) Key() string { return e.key }
func (e kvEntry) Value() []byte { return e.value }
func (e kvEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut }
func (e kvEntry) Revision() uint64 { return 1 }
func (e kvEntry) Created() time.Time { return time.Time{} }
@@ -3,7 +3,7 @@ package main
import (
"context"
"errors"
"maps"
"fmt"
"slices"
"strings"
"time"
@@ -40,12 +40,6 @@ type passer struct {
// noWant is, by node and module, the reason last said why the node gets no want for the module (issue 1945): a reason is
// said in the journal when it appears or changes, and its clearing when the node has a want again; one line per change.
noWant map[string]string
// nodeFailed is, by node and module, the words of the condition last raised for a failure that is the node's own (ADR 0557
// §1, issue 1496): raised once per words, cleared once when it ends.
nodeFailed map[string]string
// wakeAt asks a pass at a time: the end of a placement candidate's answer bound (ADR 0559 §5), as a context's deadline
// (ADR 0437 §8: no timer). Nil asks none.
wakeAt func(at time.Time)
}
func (p *passer) clock() time.Time {
@@ -67,23 +61,7 @@ func (p *passer) pass(ctx context.Context) {
// controller has composed it and again whenever that digest changes (ADR 0539 §6).
func (p *passer) nodeWants(ctx context.Context) {
written := p.w.read(delivery.AssignerWantsBucket)
placed := p.w.placedNodes()
// A node the catalogue places composes its own composition (ADR 0547 §2): its own want names none.
for node := range placed {
key, err := delivery.WantNodeKey(node)
if err != nil {
continue
}
if before, err := delivery.DecodeNodeWant(written[key]); err == nil && before.Self {
continue
}
want := delivery.NodeWant{Self: true, Words: "the node composes its own composition (ADR 0547 §2)"}
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeNodeWant(want) })
}
for node, d := range p.w.nodeComposed() {
if placed[node] {
continue
}
key, err := delivery.WantNodeKey(node)
if err != nil {
continue
@@ -120,8 +98,6 @@ func (p *passer) wants(ctx context.Context) {
}
}
wanted := p.w.wantsOf()
frozen := p.sayPlacement(p.w.placementDiffers())
placed := p.w.placedNodes()
puts := p.putsByModule()
p.staleAsks(ctx, trunk)
names := map[string]bool{}
@@ -138,21 +114,12 @@ func (p *passer) wants(ctx context.Context) {
// quiet at the router.
first := p.neither == nil
stale := map[string]neitherOne{}
left := map[string]string{}
defer func() { p.sayNeither(neither, stale, left) }()
defer func() { p.sayNeither(neither, stale) }()
for name := range names {
fp := trunk[name].FP2
name := name
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name],
Self: func(node string) bool { return placed[node] },
Composed: func(node, fp string) (delivery.ComposedDigest, bool) {
if placed[node] {
// The node composes its own (ADR 0547 §2, §4): no composition of the controller's is waited for, the want
// names the build alone, and the node judges the move itself.
return delivery.ComposedDigest{Judgement: delivery.JudgedNeither, Words: "the node composes its own"}, true
}
return p.w.composed(node, name, fp)
},
Composed: func(node, fp string) (delivery.ComposedDigest, bool) { return p.w.composed(node, name, fp) },
GrantedAt: func(fp string) bool {
key, err := delivery.ControllerGrantKey(name, fp)
return err == nil && p.w.has(delivery.ControllerGrantsBucket, key)
@@ -186,11 +153,8 @@ func (p *passer) wants(ctx context.Context) {
last[node] = p.w.lastHealthy(node, name, fp)
}
out := assign.Wants(m, nodes, wanted[name], last)
p.yeses(ctx, name, m, wanted[name], &out)
gatherNeither(neither, name, out)
leftNeither(left, m, wanted[name], out)
p.sayNoWant(name, m.Nodes, wanted[name], out)
p.sayNodeFailed(name, fp, m.Nodes, nodes, out, first)
if first {
startClears(stale, m, out)
if fp != "" && !m.Held && out.Held == nil && !out.Lifted {
@@ -218,9 +182,6 @@ func (p *passer) wants(ctx context.Context) {
p.recordAsk(ctx, name, fp, need)
}
for node, want := range out.Wants {
if frozen[node+"."+name] {
continue // one source alone names it (ADR 0549 §5): no want, and the one written before stays
}
key, err := delivery.WantKey(node, name)
if err != nil {
p.logf("no want of %s on %s is written: %v", name, node, err)
@@ -249,7 +210,7 @@ func (p *passer) sayNoWant(module string, nodes []string, before map[string]deli
}
reason := ""
if h, held := out.Rules[node]; held && (!written || w.Removed) {
reason = h.Reason()
reason = fmt.Sprintf("rule %d: %s", h.Rule, h.Words)
}
last, said := p.noWant[key]
switch {
@@ -268,60 +229,6 @@ func (p *passer) sayNoWant(module string, nodes []string, before map[string]deli
}
}
// sayNodeFailed raises, once per node and words, the condition of a failed health check of a module's newest fingerprint fp
// that is the node's own, since fp runs healthy elsewhere and is not held (ADR 0557 §1, issue 1496), and says it in the journal;
// it clears the condition once when the node no longer fails alone, or is no longer one of the module's nodes. The first pass
// after a start remembers nothing a previous process raised: it clears, for every module, the condition of every node the
// presence bucket knows that does not fail alone, since a node that recovered while no process watched leaves no failed have to
// find it by (a clear of a condition not open is quiet at the router). A condition the router did not take is said again at the next pass.
func (p *passer) sayNodeFailed(module, fp string, nodes []string, known assign.Nodes, out assign.Outcome, first bool) {
if p.nodeFailed == nil {
p.nodeFailed = map[string]string{}
}
look := map[string]bool{}
for _, node := range nodes {
look[node] = true
}
for key := range p.nodeFailed {
if node, m, _ := strings.Cut(key, "."); m == module {
look[node] = true
}
}
if first {
for node := range known.Online {
look[node] = true
}
}
for _, node := range slices.Sorted(maps.Keys(look)) {
key := node + "." + module
f, failing := out.NodeFailed[node]
last, said := p.nodeFailed[key]
switch {
case failing && f.Words+"|"+strings.Join(f.Others, ",") != last:
p.logf("%s on %s: the health check at %s failed here (%s), while it runs healthy on %s: the failure is %s's, so nothing "+
"is held or set back", module, node, fp, f.Words, strings.Join(f.Others, ", "), node)
if p.say != nil {
if err := p.say(assign.NodeFailedCondition(node, module, fp, f, true)); err != nil {
p.logf("the condition of %s failing on %s alone could not be said, and is said at the next pass: %v", module, node, err)
continue
}
}
p.nodeFailed[key] = f.Words + "|" + strings.Join(f.Others, ",")
case !failing && (said || first):
if said {
p.logf("%s on %s no longer fails its health check alone", module, node)
}
if p.say != nil {
if err := p.say(assign.NodeFailedCondition(node, module, fp, assign.NodeFailure{}, false)); err != nil && said {
p.logf("the condition of %s failing on %s alone could not be cleared, and is cleared at the next pass: %v", module, node, err)
continue
}
}
delete(p.nodeFailed, key)
}
}
}
// neitherOne is a module assigned to a node that holds no want for it after a pass, and the rule holding its first want back.
type neitherOne struct {
node, module string
@@ -335,48 +242,6 @@ func gatherNeither(into map[string]neitherOne, module string, out assign.Outcome
}
}
// leftNeither says, by node and module, for every node of m not on neither list after the pass, why: "" when the node has a live
// want for it (written before and kept, or written in this pass), else why it has none. sayNeither clears a neither condition in
// these words, so a clear never says a want the node does not hold (issue 1945: "nats on novox has a want again" while the bus bar
// left it with none).
func leftNeither(into map[string]string, m assign.Module, before map[string]delivery.Want, out assign.Outcome) {
for _, node := range m.Nodes {
if _, on := out.Neither[node]; on {
continue
}
w, written := out.Wants[node]
if !written {
w, written = before[node]
}
why := ""
h, ruled := out.Rules[node]
switch {
case written && !w.Removed:
case (m.Held && !out.Lifted) || out.Held != nil:
why = "it is held, and its held condition says so"
case slices.ContainsFunc(out.Record, func(r assign.RecordNeed) bool { return r.Node == node }):
why = "its move waits for the operator's yes on its record ask"
case putOn(m, node):
why = "its put waits on its own steps and the operator's answer"
case m.FP == "":
why = "the trunk has no build of it"
case ruled:
why = h.Reason()
default:
why = "the node does not run it"
}
into[node+"."+m.Name] = why
}
}
// putOn says whether an operator's put on node owns m's want there: one still on its steps (ADR 0546), so the rules write none
// meanwhile. An ended put (done, refused, released) stays in the bucket and owns nothing.
func putOn(m assign.Module, node string) bool {
p, put := m.Puts[node]
_, owns := assign.PutOwns(p)
return put && owns
}
// startClears adds, for the first pass after a start, every node of m that is not on neither list after the pass: a neither
// condition a previous process raised for it is cleared. That covers a node with a want and one with none that is not on neither
// list, such as a held module (issue 1945: the hold is said on its own, so a neither raised for it before the restart would stay up
@@ -393,9 +258,7 @@ func startClears(into map[string]neitherOne, m assign.Module, out assign.Outcome
// once, the condition of each one said before that has a want again (issue 1945). A say that fails is made again at the next pass.
// stale is, on the first pass after a start, the modules whose condition a previous process may have raised and that have a want:
// each is cleared once.
// left is, by node and module, why each assigned node not on neither list after this pass has no want ("" when it has one); a key
// it does not hold is no longer assigned.
func (p *passer) sayNeither(now, stale map[string]neitherOne, left map[string]string) {
func (p *passer) sayNeither(now, stale map[string]neitherOne) {
if p.neither == nil {
p.neither = map[string]neitherOne{}
for key, one := range stale {
@@ -415,29 +278,21 @@ func (p *passer) sayNeither(now, stale map[string]neitherOne, left map[string]st
continue
}
}
p.logf("%s is assigned to %s and on neither list: no want, and not in the node's own composition; what holds its first want back: %s",
one.module, one.node, one.holding.Reason())
p.logf("%s is assigned to %s and on neither list: no want, and not in the node's own composition; rule %d holds its first want back: %s",
one.module, one.node, one.holding.Rule, one.holding.Words)
p.neither[key] = one
}
for key, one := range p.neither {
if _, still := now[key]; still {
continue
}
why, assigned := left[key]
if !assigned {
why = "it is no longer assigned there"
}
if p.say != nil {
if err := p.say(assign.NeitherLeftCondition(one.node, one.module, why)); err != nil {
if err := p.say(assign.NeitherCondition(one.node, one.module, assign.Holding{}, false)); err != nil {
p.logf("the condition of %s on neither list on %s could not be cleared, and is cleared at the next pass: %v", one.module, one.node, err)
continue
}
}
if why == "" {
p.logf("%s on %s has a want again", one.module, one.node)
} else {
p.logf("%s on %s is no longer on neither list and has no want: %s", one.module, one.node, why)
}
p.logf("%s on %s has a want again", one.module, one.node)
delete(p.neither, key)
}
}
@@ -482,9 +337,6 @@ func (p *passer) compositions(ctx context.Context) {
} else {
v = assign.Compose(*mesh, ch, p.sdk, time.Since(began))
}
if ch.Placement != nil && v.State != delivery.CompositionWaits {
v = p.answers(*mesh, ch, v)
}
}
if before, err := delivery.DecodeCompositionVerdict(written[out]); err == nil {
before.Took, before.V = v.Took, 0
@@ -496,40 +348,6 @@ func (p *passer) compositions(ctx context.Context) {
}
}
// answers judges every placed node's answer to the change's placement into v (ADR 0559 §5). While a node has not answered
// within the bound the verdict waits, and the pass at the bound's end is asked (wakeAt), so a silent node is failed then,
// named, and a later answer of the same head is judged on its own have's change.
func (p *passer) answers(mesh assign.Mesh, ch assign.Change, v delivery.CompositionVerdict) delivery.CompositionVerdict {
trunkPlaced := map[string]delivery.Placement{}
for key, value := range p.w.read(delivery.PlacementBucket) {
node, err := delivery.ParsePlacementKey(key)
if err != nil {
continue
}
if pl, err := delivery.DecodePlacement(value); err == nil && pl.Node == node {
trunkPlaced[node] = pl
}
}
fps := map[string]string{}
for name, m := range p.w.trunk() {
if m.FP2 != "" {
fps[name] = m.FP2
} else if m.FP != "" {
fps[name] = m.FP
}
}
intended, notes := assign.Intended(mesh.Input, trunkPlaced, ch.Placement, fps)
a := assign.JudgeAnswers(ch.Placement, assign.PlacedNodes(trunkPlaced, ch.Placement), p.w.candidateAnswers(ch.Group), intended, p.w.online(), p.clock())
v = assign.WithAnswers(v, a)
if v.State != delivery.CompositionWaits {
v.Warnings = append(v.Warnings, notes...)
}
if a.Waits && p.wakeAt != nil {
p.wakeAt(a.Until)
}
return v
}
// put writes one key of the assigner's own bucket; a write that fails is said and made again at the next pass.
func (p *passer) put(ctx context.Context, bucket, key string, encode func() ([]byte, error)) {
value, err := encode()
@@ -857,83 +675,3 @@ func (p *passer) backupAsk(ctx context.Context, key, node, module string, want d
}
p.said[c.Key] = true
}
// sayPlacement raises, once, the condition of every pair the controller's assignments and the catalogue's placement do not
// both name, clears it once they agree, and answers those pairs by node and module (ADR 0549 §5): the pass writes no want
// for them and keeps the one written before. A say that fails is made again at the next pass.
func (p *passer) sayPlacement(differ []assign.Pair) map[string]bool {
frozen := map[string]bool{}
now := map[string]bool{}
for _, d := range differ {
frozen[d.Node+"."+d.Module] = true
c := assign.PlacementCondition(d, true)
now[c.Key] = true
if p.said[c.Key+".raised"] {
continue
}
p.logf("%s on %s is named by %s alone: no want is written for it there until both sources name it (ADR 0549 §5)", d.Module, d.Node, d.Only)
if err := p.say(c); err != nil {
p.logf("the placement condition of %s on %s could not be said, and is said at the next pass: %v", d.Module, d.Node, err)
continue
}
p.said[c.Key+".raised"] = true
}
prefix := delivery.AssignerModule + ".placement."
for key := range p.said {
c, raised := strings.CutSuffix(key, ".raised")
if !raised || !strings.HasPrefix(c, prefix) || now[c] {
continue
}
node, module, _ := strings.Cut(strings.TrimPrefix(c, prefix), ".")
if err := p.say(assign.PlacementCondition(assign.Pair{Node: node, Module: module}, false)); err != nil {
p.logf("the placement condition of %s on %s could not be cleared, and is cleared at the next pass: %v", module, node, err)
continue
}
delete(p.said, key)
}
return frozen
}
// yeses writes a placed node's wants naming the build alone, and sees to the moves a node that composes its own judged a
// person must allow (ADR 0547 §3): for a have of phase
// waiting-yes it raises the record ask once, naming the composition waiting and the node's judgement, and on a yes writes
// the want again naming it; a yes once written is kept while the want stays at its build, so the node is not asked again.
func (p *passer) yeses(ctx context.Context, module string, m assign.Module, before map[string]delivery.Want, out *assign.Outcome) {
for _, node := range m.Nodes {
if m.Self == nil || !m.Self(node) {
continue
}
was, had := before[node]
w, written := out.Wants[node]
if !written {
if !had || was.Removed {
continue
}
w = was
}
if w.Removed {
continue
}
yes := ""
if had && was.FP == w.FP {
yes = was.Yes
}
key, err := delivery.HaveKey(node, module)
if err != nil {
continue
}
if h, err := delivery.DecodeHave(p.w.value(delivery.HaveBucket, key)); err == nil && h.Phase == delivery.PhaseWaitingYes && h.Waits != yes {
if m.Allowed != nil && m.Allowed(node, h.Waits) {
yes = h.Waits
} else {
p.recordAsk(ctx, module, w.FP, assign.RecordNeed{Node: node, Digest: h.Waits, Judgement: h.Judgement, Words: h.Detail})
}
}
// A want carried over from before the node composed its own may still name the controller's composition; a placed
// node's want names the build alone (ADR 0547 §2).
if yes != w.Yes || w.Composed != "" || !w.Self {
w.Yes, w.Composed, w.Self = yes, "", true
out.Wants[node] = w
}
}
}
@@ -342,9 +342,6 @@ func putOfTheBus(t *testing.T) {
// again after a say that failed, and clears it once when the module has a want again; a module on neither list is also written to
// the journal, so it is loud with no operator channel.
func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
// A pair only one source names is a condition of the same kind (issue 1126): folded here to hold the count.
aPairOneSourceNamesAloneGetsNoWantAndOneCondition(t)
t.Run("gathers", testThePassGathersTheModulesOnNeitherList)
p := newPasser(buckets{}, time.Date(2026, 10, 25, 15, 0, 0, 0, time.UTC))
var said []asks.Condition
failing := true
@@ -360,66 +357,30 @@ func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
one := map[string]neitherOne{"g14.screen-lock": {node: "g14", module: "screen-lock",
holding: assign.Holding{Rule: delivery.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}}}
p.sayNeither(one, nil, nil) // the say fails
p.sayNeither(one, nil, nil)
p.sayNeither(one, nil, nil)
p.sayNeither(one, nil) // the say fails
p.sayNeither(one, nil)
p.sayNeither(one, nil)
if len(said) != 1 || said[0].State != asks.StateRaised || said[0].Subject.Machine != "g14" || said[0].Subject.ID != "screen-lock" {
t.Fatalf("said = %+v, want one raised condition of screen-lock on g14", said)
}
if !strings.Contains(strings.Join(logged, "\n"), "screen-lock") {
t.Errorf("the journal does not name the module on neither list: %q", logged)
}
wants := map[string]string{"g14.screen-lock": ""}
p.sayNeither(map[string]neitherOne{}, nil, wants)
p.sayNeither(map[string]neitherOne{}, nil, wants)
p.sayNeither(map[string]neitherOne{}, nil)
p.sayNeither(map[string]neitherOne{}, nil)
if len(said) != 2 || said[1].State != asks.StateCleared || said[1].Key != said[0].Key {
t.Errorf("said = %+v, want the condition cleared once under its key", said)
}
if last := logged[len(logged)-1]; last != "screen-lock on g14 has a want again" {
t.Errorf("the clear said %q", last)
}
// 1945 (the bus bar): a module that leaves neither list with no want is cleared in words saying so, never "has a want again".
bus := map[string]neitherOne{"novox.nats": {node: "novox", module: "nats", holding: assign.Holding{Bus: true, Words: "the bus moves only by the operator's put"}}}
p.sayNeither(bus, nil, nil)
if c := said[len(said)-1]; c.State != asks.StateRaised || !strings.HasPrefix(c.Explanation, asks.NeedsYou) || !strings.Contains(c.Explanation, "the bus moves only by the operator's put") {
t.Errorf("the bus's neither condition = %+v", c)
}
p.sayNeither(map[string]neitherOne{}, nil, map[string]string{"novox.nats": "it is held, and its held condition says so"})
if c := said[len(said)-1]; c.State != asks.StateCleared || strings.Contains(c.Resolved, "has a want") {
t.Errorf("a module that left neither list with no want was cleared as %+v", c)
}
if last := logged[len(logged)-1]; last != "nats on novox is no longer on neither list and has no want: it is held, and its held condition says so" {
t.Errorf("the clear without a want said %q", last)
}
}
// 1945/AC21: the pass gathers the modules on neither list of every module's outcome, keyed by node and module.
func testThePassGathersTheModulesOnNeitherList(t *testing.T) {
func TestThePassGathersTheModulesOnNeitherList(t *testing.T) {
got := map[string]neitherOne{}
gatherNeither(got, "screen-lock", assign.Outcome{Neither: map[string]assign.Holding{"g14": {Rule: delivery.RuleGrant, Words: "w"}}})
gatherNeither(got, "rofi", assign.Outcome{})
if len(got) != 1 || got["g14.screen-lock"].node != "g14" || got["g14.screen-lock"].module != "screen-lock" || got["g14.screen-lock"].holding.Rule != delivery.RuleGrant {
t.Errorf("gathered = %+v", got)
}
// 1945 (the bus bar): a node off neither list is cleared in words that say why it has no want; an operator's put running
// through its steps owns the want, so it is named, never "the node does not run it".
for _, c := range []struct {
name string
puts map[string]delivery.Put
before map[string]delivery.Want
why string
}{
{"a live want", nil, map[string]delivery.Want{"novox": {FP: fpTest}}, ""},
{"a put on its steps", map[string]delivery.Put{"novox": {FP: fpTest, Step: delivery.PutPerson}}, nil, "its put waits on its own steps and the operator's answer"},
{"nothing", nil, nil, "the node does not run it"},
{"an ended put", map[string]delivery.Put{"novox": {FP: fpTest, Step: delivery.PutRefused}}, nil, "the node does not run it"},
} {
left := map[string]string{}
leftNeither(left, assign.Module{Name: "nats", FP: fpTest, Nodes: []string{"novox"}, Puts: c.puts}, c.before, assign.Outcome{})
if why, there := left["novox.nats"]; !there || why != c.why {
t.Errorf("%s: left %q (%v), want %q", c.name, why, there, c.why)
}
}
// 1945 (a first want held back silently): the journal says why a node gets no want when the reason appears or changes, and
// when it has a want again: one line per change, not per pass.
p := newPasser(buckets{}, time.Date(2026, 10, 25, 21, 30, 0, 0, time.UTC))
@@ -438,7 +399,6 @@ func testThePassGathersTheModulesOnNeitherList(t *testing.T) {
{nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: delivery.RuleGrant, Words: "no credential"}}}, "rofi on shanks gets no want: rule 6: no credential"},
{nil, wanted, "rofi on shanks has a want"},
{map[string]delivery.Want{"shanks": wanted.Wants["shanks"]}, assign.Outcome{}, ""},
{nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Bus: true, Words: "the bus moves only by the operator's put"}}}, "rofi on shanks gets no want: the bus moves only by the operator's put"},
} {
logged = nil
p.sayNoWant("rofi", []string{"shanks"}, c.before, c.out)
@@ -446,71 +406,12 @@ func testThePassGathersTheModulesOnNeitherList(t *testing.T) {
t.Errorf("pass %d said %q, want %q", i, got, c.line)
}
}
// 1496, ADR 0557 §1: a failure that is the node's own (its fingerprint healthy elsewhere, not held) raises the node's condition
// once, naming the node, the module, F and the have's words, and clears it once when it ends.
var conds []asks.Condition
p.say = func(c asks.Condition) error { conds = append(conds, c); return nil }
failing := map[string]assign.NodeFailure{"shanks": {Others: []string{"ace"}, Words: "health failed, phase running: w"}}
for i, c := range []struct {
failed map[string]assign.NodeFailure
state string // the condition said in this pass, "" for none
}{
{failing, asks.StateRaised},
{failing, ""},
{nil, asks.StateCleared},
{nil, ""},
} {
conds = nil
p.sayNodeFailed("rofi", fpTest, []string{"shanks"}, assign.Nodes{}, assign.Outcome{NodeFailed: c.failed}, false)
switch {
case c.state == "" && len(conds) != 0, c.state != "" && (len(conds) != 1 || conds[0].State != c.state):
t.Errorf("node failed, pass %d said %+v, want %q", i, conds, c.state)
case c.state == asks.StateRaised && (conds[0].Key != "mesh-delivery-assigner.failed-alone.shanks.rofi" ||
!strings.Contains(conds[0].Explanation, "failed on shanks (health failed, phase running: w), while it runs healthy on ace")):
t.Errorf("node failed, raised %+v", conds[0])
}
}
// D2: a node that leaves the module's nodes while it failed alone is cleared once; the first pass after a start clears every
// node the presence bucket knows, for every module, failed have or not (a node that recovered unwatched leaves none).
conds = nil
p.sayNodeFailed("rofi", fpTest, []string{"shanks"}, assign.Nodes{}, assign.Outcome{NodeFailed: failing}, false)
p.sayNodeFailed("rofi", fpTest, nil, assign.Nodes{}, assign.Outcome{}, false)
p.sayNodeFailed("rofi", fpTest, nil, assign.Nodes{}, assign.Outcome{}, false)
if len(conds) != 2 || conds[1].State != asks.StateCleared || conds[1].Key != "mesh-delivery-assigner.failed-alone.shanks.rofi" {
t.Errorf("a node leaving the module said %+v, want one raise and one clear", conds)
}
known := assign.Nodes{Online: map[string]bool{"ace": true, "g14": false},
Haves: map[string]map[string]delivery.Have{"g14": {"rofi": {FP: fpTest, Health: delivery.HealthFailed}}}}
for _, c := range []struct {
name string
known assign.Nodes
keys []string
}{
{"a failed have", known, []string{"mesh-delivery-assigner.failed-alone.ace.rofi", "mesh-delivery-assigner.failed-alone.g14.rofi"}},
{"no failed have", assign.Nodes{Online: known.Online}, []string{"mesh-delivery-assigner.failed-alone.ace.rofi", "mesh-delivery-assigner.failed-alone.g14.rofi"}},
} {
conds = nil
fresh := newPasser(buckets{}, time.Date(2026, 10, 26, 22, 0, 0, 0, time.UTC))
fresh.logf = func(string, ...any) {}
fresh.say = func(c asks.Condition) error { conds = append(conds, c); return nil }
fresh.sayNodeFailed("rofi", fpTest, nil, c.known, assign.Outcome{}, true)
var keys []string
for _, cond := range conds {
if cond.State == asks.StateCleared {
keys = append(keys, cond.Key)
}
}
if !slices.Equal(keys, c.keys) || len(conds) != len(c.keys) {
t.Errorf("first pass, %s: said %+v, want clears of %q", c.name, conds, c.keys)
}
}
}
// A neither or held condition a previous process raised is cleared by the first pass after a start: a fresh pass remembers
// nothing it said, so it clears, once, the neither condition of every assigned module not on neither list and the held condition of
// every module that is not held. Clearing a condition that is not open is quiet at the router.
func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) {
t.Run("a held module", testAHeldModuleRaisesNoNeitherAndClearsAStandingOne)
wantsKV := &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerWantsBucket: wantsKV, delivery.AssignerAsksBucket: &memKV{values: map[string][]byte{}}}, time.Now())
var said []asks.Condition
@@ -548,7 +449,7 @@ func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) {
// 1945/AC24: given a module held at F under rule 2 on a node that runs it with no want, when the assigner passes, then it raises no
// neither condition for it, a neither condition this process raised for it before is cleared, and so is one a previous process
// raised (the first pass after a start), each once.
func testAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
func TestAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
setup := func() (*passer, *[]asks.Condition) {
p := newPasser(buckets{delivery.AssignerWantsBucket: &memKV{values: map[string][]byte{}}, delivery.AssignerAsksBucket: &memKV{values: map[string][]byte{}}}, time.Now())
said := &[]asks.Condition{}
@@ -764,8 +665,6 @@ func mustKey(key string, err error) string {
// (ADR 0545). That record ask is not about the trunk's fingerprint: it survives every pass, so the operator is asked, and on a
// yes the want names the revised digest.
func TestARecordAskOfAHeldBackNodeSurvivesPassesAndItsYesMovesTheWant(t *testing.T) {
// A placed node's yes comes from its have (issue 1126): folded here to hold the count.
aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t)
fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64)
revised, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64)
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
@@ -814,133 +713,3 @@ func TestARecordAskOfAHeldBackNodeSurvivesPassesAndItsYesMovesTheWant(t *testing
t.Error("the answered record ask was taken away")
}
}
// 1126 (ADR 0549 §5): until the third milestone a pair the controller's assignments and the catalogue's placement do not
// both name gets no want and one condition, raised once and cleared once both name it; before any placement was read the
// assignments are the one source and nothing is raised.
func aPairOneSourceNamesAloneGetsNoWantAndOneCondition(t *testing.T) {
p := newPasser(buckets{}, time.Date(2026, 10, 26, 18, 0, 0, 0, time.UTC))
var said []asks.Condition
p.say = func(c asks.Condition) error { said = append(said, c); return nil }
assigned := func(node, module string) {
raw, _ := json.Marshal(map[string]string{"node": node, "module": module})
p.w.put(provider.AssignmentsBucket, node+"."+module, raw)
}
place := func(node string, modules ...string) {
pl := delivery.Placement{Node: node}
for _, m := range modules {
pl.Modules = append(pl.Modules, delivery.PlacedModule{Module: m})
}
raw, err := delivery.EncodePlacement(pl)
if err != nil {
t.Fatal(err)
}
key, _ := delivery.PlacementKey(node)
p.w.put(delivery.PlacementBucket, key, raw)
}
assigned("g14", "a")
assigned("g14", "b")
if got, _ := p.w.assigned(); !slices.Equal(got["g14"], []string{"a", "b"}) || len(p.sayPlacement(p.w.placementDiffers())) != 0 || len(said) != 0 {
t.Fatalf("before any placement: assigned %v, said %v", got, said)
}
none := func(context.Context, string) ([]byte, error) { return nil, errNotThere }
if m := p.w.mesh(context.Background(), none); m.Input != nil || m.InputUnread != "" {
t.Errorf("before any placement the check composes over %+v (%q)", m.Input, m.InputUnread)
}
place("g14", "a", "c")
got, _ := p.w.assigned()
frozen := p.sayPlacement(p.w.placementDiffers())
p.sayPlacement(p.w.placementDiffers())
if !slices.Equal(got["g14"], []string{"a"}) || !frozen["g14.b"] || !frozen["g14.c"] || frozen["g14.a"] || len(said) != 2 ||
said[0].State != asks.StateRaised || !strings.Contains(said[0].Explanation, "the controller's assignments names b on g14") ||
!strings.Contains(said[1].Explanation, "the catalogue's placement names c on g14") {
t.Fatalf("one source alone: assigned %v, frozen %v, said %+v", got, frozen, said)
}
place("g14", "a", "b", "c")
assigned("g14", "c")
if frozen := p.sayPlacement(p.w.placementDiffers()); len(frozen) != 0 || len(said) != 4 || said[2].State != asks.StateCleared || said[3].State != asks.StateCleared {
t.Errorf("both agree again: frozen %v, said %+v", frozen, said)
}
// 1126 (ADR 0547 §2): placed, the check composes over the input the nodes compose from (frombus), once each placed node
// wrote its record; until then it says why and composes the assignments alone.
if m := p.w.mesh(context.Background(), none); m.Input != nil || !strings.Contains(m.InputUnread, "g14") {
t.Errorf("a placed node with no record: input %+v, unread %q", m.Input, m.InputUnread)
}
rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "g14", TunnelKey: "k"}})
p.w.put(delivery.HaveBucket, "g14.node", rec)
m := p.w.mesh(context.Background(), none)
if m.Input == nil || len(m.Input.Nodes) != 1 || !slices.Equal(m.Input.Nodes[0].Assigned, []string{"a", "b", "c"}) || m.Input.Nodes[0].Place.Key != "k" {
t.Errorf("the check's input from the bus: %+v (%q)", m.Input, m.InputUnread)
}
}
// 1126 (ADR 0547 §2 to §4): a node the catalogue places composes its own: its wants name the build alone and its own want
// names no composition; a have waiting for a yes raises the record ask once, and the yes is written into the want and
// kept there while the want stays at its build.
func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) {
fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64)
waiting, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64)
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
p.logf = t.Logf
put := func(bucket, key string, v []byte) { p.w.put(bucket, key, v) }
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
put(delivery.ModulesBucket, "web", mod) // the trunk's build is not done: rule 1 holds g14 at fpOld
put(provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-1"}`))
pl, _ := delivery.EncodePlacement(delivery.Placement{Node: "g14", Modules: []delivery.PlacedModule{{Module: "web"}}})
put(delivery.PlacementBucket, "placement.g14", pl)
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
put(delivery.PresenceBucket, "g14", on)
have, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: was, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
put(delivery.HaveBucket, "g14.web", have)
wkey, _ := delivery.WantKey("g14", "web")
want, _ := delivery.EncodeWant(delivery.Want{FP: fpOld, Composed: was})
wantsKV.values[wkey] = want
put(delivery.AssignerWantsBucket, wkey, want)
sync := func() {
for k, v := range wantsKV.values {
put(delivery.AssignerWantsBucket, k, v)
}
}
p.wants(context.Background())
p.nodeWants(context.Background())
sync()
got, err := delivery.DecodeWant(wantsKV.values[wkey])
if err != nil || got.FP != fpOld || got.Composed != "" || !got.Self {
t.Fatalf("the placed node's want is %+v (%v), want it at %s, the node composing it", got, err, fpOld)
}
nkey, _ := delivery.WantNodeKey("g14")
if nw, err := delivery.DecodeNodeWant(wantsKV.values[nkey]); err != nil || nw.Composed != "" || !nw.Self {
t.Fatalf("the placed node's own want is %+v (%v), want one naming no composition", nw, err)
}
waits, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: was, Phase: delivery.PhaseWaitingYes, Health: delivery.HealthOK,
Judgement: delivery.JudgedRecreates, Waits: waiting, Detail: "it recreates web.server"})
put(delivery.HaveBucket, "g14.web", waits)
akey, _ := delivery.RecordAskKey("g14", "web", waiting)
p.wants(context.Background())
if _, there := asksKV.values[akey]; !there {
t.Fatal("a have waiting for a yes raised no record ask")
}
a, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
if err != nil {
t.Fatal(err)
}
put(delivery.AssignerAsksBucket, akey, asksKV.values[akey])
yes, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a.Ask.ID + "@2026-10-26T19:00:00Z", Words: "the operator"})
put(delivery.AnswersBucket, akey, yes)
p.wants(context.Background())
sync()
if got, err = delivery.DecodeWant(wantsKV.values[wkey]); err != nil || got.Yes != waiting || got.FP != fpOld {
t.Fatalf("after the yes the want is %+v (%v), want it to name the yes %s", got, err, waiting)
}
// The node applied it: its have runs the composition; the yes stays while the want stays at its build.
ran, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: waiting, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
put(delivery.HaveBucket, "g14.web", ran)
p.wants(context.Background())
if got, _ = delivery.DecodeWant(wantsKV.values[wkey]); got.Yes != waiting {
t.Errorf("a pass after the move dropped the yes: %+v", got)
}
}
@@ -14,8 +14,6 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/compose"
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provider"
@@ -48,14 +46,6 @@ type world struct {
healthy map[string][]string
// changes counts, by bucket and key, the values this process has seen the key take: a value written again is another.
changes map[string]uint64
// meta is, by bucket and key, the revision and the time of the value held now, where the watch said them (putEntry).
meta map[string]entryMeta
}
// entryMeta is a value's revision in its bucket and when it was written.
type entryMeta struct {
revision uint64
at time.Time
}
// parsed is a manifest read by digest: its module and its manifest, or why it cannot be read.
@@ -66,27 +56,7 @@ type parsed struct {
}
func newWorld() *world {
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{},
meta: map[string]entryMeta{}}
}
// putEntry is put, keeping the value's revision and the time it was written.
func (w *world) putEntry(bucket, key string, value []byte, revision uint64, at time.Time) {
w.put(bucket, key, value)
w.mu.Lock()
defer w.mu.Unlock()
if value == nil {
delete(w.meta, bucket+"|"+key)
return
}
w.meta[bucket+"|"+key] = entryMeta{revision: revision, at: at}
}
// metaOf is the revision and time of the value a bucket holds at key, as the watch said them.
func (w *world) metaOf(bucket, key string) entryMeta {
w.mu.Lock()
defer w.mu.Unlock()
return w.meta[bucket+"|"+key]
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{}}
}
// put takes a change of a watched bucket in: a value, or nil for a key deleted or purged.
@@ -192,56 +162,8 @@ func (w *world) manifest(ctx context.Context, get getter, digest string) parsed
return p
}
// assigned is each node's modules both the controller's assignments and the indexer's placement name, in name order
// (ADR 0549 §5; the assignments alone before any placement was read), with every open assignment by node and module.
// assigned is each node's modules with an open assignment, read from the controller's assignments, in name order.
func (w *world) assigned() (map[string][]string, map[string]assignment) {
byAssignment, all := w.assignedByController()
placed, seeded := w.placed()
both, _ := assign.Placed(byAssignment, placed, seeded)
return both, all
}
// placementDiffers is every pair the controller's assignments and the indexer's placement do not both name.
func (w *world) placementDiffers() []assign.Pair {
byAssignment, _ := w.assignedByController()
placed, seeded := w.placed()
_, differ := assign.Placed(byAssignment, placed, seeded)
return differ
}
// placed is each node's modules in the indexer's placement (ADR 0549 §2), and whether any placement key was read: a key
// that does not read places nothing on its node and is said by the indexer's own read tool.
func (w *world) placed() (map[string][]string, bool) {
out := map[string][]string{}
values := w.read(delivery.PlacementBucket)
for key, value := range values {
node, err := delivery.ParsePlacementKey(key)
if err != nil {
continue
}
p, err := delivery.DecodePlacement(value)
if err != nil || p.Node != node {
continue
}
out[node] = p.Assigned()
}
return out, len(values) > 0
}
// placedNodes is every node the indexer's placement names (ADR 0549 §2): those compose their own modules (ADR 0547 §2),
// so their wants name the build alone and the seventh rule does not hold them.
func (w *world) placedNodes() map[string]bool {
placed, _ := w.placed()
out := make(map[string]bool, len(placed))
for node := range placed {
out[node] = true
}
return out
}
// assignedByController is each node's modules with an open assignment, read from the controller's assignments, in
// name order.
func (w *world) assignedByController() (map[string][]string, map[string]assignment) {
out, all := map[string][]string{}, map[string]assignment{}
for _, value := range w.read(provider.AssignmentsBucket) {
var a assignment
@@ -288,100 +210,9 @@ func (w *world) mesh(ctx context.Context, manifests getter) assign.Mesh {
}
m.Manifests[name] = p.manifest
}
// Once the catalogue places the nodes, the check composes over the input they compose from (ADR 0547 §2, ADR 0549
// §3): one composer's input for the check and the nodes. An input that cannot be read is said, and the nodes and their
// assignments alone are composed, as before any placement.
if _, seeded := w.placed(); seeded {
in, err := frombus.Input(ctx, worldReader{w: w, manifests: manifests}, frombus.Mesh{}, frombus.Scope{Every: true})
if err != nil {
m.InputUnread = err.Error()
} else {
m.Input = &in
}
}
return m
}
// worldReader is what the assigner watched as frombus reads the bus: the indexer's placement and modules, each node's
// record in its own have key, the manifests by digest; the check reads no config value, own secret or credential (the
// composition it asks resolves, and stands in for what it does not carry).
type worldReader struct {
w *world
manifests getter
}
func (r worldReader) Placements(context.Context) (map[string]delivery.Placement, error) {
out := map[string]delivery.Placement{}
for key, value := range r.w.read(delivery.PlacementBucket) {
node, err := delivery.ParsePlacementKey(key)
if err != nil {
continue
}
if p, err := delivery.DecodePlacement(value); err == nil && p.Node == node {
out[node] = p
}
}
return out, nil
}
func (r worldReader) Records(context.Context) (map[string]delivery.NodeRecord, error) {
out := map[string]delivery.NodeRecord{}
for key, value := range r.w.read(delivery.HaveBucket) {
node, _, isNode, err := delivery.ParseHaveKey(key)
if err != nil || !isNode {
continue
}
if h, err := delivery.DecodeHaveNode(value); err == nil && h.Record != nil {
out[node] = *h.Record
}
}
return out, nil
}
func (r worldReader) Modules(context.Context) (map[string]delivery.Module, error) {
return r.w.trunk(), nil
}
func (r worldReader) Manifest(ctx context.Context, digest string) ([]byte, error) {
key, err := delivery.ManifestsKey(digest)
if err != nil {
return nil, err
}
raw, err := r.manifests(ctx, key)
if err != nil {
return nil, err
}
m, err := delivery.DecodeManifest(raw)
if err != nil {
return nil, err
}
return []byte(m.JSON), nil
}
func (worldReader) Values(context.Context, string, string) (map[string]any, error) { return nil, nil }
func (worldReader) OwnSecrets(context.Context, string) ([]compose.OwnSecretRow, error) {
return nil, nil
}
func (worldReader) Credentials(context.Context, string, []compose.CredentialKey, []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error) {
return nil, nil, nil
}
// Build, Archive and Served are read only for a node composing itself (frombus.Scope without Every): the check of every
// node composes from the manifests the indexer read, reads no build and no archive, and reads no config value.
func (worldReader) Build(context.Context, string, string) (*delivery.BuilderBuild, error) {
return nil, nil
}
func (worldReader) Archive(_ context.Context, url, _ string) ([]byte, error) {
return nil, fmt.Errorf("the composition check reads no archive (%s)", url)
}
func (worldReader) Served(context.Context, string, string, []string) (map[string]any, error) {
return nil, nil
}
// change is a candidate's change for the composition check, or why the composition waits for it.
func (w *world) change(ctx context.Context, manifests getter, c delivery.Candidate, trunk map[string]delivery.Module) (assign.Change, string) {
ch := assign.Change{Group: c.Group, Heads: c.HeadsHash, Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{},
@@ -431,92 +262,9 @@ func (w *world) change(ctx context.Context, manifests getter, c delivery.Candida
sort.Strings(ch.Twice[name])
}
}
cp, waits := w.placementOf(ctx, manifests, c)
if waits != "" {
return ch, waits
}
ch.Placement = cp
return ch, ""
}
// placementOf is the change's own placement (ADR 0559 §5): the indexer's placement candidates of the group's catalogue
// pull request, with the input the nodes compose from with them laid over the trunk's placement. None when the group has
// no catalogue pull request or it changes no placement file. It waits while a key names another head than the group's.
func (w *world) placementOf(ctx context.Context, manifests getter, c delivery.Candidate) (*assign.ChangePlacement, string) {
var pull *delivery.CandidatePull
for i := range c.Pulls {
if delivery.SameRepository(c.Pulls[i].Repository, delivery.CatalogueRepository) {
pull = &c.Pulls[i]
}
}
if pull == nil {
return nil, ""
}
cp := &assign.ChangePlacement{Placements: map[string]*delivery.Placement{}, Revisions: map[string]uint64{}, Head: pull.Head.Head}
for key, value := range w.read(delivery.PlacementCandidatesBucket) {
group, node, err := delivery.ParsePlacementCandidateKey(key)
if err != nil || group != c.Group {
continue
}
pc, err := delivery.DecodePlacementCandidate(value)
if err != nil || pc.Pull.Number != pull.Number || !delivery.SameRepository(pc.Pull.Repository, pull.Repository) {
continue
}
if pc.Pull.Head != pull.Head.Head {
return nil, fmt.Sprintf("the indexer holds the placement candidates of %s#%d at %s, not yet at its head %s", pull.Repository, pull.Number, pc.Pull.Head, pull.Head.Head)
}
meta := w.metaOf(delivery.PlacementCandidatesBucket, key)
cp.Placements[node], cp.Revisions[node] = pc.Placement, meta.revision
if meta.at.After(cp.Written) {
cp.Written = meta.at
}
}
if len(cp.Placements) == 0 {
return nil, ""
}
in, err := frombus.Input(ctx, worldReader{w: w, manifests: manifests}, frombus.Mesh{}, frombus.Scope{Every: true, Placements: cp.Placements})
if err != nil {
cp.Unread = err.Error()
} else {
cp.Input = &in
}
return cp, ""
}
// candidateAnswers is, by node, each node's answer to the group's placement candidates in its `<node>.node` have.
func (w *world) candidateAnswers(group string) map[string]*delivery.CandidateAnswer {
out := map[string]*delivery.CandidateAnswer{}
for key, value := range w.read(delivery.HaveBucket) {
node, _, isNode, err := delivery.ParseHaveKey(key)
if err != nil || !isNode {
continue
}
h, err := delivery.DecodeHaveNode(value)
if err != nil {
continue
}
if a, ok := h.Candidates[group]; ok {
out[node] = &a
}
}
return out
}
// online is, by node, whether the watchman holds the node online.
func (w *world) online() map[string]bool {
out := map[string]bool{}
for key, value := range w.read(delivery.PresenceBucket) {
node, _, _, err := delivery.ParsePresenceKey(key)
if err != nil || node == "" {
continue
}
if p, err := delivery.DecodePresence(value); err == nil {
out[node] = p.Online
}
}
return out
}
// wantsOf is the wants the assigner wrote, by module then node.
func (w *world) wantsOf() map[string]map[string]delivery.Want {
out := map[string]map[string]delivery.Want{}
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027201213-7a8e9f970648
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+10 -4
View File
@@ -1,7 +1,13 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952 h1:TmveT3Bija643RGWkilvkzP/h8UWlVvHFZZtQfJK5oU=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027201213-7a8e9f970648 h1:XoMsgg9kSgjQWa86NZM8h8OpxJ2kovxcMS2C0i6WSVU=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027201213-7a8e9f970648/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025195051-20a6dc4920f8 h1:jA7V7hlfJAigVUvJDpKwfgd9ylQ9bqppg3rK9XgLWoA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025195051-20a6dc4920f8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8 h1:vhYk/iv43u2WGSTrFyQGDvGBxLaRITgMAiiI9DjuA10=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 h1:gE80bR0tPME7MTMBLcdP62v+F/9soVXmdcBYNMOye44=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -81,29 +81,6 @@ func HeldCondition(module, fp, node, words string, raised, needsYou bool) asks.C
return c
}
// NodeFailedCondition is the condition the assigner raises for a module whose newest fingerprint fp failed its health check on
// node while other nodes run it healthy (ADR 0557 §1, issue 1496): the failure is the node's, so nothing is held or set back and
// the node keeps its want. It names the node, the module, fp and the have's words, keyed by node and module, and is cleared
// (raised false) once the node no longer fails alone.
func NodeFailedCondition(node, module, fp string, f NodeFailure, raised bool) asks.Condition {
key := delivery.AssignerModule + ".failed-alone." + node + "." + module
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
if !raised {
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared,
Resolved: fmt.Sprintf("%s on %s no longer fails its health check alone.", module, node)}
}
headline := fmt.Sprintf("%s fails on %s alone", module, node)
if n := len([]rune(headline)); n > asks.HeadlineLength {
headline = string([]rune(headline)[:asks.HeadlineLength])
}
return asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
Summary: headline, Headline: headline,
Explanation: fmt.Sprintf("%s The health check of %s at %s failed on %s (%s), while it runs healthy on %s. The failure is %s's, "+
"so the assigner holds nothing and sets no node back, and %s keeps its want. This clears once %s runs it healthy or "+
"another version is the newest.", asks.NothingToDo, module, short(fp), node, f.Words, strings.Join(f.Others, ", "), node,
node, node)}
}
// NeitherCondition is the condition of a module assigned to node that holds no want for it (issue 1945): the node's own composition
// gives up every module with a fingerprint of the new form, so nothing applies it there and its files are gone. It is raised (urgent)
// naming the module, the node and the rule holding its first want back with the rule's words, under the assigner's own name keyed by
@@ -119,14 +96,6 @@ func NeitherCondition(node, module string, h Holding, raised bool) asks.Conditio
if n := len([]rune(headline)); n > asks.HeadlineLength {
headline = string([]rune(headline)[:asks.HeadlineLength])
}
if h.Bus {
needs := fmt.Sprintf("put %s on %s with assigner_put", module, node)
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
Summary: headline, Headline: headline, Needs: needs,
Explanation: fmt.Sprintf("%s%s. %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
"hold it either, so nothing applies it there. The assigner never moves the bus: %s. This clears once the node has a want for it.",
asks.NeedsYou, needs, module, node, h.Words)}
}
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
Summary: headline, Headline: headline,
Explanation: fmt.Sprintf("%s %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
@@ -134,17 +103,6 @@ func NeitherCondition(node, module string, h Holding, raised bool) asks.Conditio
"rule holds, and clears this.", asks.NothingToDo, module, node, h.Rule, h.Words)}
}
// NeitherLeftCondition clears the neither condition of module on node when it is no longer on neither list: why is "" when the
// node has a live want for it again, else why it has none (it is held, its move waits for an ask, the node no longer runs it), so
// the clear never says a want the node does not hold (issue 1945: "nats on novox has a want again" while it had none).
func NeitherLeftCondition(node, module, why string) asks.Condition {
c := NeitherCondition(node, module, Holding{}, false)
if why != "" {
c.Resolved = fmt.Sprintf("%s on %s is no longer on neither list and has no want: %s.", module, node, why)
}
return c
}
// LiftedCondition clears the condition of a held fingerprint the assigner lifted by itself, saying why (issue 1906).
func LiftedCondition(module, fp string) asks.Condition {
c := HeldCondition(module, fp, "", "", false, false)
@@ -1,242 +0,0 @@
package assign
import (
"fmt"
"reflect"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/compose"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// A change's own placement (novox/hq ADR 0559 §5, issue 1126): a catalogue pull request that changes placement files has
// the indexer's placement candidates, one key `<group>.<node>` for each node whose file at its head differs from the
// trunk's. The check composes the change with those placements laid over the trunk's (frombus, Scope.Placements), counts
// every such node among the machines it touches, and then reads each placed node's own answer in its `<node>.node`
// have: it fails on what a node composes differently that the change does not intend, on a refusal, and on a node
// placed at the head or on the trunk that has not answered at the head within AnswerBound of the keys being written.
// AnswerBound is how long the check waits for every placed node's answer after the candidate keys were written (ADR 0559
// §5): a node asleep or away is named past it, never passed. A variable so a test on the bus can shorten it.
var AnswerBound = 10 * time.Minute
// ChangePlacement is the change's own placement, read from the indexer's placement candidates of its group.
type ChangePlacement struct {
// Placements is by node the candidate placement; nil when the head removes the node's file.
Placements map[string]*delivery.Placement
// Revisions is by node the revision of its candidate key: the answer to it names that revision.
Revisions map[string]uint64
// Head is the catalogue pull request's head the candidates were read at, and Written when the newest key was written.
Head string
Written time.Time
// Input is the trunk's composition input with Placements laid over it (frombus.Input, Scope.Placements), its catalogue
// the trunk's; nil with Unread saying why when it could not be read.
Input *compose.Input
Unread string
}
// Nodes is every node the change's placement names, sorted.
func (p *ChangePlacement) Nodes() []string {
if p == nil {
return nil
}
out := make([]string, 0, len(p.Placements))
for node := range p.Placements {
out = append(out, node)
}
sort.Strings(out)
return out
}
// Intended is, by node the trunk places, the modules whose composition the change's placement changes: those it adds,
// removes or whose placed entry differs, and those whose composition (with the trunk's catalogue, the same stand-ins
// either side) differs between the trunk's input and the change's (delivery.CompareComposed), the node's own named
// "node". A node the trunk does not place yet has none (ADR 0559 §5: at the seed every node must answer the same). fps
// gives the build each module is composed at, the trunk's. A node whose compositions cannot be made either side keeps
// the modules its placement entries name, and is said in notes.
func Intended(trunk *compose.Input, trunkPlaced map[string]delivery.Placement, cp *ChangePlacement, fps map[string]string) (intended map[string]map[string]bool, notes []string) {
intended = map[string]map[string]bool{}
if cp == nil {
return intended, nil
}
for _, node := range cp.Nodes() {
before, placed := trunkPlaced[node]
if !placed {
continue
}
set := map[string]bool{}
after := cp.Placements[node]
var afterModules []delivery.PlacedModule
if after != nil {
afterModules = after.Modules
}
entries := map[string]delivery.PlacedModule{}
for _, m := range before.Modules {
entries[m.Module] = m
}
seen := map[string]bool{}
for _, m := range afterModules {
seen[m.Module] = true
if e, ok := entries[m.Module]; !ok || !reflect.DeepEqual(e, m) {
set[m.Module] = true
}
}
for name := range entries {
if !seen[name] {
set[name] = true
}
}
if trunk != nil && cp.Input != nil {
was, _, _, errB := compose.CompositionsShadow(*trunk, node, fps)
now, _, _, errA := compose.CompositionsShadow(*cp.Input, node, fps)
if errA == nil && errB == nil {
running := make([]delivery.Composed, 0, len(was))
for _, c := range was {
running = append(running, c)
}
cmp := delivery.CompareComposed(now, running)
for _, d := range cmp.Differs {
set[d.Module] = true
}
for _, d := range cmp.Added {
set[d.Module] = true
}
for _, m := range cmp.Removed {
set[m] = true
}
} else {
notes = append(notes, fmt.Sprintf("what the change intends on %s is read from its placement file alone: its compositions cannot be made (trunk: %v; change: %v)", node, errB, errA))
}
}
intended[node] = set
}
return intended, notes
}
// Answered is what the placed nodes answered: the failures to name, the nodes that have not answered, and the notes.
type Answered struct {
Fails []string
Silent []string
Machine string
Waits bool
Until time.Time
Warnings []string
}
// JudgeAnswers reads each placed node's answer to the change's placement (ADR 0559 §5). placed is every node placed at
// the head or on the trunk; answers is by node its HaveNode.Candidates entry for the group, nil when it has none; online
// says whether the watchman holds the node online. An answer counts when it names the candidates' head and, for a node
// with a key of its own, that key's revision. It fails on a refusal, on a module that differs, is added or is removed
// that intended does not name, and, from Written plus AnswerBound on, on every node with no answer that counts, named,
// asleep or not; before that the check waits for them.
func JudgeAnswers(cp *ChangePlacement, placed []string, answers map[string]*delivery.CandidateAnswer, intended map[string]map[string]bool, online map[string]bool, now time.Time) Answered {
var out Answered
if cp == nil {
return out
}
failAt := func(node, words string) {
if out.Machine == "" {
out.Machine = node
}
out.Fails = append(out.Fails, words)
}
for _, node := range placed {
a := answers[node]
rev, keyed := cp.Revisions[node]
if a == nil || a.Head != cp.Head || (keyed && a.Revision != rev) {
out.Silent = append(out.Silent, node)
continue
}
if a.Refused != "" {
failAt(node, fmt.Sprintf("%s cannot compose the candidate placement: %s", node, a.Refused))
continue
}
for _, r := range a.Refusals {
failAt(node, fmt.Sprintf("%s refuses the candidate placement for %s: %s", node, r.Module, r.Why))
}
want := intended[node]
for _, d := range a.Differs {
if !want[d.Module] {
failAt(node, fmt.Sprintf("%s composes %s differently with the candidate placement, which the change does not intend: %s", node, d.Module, d.Why))
}
}
for _, d := range a.Added {
if !want[d.Module] {
failAt(node, fmt.Sprintf("%s would run %s with the candidate placement, which the change does not intend", node, d.Module))
}
}
for _, m := range a.Removed {
if !want[m] {
failAt(node, fmt.Sprintf("%s would stop running %s with the candidate placement, which the change does not intend", node, m))
}
}
}
if len(out.Silent) == 0 {
return out
}
out.Until = cp.Written.Add(AnswerBound)
named := make([]string, len(out.Silent))
for i, node := range out.Silent {
named[i] = node
if !online[node] {
named[i] = node + " (offline)"
}
}
if now.Before(out.Until) {
out.Waits = true
out.Warnings = append(out.Warnings, fmt.Sprintf("waits for the answers of %s to the candidate placement at %s, until %s",
strings.Join(named, ", "), short(cp.Head), out.Until.UTC().Format(time.RFC3339)))
return out
}
for _, node := range out.Silent {
if out.Machine == "" {
out.Machine = node
}
}
out.Fails = append(out.Fails, fmt.Sprintf("%s %s (%s after the candidate keys were written at %s)",
strings.Join(named, ", "), delivery.CompositionUnanswered, AnswerBound, cp.Written.UTC().Format(time.RFC3339)))
return out
}
// PlacedNodes is every node placed at the head or on the trunk: the trunk's placed nodes with the change's placement laid
// over them (a node whose file the head removes stays, as the trunk places it), sorted.
func PlacedNodes(trunkPlaced map[string]delivery.Placement, cp *ChangePlacement) []string {
set := map[string]bool{}
for node := range trunkPlaced {
set[node] = true
}
if cp != nil {
for node, p := range cp.Placements {
if p != nil {
set[node] = true
}
}
}
out := make([]string, 0, len(set))
for node := range set {
out = append(out, node)
}
sort.Strings(out)
return out
}
// WithAnswers is v with the placed nodes' answers judged in: a fail adds its failures (the words of nodes not heard end
// with delivery.CompositionUnanswered, so the checker takes a later composition of the same heads in when they answer), a
// wait makes a passing verdict wait (a composition that waits composed no machine), and a failing one stands.
func WithAnswers(v delivery.CompositionVerdict, a Answered) delivery.CompositionVerdict {
if len(a.Fails) > 0 {
if v.State == delivery.CompositionFail {
v.Words += "; " + strings.Join(a.Fails, "; ")
} else {
v.State, v.Words, v.Machine = delivery.CompositionFail, strings.Join(a.Fails, "; "), a.Machine
}
return v
}
if a.Waits && v.State == delivery.CompositionPass {
return delivery.CompositionVerdict{Group: v.Group, Heads: v.Heads, State: delivery.CompositionWaits, SDK: v.SDK, Took: v.Took,
Words: strings.Join(a.Warnings, "; ")}
}
return v
}
@@ -35,13 +35,6 @@ type Mesh struct {
// Digests is every trunk module's manifest digest in the indexer's manifests bucket, readable or not: what a
// composer at another mesh-sdk reads (issue 1622).
Digests map[string]string
// Input is the rest of the mesh as the nodes compose from it, filled by mesh-sdk's frombus from the catalogue's
// placement and the nodes' records (novox/hq ADR 0547 §2, ADR 0549 §3, issue 1126), so the check and the nodes share
// one composer's input; nil before any placement is read, when the nodes and their assignments alone are composed.
Input *compose.Input
// InputUnread says why that input could not be read from the bus, when it could not: the check then composes the
// nodes and their assignments alone, and says so in its warnings.
InputUnread string
}
// Change is a group's change as the indexer's candidate holds it: the manifest of every module it moves or adds, those
@@ -59,15 +52,12 @@ type Change struct {
// Members the head of each member by repository: what a composer at the group's mesh-sdk reads (issues 1622, 1623).
Digests map[string]string
Members map[string]string
// Placement is the change's own placement, read from the indexer's placement candidates of the group (ADR 0559 §5);
// nil when the group changes no placement file.
Placement *ChangePlacement
}
// Touched is the machines a change touches (ADR 0446 clause 3): those running a module the change moves, adds or
// removes, those running a consumer of a provision such a module provides (before or after), and those running a
// provider of a provision such a module requires after the change. A module the change adds that is assigned nowhere
// touches no machine. Every node whose placement the change changes is touched too (ADR 0559 §5, changing ADR 0446 §3).
// touches no machine.
func Touched(m Mesh, c Change) []string {
moved := map[string]bool{}
for name := range c.Manifests {
@@ -101,11 +91,6 @@ func Touched(m Mesh, c Change) []string {
}
}
}
for _, node := range c.Placement.Nodes() {
if !slices.Contains(out, node) {
out = append(out, node)
}
}
sort.Strings(out)
return out
}
@@ -178,25 +163,6 @@ func Compose(m Mesh, c Change, sdk string, took time.Duration) delivery.Composit
delete(changed, name)
}
after := input(changed, m.Assigned)
if c.Placement != nil {
// A change of placement is composed over the input the nodes compose from, its placement laid over the trunk's
// (frombus, ADR 0559 §5): the trunk as the nodes compose it where the catalogue places them, the change's
// manifests over the trunk's catalogue.
if m.Input != nil {
trunk = *m.Input
trunk.Catalogue = withOverlay(m.Input.Catalogue, m.Manifests)
}
if c.Placement.Input == nil {
failAt("", "the change's placement cannot be composed: the input with it cannot be read from the bus: "+c.Placement.Unread)
} else {
after = *c.Placement.Input
cat := withOverlay(c.Placement.Input.Catalogue, changed)
for _, name := range c.Removed {
delete(cat, name)
}
after.Catalogue = cat
}
}
for _, node := range machines {
_, before := compose.Resolve(trunk, node)
_, now := compose.Resolve(after, node)
@@ -306,18 +272,6 @@ func GrantsAlone(m Mesh, c Change, composed []string) []string {
return out
}
// withOverlay is base with every manifest of over laid over it, a copy.
func withOverlay(base, over map[string]catalogue.Manifest) map[string]catalogue.Manifest {
out := make(map[string]catalogue.Manifest, len(base)+len(over))
for name, man := range base {
out[name] = man
}
for name, man := range over {
out[name] = man
}
return out
}
func sortedKeys[V any](m map[string]V) []string {
out := make([]string, 0, len(m))
for k := range m {
@@ -4,7 +4,6 @@ import (
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
@@ -58,79 +57,6 @@ func TestAMachineTheChangeBreaksFails(t *testing.T) {
if v.State != delivery.CompositionFail || v.Machine != "ace" || !strings.Contains(v.Words, "database") {
t.Fatalf("composition = %+v, want a fail on ace naming the provision nothing provides", v)
}
// ADR 0559 §5 (issue 1126): a change of placement composes its placement laid over the trunk's input and fails naming
// the node it breaks: a module no manifest names placed on g14; ace,
// is no longer composed with it; a placement whose input cannot be read fails; a placement that breaks nothing passes, g14 touched.
m := meshOf()
placed := func(assigned map[string][]string) *ChangePlacement {
in := input(m.Manifests, assigned)
return &ChangePlacement{Placements: map[string]*delivery.Placement{"g14": {Node: "g14"}}, Input: &in}
}
for _, row := range []struct {
name string
cp *ChangePlacement
state string
machine string
words string
}{
{"a placement that breaks g14", placed(map[string][]string{"ace": {"mail", "store"}, "g14": {"web", "ghost"}}), delivery.CompositionFail, "g14", "ghost"},
{"a placement whose input is not read", &ChangePlacement{Placements: map[string]*delivery.Placement{"g14": nil}, Unread: "no record of g14"}, delivery.CompositionFail, "", "no record of g14"},
{"a placement that breaks nothing", placed(map[string][]string{"ace": {"mail", "store"}, "g14": {"web"}}), delivery.CompositionPass, "", ""},
} {
v := Compose(m, Change{Group: "g", Heads: "0123456789ab", Placement: row.cp}, "sdk", 0)
if v.State != row.state || v.Machine != row.machine || !strings.Contains(v.Words, row.words) || !slices.Contains(v.Machines, "g14") {
t.Errorf("%s: %+v", row.name, v)
}
}
// ADR 0559 §5: the placed nodes' answers. ace is placed on the trunk and its candidate moves web there (intended);
// g14 is not placed on the trunk (the seed), so nothing is intended there; novox is placed and unchanged.
at := time.Date(2026, 10, 27, 2, 0, 0, 0, time.UTC)
cp := &ChangePlacement{Head: "h2", Written: at, Revisions: map[string]uint64{"ace": 7, "g14": 8},
Placements: map[string]*delivery.Placement{"ace": {Node: "ace"}, "g14": {Node: "g14"}}}
intended := map[string]map[string]bool{"ace": {"web": true}}
same := func(rev uint64) *delivery.CandidateAnswer {
return &delivery.CandidateAnswer{Head: "h2", Revision: rev, Verdict: delivery.CandidateSame, Same: []string{"mail"}}
}
placedNodes := []string{"ace", "g14", "novox"}
for _, row := range []struct {
name string
answers map[string]*delivery.CandidateAnswer
now time.Time
state string
words string
}{
{"all the same", map[string]*delivery.CandidateAnswer{"ace": same(7), "g14": same(8), "novox": same(0)}, at, delivery.CompositionPass, ""},
{"a module added where intended", map[string]*delivery.CandidateAnswer{"ace": {Head: "h2", Revision: 7, Verdict: delivery.CandidateDiffers,
Added: []delivery.ModuleAdded{{Module: "web", Unissued: []string{"its own secret"}}}}, "g14": same(8), "novox": same(0)}, at, delivery.CompositionPass, ""},
{"a module differing where not intended", map[string]*delivery.CandidateAnswer{"ace": {Head: "h2", Revision: 7, Verdict: delivery.CandidateDiffers,
Differs: []delivery.ModuleDiffers{{Module: "mail", Why: "the port differs"}}}, "g14": same(8), "novox": same(0)}, at, delivery.CompositionFail, "ace composes mail differently"},
{"a node's first placement file with a module differing", map[string]*delivery.CandidateAnswer{"ace": same(7), "g14": {Head: "h2", Revision: 8, Verdict: delivery.CandidateDiffers,
Differs: []delivery.ModuleDiffers{{Module: "web", Why: "a binding differs"}}}, "novox": same(0)}, at, delivery.CompositionFail, "g14 composes web differently"},
{"a refusal", map[string]*delivery.CandidateAnswer{"ace": same(7), "g14": {Head: "h2", Revision: 8, Verdict: delivery.CandidateRefused,
Refusals: []delivery.ModuleRefused{{Module: "web", Why: "no own secret"}}}, "novox": same(0)}, at, delivery.CompositionFail, "g14 refuses"},
{"a node not answering, within its bound", map[string]*delivery.CandidateAnswer{"ace": same(7), "novox": same(0)}, at.Add(AnswerBound - time.Second), delivery.CompositionWaits, "g14 (offline)"},
{"an answer of an earlier revision, past its bound", map[string]*delivery.CandidateAnswer{"ace": same(6), "g14": same(8), "novox": same(0)}, at.Add(AnswerBound), delivery.CompositionFail, "ace " + delivery.CompositionUnanswered},
{"its later answer", map[string]*delivery.CandidateAnswer{"ace": same(7), "g14": same(8), "novox": same(0)}, at.Add(time.Hour), delivery.CompositionPass, ""},
} {
pass := delivery.CompositionVerdict{Group: "g", Heads: "0123456789ab", State: delivery.CompositionPass, SDK: "sdk", Machines: []string{"ace", "g14"}}
v := WithAnswers(pass, JudgeAnswers(cp, placedNodes, row.answers, intended, map[string]bool{"ace": true, "novox": true}, row.now))
if v.State != row.state || !strings.Contains(v.Words, row.words) {
t.Errorf("%s: %+v", row.name, v)
}
if _, err := delivery.EncodeCompositionVerdict(v); err != nil {
t.Errorf("%s: the verdict cannot be written: %v", row.name, err)
}
}
trunk := map[string]delivery.Placement{"ace": {Node: "ace", Modules: []delivery.PlacedModule{{Module: "mail"}}}, "novox": {Node: "novox"}}
if got := PlacedNodes(trunk, &ChangePlacement{Placements: map[string]*delivery.Placement{"novox": nil, "g14": {Node: "g14"}}}); !slices.Equal(got, []string{"ace", "g14", "novox"}) {
t.Errorf("placed at the head or on the trunk = %v", got)
}
got, _ := Intended(nil, trunk, &ChangePlacement{Placements: map[string]*delivery.Placement{"ace": {Node: "ace", Modules: []delivery.PlacedModule{{Module: "mail", Ports: map[string]int{"25": 2525}}, {Module: "web"}}}, "g14": {Node: "g14", Modules: []delivery.PlacedModule{{Module: "web"}}}}}, nil)
if !got["ace"]["mail"] || !got["ace"]["web"] || len(got["g14"]) != 0 {
t.Errorf("intended = %v, want ace's mail and web, nothing on g14 the trunk does not place", got)
}
}
// 1297/B3c: what was already broken on the trunk is said and fails nothing.
@@ -183,10 +109,6 @@ func TestTouched(t *testing.T) {
if got := Touched(m, Change{Manifests: map[string]catalogue.Manifest{"store": m.Manifests["store"]}}); !slices.Equal(got, []string{"ace", "g14"}) {
t.Errorf("touched by store with a consumer on g14 = %v", got)
}
// ADR 0559 §5: every node whose placement the change changes is touched, its file removed or not.
if got := Touched(m, Change{Placement: &ChangePlacement{Placements: map[string]*delivery.Placement{"novox": nil, "shanks": {Node: "shanks"}}}}); !slices.Equal(got, []string{"novox", "shanks"}) {
t.Errorf("touched by a placement = %v", got)
}
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"new": {Module: "new", Version: "1"}}}
if v := Compose(m, c, "sdk", 0); len(v.Machines) != 0 || !hasWarning(v, "touches no machine") || v.State != delivery.CompositionPass {
t.Errorf("an added module assigned nowhere: %+v", v)
@@ -42,28 +42,9 @@ func HeadJob(m Mesh, c Change) (delivery.ComposeJob, error) {
for _, name := range c.Removed {
delete(change, name)
}
in, fromClaims := plainInput(m.Assigned), true
if m.Input != nil {
// The input the nodes compose from, without its catalogue: the cases carry the manifests (issue 1126).
in = *m.Input
in.Catalogue = nil
fromClaims = len(in.Holdings) == 0
}
changeCase := composer.Case{Name: CaseChange, Manifests: change}
if c.Placement != nil {
// The change's own placement, laid over the input's in the change case (ADR 0559 §5; composer.Case's Placements):
// a node the input does not hold yet is added bare, so the composer places it.
changeCase.Placements = c.Placement.Placements
in.Nodes = slices.Clone(in.Nodes)
for _, node := range c.Placement.Nodes() {
if !slices.ContainsFunc(in.Nodes, func(n compose.Node) bool { return n.Name == node }) {
in.Nodes = append(in.Nodes, compose.Node{Name: node})
}
}
}
req := composer.Request{V: composer.Version, Input: in, Depth: composer.DepthResolve,
HoldingsFromClaims: fromClaims, Machines: Touched(m, c),
Cases: []composer.Case{{Name: CaseTrunk, Manifests: trunk}, changeCase}}
req := composer.Request{V: composer.Version, Input: plainInput(m.Assigned), Depth: composer.DepthResolve,
HoldingsFromClaims: true, Machines: Touched(m, c),
Cases: []composer.Case{{Name: CaseTrunk, Manifests: trunk}, {Name: CaseChange, Manifests: change}}}
return job(SDKRepository, c.SDKHead, SDKDir, req, "the mesh-sdk head of "+c.Group)
}
@@ -214,10 +195,6 @@ func FromComposer(m Mesh, c Change, head Ran, readers []Reader, took time.Durati
readerFails, readerWarnings := judgeReaders(c, change, readers)
fails = append(fails, readerFails...)
v.Warnings = append(v.Warnings, readerWarnings...)
if m.InputUnread != "" {
v.Warnings = append(v.Warnings, "composed over the nodes and their assignments alone: the input the nodes compose from "+
"cannot be read from the bus ("+m.InputUnread+")")
}
return finish(v, m, c, machines, fails)
}
@@ -8,7 +8,6 @@ import (
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/compose"
"git.novox.be/novox/mesh-sdk/go/composer"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -101,41 +100,6 @@ func TestAGroupWithAnSDKMemberAsksTheComposerAtItsHead(t *testing.T) {
if !slices.Equal(req.Machines, []string{"g14"}) || req.Depth != composer.DepthResolve || !req.HoldingsFromClaims {
t.Errorf("request = machines %v, depth %q, holdings from claims %v", req.Machines, req.Depth, req.HoldingsFromClaims)
}
// 1126 (ADR 0547 §2, ADR 0549 §3): once the catalogue places the nodes, the check composes over the input the nodes
// compose from (frombus), its catalogue left to the cases, and the seats held as placement says.
m := digestedMesh()
m.Input = &compose.Input{Catalogue: map[string]catalogue.Manifest{"web": {Module: "web"}},
Nodes: []compose.Node{{Name: "g14", Assigned: []string{"web"}, Place: compose.Place{Endpoint: "g14.example:51820"}}},
Holdings: []catalogue.Held{{Claim: "web-seat", Scope: "mesh", Node: "g14", Module: "web"}}}
placedJob, err := HeadJob(m, c)
if err != nil {
t.Fatal(err)
}
placed, err := composer.Decode(placedJob.Request)
if err != nil {
t.Fatal(err)
}
if placed.Input.Catalogue != nil || len(placed.Input.Nodes) != 1 || placed.Input.Nodes[0].Place.Endpoint != "g14.example:51820" ||
len(placed.Input.Holdings) != 1 || placed.HoldingsFromClaims {
t.Errorf("the placed check's request = %+v, holdings from claims %v", placed.Input, placed.HoldingsFromClaims)
}
// ADR 0559 §5: a change of placement composes its own placement in the change case alone (composer.Case's Placements),
// a node the input does not hold yet added bare so the composer places it, and the node touched.
c.Placement = &ChangePlacement{Placements: map[string]*delivery.Placement{"novox": {Node: "novox"}}}
candJob, err := HeadJob(m, c)
if err != nil {
t.Fatal(err)
}
cand, err := composer.Decode(candJob.Request)
if err != nil {
t.Fatal(err)
}
if cand.Cases[0].Placements != nil || cand.Cases[1].Placements["novox"] == nil || len(cand.Input.Nodes) != 2 || cand.Input.Nodes[1].Name != "novox" ||
!slices.Equal(cand.Machines, []string{"g14", "novox"}) || len(m.Input.Nodes) != 1 {
t.Errorf("the placement's request = cases %+v, nodes %+v, machines %v", cand.Cases, cand.Input.Nodes, cand.Machines)
}
}
// 1622/AC2, AC3: a catalogue member using a field only the SDK head reads passes with the head's composer, and the
@@ -1,99 +0,0 @@
package assign
import (
"fmt"
"sort"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// Until ADR 0547's third milestone two sources name which module runs on which node: the controller's assignments and
// the indexer's placement, read from the catalogue (ADR 0549 §5). The assigner writes wants only for a pair both name:
// for a pair one source names alone it writes no want and touches none it wrote before, so nothing moves and nothing is
// removed on one source's word, and it raises a condition naming the pair until both agree. Before the first placement
// file reaches the indexer (no placement key at all) the controller's assignments are the one source, as before.
// Pair is a module on a node that one source names and the other does not; Only says which: OnlyAssignments or
// OnlyPlacement.
type Pair struct {
Node, Module, Only string
}
// The source that names a pair alone.
const (
OnlyAssignments = "the controller's assignments"
OnlyPlacement = "the catalogue's placement"
)
// Placed is, by node, the modules both sources name, in name order, and every pair one names alone, by node then
// module. With no placement read yet (seeded false) it is the assignments, and no pair differs.
func Placed(assigned, placed map[string][]string, seeded bool) (map[string][]string, []Pair) {
if !seeded {
return assigned, nil
}
set := func(m map[string][]string) map[string]bool {
out := map[string]bool{}
for node, modules := range m {
for _, module := range modules {
out[node+"\x00"+module] = true
}
}
return out
}
inAssigned, inPlaced := set(assigned), set(placed)
both := map[string][]string{}
var differ []Pair
for node, modules := range assigned {
for _, module := range modules {
if inPlaced[node+"\x00"+module] {
both[node] = append(both[node], module)
} else {
differ = append(differ, Pair{Node: node, Module: module, Only: OnlyAssignments})
}
}
}
for node, modules := range placed {
for _, module := range modules {
if !inAssigned[node+"\x00"+module] {
differ = append(differ, Pair{Node: node, Module: module, Only: OnlyPlacement})
}
}
}
for _, modules := range both {
sort.Strings(modules)
}
sort.Slice(differ, func(i, j int) bool {
if differ[i].Node != differ[j].Node {
return differ[i].Node < differ[j].Node
}
return differ[i].Module < differ[j].Module
})
return both, differ
}
// PlacementCondition is the condition of a pair the two sources do not both name: raised while they differ, cleared once
// both name it or neither does.
func PlacementCondition(p Pair, raised bool) asks.Condition {
key := delivery.AssignerModule + ".placement." + p.Node + "." + p.Module
subject := asks.Scope{Scope: "module", ID: p.Module, Machine: p.Node}
if !raised {
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared,
Resolved: fmt.Sprintf("the controller's assignments and the catalogue's placement agree on %s on %s.", p.Module, p.Node)}
}
headline := fmt.Sprintf("%s on %s is placed by one source only", p.Module, p.Node)
if n := len([]rune(headline)); n > asks.HeadlineLength {
headline = string([]rune(headline)[:asks.HeadlineLength])
}
var needs string
if p.Only == OnlyAssignments {
needs = fmt.Sprintf("add %s to the catalogue's placement file of %s, or unassign it at the controller", p.Module, p.Node)
} else {
needs = fmt.Sprintf("assign %s to %s at the controller, as the merged placement file says", p.Module, p.Node)
}
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
Summary: headline, Headline: headline, Needs: needs,
Explanation: fmt.Sprintf("%s%s. %s names %s on %s and the other source does not. Until ADR 0547's third milestone both must "+
"name a module on a node (ADR 0549 §5): the assigner writes no want for it there and leaves the one it wrote before, so "+
"nothing moves or is removed. This clears once both name it or neither does.", asks.NeedsYou, needs, p.Only, p.Module, p.Node)}
}
@@ -2,7 +2,6 @@ package assign
import (
"fmt"
"maps"
"slices"
"sort"
"strings"
@@ -47,14 +46,8 @@ type Module struct {
Now time.Time
// Puts are the puts of the module, by node (ADR 0546): while one owns a node's want (PutOwns), the rules pass over it.
Puts map[string]delivery.Put
// Self says whether node composes the module itself: the catalogue's placement names the node (ADR 0547 §2). It is the one
// source of a want's Self: every want the rules write or keep is derived from it (selfDerived), so a want written Self while
// the node was placed does not outlive the placement (issue 1126). Nil: no node does.
Self func(node string) bool
}
func (m Module) selfOn(node string) bool { return m.Self != nil && m.Self(node) }
func (m Module) composition(node, fp string) (delivery.ComposedDigest, bool) {
if m.Composed == nil {
return delivery.ComposedDigest{}, false
@@ -176,9 +169,6 @@ type Outcome struct {
// Lifted is the hold of m.FP lifted by itself: every node that runs the module runs m.FP healthy (issue 1906). The held
// record and its ask are taken away, and the wants move by the rules again in this pass.
Lifted bool
// NodeFailed is, by node, a failed health check of m.FP on a node while another node runs m.FP healthy: the failure is
// that node's, so m.FP is not held and nothing is set back (ADR 0557 §1, issue 1496). The pass raises its condition once.
NodeFailed map[string]NodeFailure
// Record is the moves a person must allow before the want names them: each is asked once (ADR 0539 §14).
Record []RecordNeed
// Restore is the nodes of a module holding data whose held fingerprint failed its health check there and that are not set
@@ -202,27 +192,10 @@ type RecordNeed struct {
Words string
}
// Holding is a rule holding a node back, and why. Bus marks the bar on the bus instead of one of the seven rules (Rule is then
// 0): the bus moves only by the operator's put (ADR 0539 §15, ADR 0546 §8), never by the assigner (issue 1945: nats on novox,
// running an old-form build, got no want and the pass said nothing).
// Holding is a rule holding a node back, and why.
type Holding struct {
Rule int
Words string
Bus bool
}
// Reason is the holding in words: the rule and its words, or for the bus bar its words alone.
func (h Holding) Reason() string {
if h.Bus {
return h.Words
}
return fmt.Sprintf("rule %d: %s", h.Rule, h.Words)
}
// BusBarWords say why the assigner writes no want moving the bus on node: it runs prev, the trunk builds fp, and only the
// operator's put moves it (ADR 0539 §15, ADR 0546 §8). The words reach the operator, so they name no record.
func BusBarWords(node, prev, fp string) string {
return fmt.Sprintf("only your put moves the bus: %s runs %s, the trunk builds %s; put it with assigner_put", node, short(prev), short(fp))
}
// Wants is the pass of one module m: for each of its nodes, the want by the six rules, starting from the want written
@@ -238,25 +211,11 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
}
if !held && !out.Lifted && !m.Released && m.FP != "" {
for _, node := range slices.Sorted(slices.Values(m.Nodes)) {
h, ok := n.Haves[node][m.Name]
if !ok || h.FP != m.FP || !failed(h) {
continue
if h, ok := n.Haves[node][m.Name]; ok && h.FP == m.FP && failed(h) {
out.Held = &delivery.Held{Node: node, Words: HeldWords(h, m.Now)}
held = true
break
}
if others := healthyElsewhere(m, n, node); len(others) > 0 && !failedElsewhere(m, n, node) {
// F runs healthy on another node, so its failure here is this node's, not F's: F is not held, no node is set back,
// and the failing node keeps its want (ADR 0557 §1; issue 1496: on ace a newly assigned mesh-build-node failed its
// first check and the hold moved shanks, g14 and novox, healthy on F for an hour, back). The pass raises the node's
// condition. With no online healthy witness, or with F failed on a second node, the hold and the set-back stand (ADR 0557
// §2, ADR 0429 §6).
if out.NodeFailed == nil {
out.NodeFailed = map[string]NodeFailure{}
}
out.NodeFailed[node] = NodeFailure{Others: others, Words: HeldWords(h, time.Time{})} // said once: no clock in it
continue
}
out.Held = &delivery.Held{Node: node, Words: HeldWords(h, m.Now)}
held = true
break
}
}
mm := m
@@ -322,13 +281,6 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
ok, target = true, have.FP
}
}
if ok && target == m.FP && len(out.NodeFailed) > 0 && !(had && !before.Removed && before.FP == m.FP) &&
n.Haves[node][m.Name].FP != m.FP {
// While a node fails F alone, F reaches no further node: a node that neither wants nor runs F gets no new want
// naming it, and wants already written stay (ADR 0557 §2). A node that runs F with no live want gets its want at F, or
// it would be on neither list and its node-engine would give the module up (issue 1945).
ok, rule, words = false, delivery.RuleNotHeld, FailedAloneWords(m.Name, m.FP, slices.Sorted(maps.Keys(out.NodeFailed)))
}
switch {
case ok:
d, _ := m.composition(node, target)
@@ -351,14 +303,6 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
if need, wait := at.mustAsk(node, asked, hadLive, ran, running.FP, running.Composed == "", d); wait {
if need != nil {
out.Record = append(out.Record, *need)
} else if prev, barred := at.busBar(asked, hadLive, running.FP); barred {
// The bus is barred: no want moves it but the operator's put. Said, never silent (issue 1945): its holding is named,
// and a node that runs it with no live want has it on neither list, so its condition stays raised.
h := Holding{Bus: true, Words: BusBarWords(node, prev, at.FP)}
out.Rules[node] = h
if !hadLive && running.Phase != delivery.PhaseRemoved && !held {
out.Neither[node] = h
}
}
continue
}
@@ -424,10 +368,6 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
}
}
}
next, write := m.selfDerived(node, before, had, next, n, &out)
if !write {
continue
}
if !had || !delivery.SameWant(normal(before), normal(next)) {
out.Wants[node] = next
}
@@ -440,43 +380,6 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
return out
}
// selfDerived is next with its Self derived from the placement now (Module.Self), whichever rule made it, and whether it is
// written. A placed node's want names the build alone (ADR 0547 §2). An unplaced node's want that a rule kept from the want
// written while the node was placed (held at F, held back by rule 3 or 5, a data module asked about) names the controller's
// composition of the same fingerprint, asked as any move of a record module is (ADR 0539 §14): on 2026-10-27, after the
// placement seed's revert, g14's systemd-resolved (held) and forticlient (rule 5), and shanks's forticlient (rule 3), kept
// their Self wants, which their node agents, no longer placed, could not compose (issue 1126). With no such composition, or
// while its ask waits, the want written stays and the node's holding says why.
func (m Module) selfDerived(node string, before delivery.Want, had bool, next delivery.Want, n Nodes, out *Outcome) (delivery.Want, bool) {
switch {
case next.Removed:
return next, true
case m.selfOn(node):
next.Self, next.Composed = true, ""
return next, true
case !next.Self && next.Composed != "":
return next, true
}
next.Self, next.Yes = false, ""
d, there := m.composition(node, next.FP)
if !there || d.Digest == "" {
out.Rules[node] = Holding{Rule: delivery.RuleComposed, Words: fmt.Sprintf("%s on %s was composed by the node while it was placed, "+
"and the controller has not composed it at %s, so its want stays", m.Name, node, short(next.FP))}
return next, false
}
at := m
at.FP = next.FP
running, ran := n.Haves[node][m.Name]
if need, wait := at.mustAsk(node, before, had, ran, running.FP, running.Composed == "", d); wait {
if need != nil && !slices.Contains(out.Record, *need) { // the branch held back at an older fingerprint may have asked it
out.Record = append(out.Record, *need)
}
return next, false
}
next.Composed = d.Digest
return next, true
}
// NoWantWords say why a node gets no want for a module: the rule that holds its first want back and why (issue 1945).
func NoWantWords(rule int, words string) string {
return fmt.Sprintf("no want: rule %d: %s", rule, words)
@@ -491,43 +394,6 @@ func failed(h delivery.Have) bool {
return h.Health == delivery.HealthFailed && (h.Phase == delivery.PhaseRunning || h.Phase == delivery.PhaseFailed)
}
// healthyElsewhere is every online node of m but node that runs m.FP healthy, by name.
func healthyElsewhere(m Module, n Nodes, node string) []string {
var out []string
for _, other := range slices.Sorted(slices.Values(m.Nodes)) {
// A witness is online: a node that sleeps keeps the have it last wrote, and a stale have is no evidence that F runs
// healthy now.
if other != node && n.Online[other] && healthy(n.Haves[other][m.Name], m.FP) {
out = append(out, other)
}
}
return out
}
// failedElsewhere says whether another node of m has a have naming m.FP failed: then the failure is not one node's alone, and
// the hold and the set-back stand at the second failure (ADR 0557 §2).
func failedElsewhere(m Module, n Nodes, node string) bool {
for _, other := range m.Nodes {
if h, ok := n.Haves[other][m.Name]; ok && other != node && h.FP == m.FP && failed(h) {
return true
}
}
return false
}
// FailedAloneWords say why a node gets no new want naming fp: fp fails its health check on the nodes named, alone.
func FailedAloneWords(module, fp string, failing []string) string {
return fmt.Sprintf("%s at %s fails its health check on %s, so no further node moves to it until that is over", module, short(fp),
strings.Join(failing, ", "))
}
// NodeFailure is a failed health check of a module's newest fingerprint on one node while it runs healthy on Others: the have's
// health, phase and words (HeldWords without a time), and the healthy witnesses by name.
type NodeFailure struct {
Others []string
Words string
}
// runs says whether a have is of a module the node runs: running, whatever its health.
func runs(h delivery.Have) bool {
return h.Phase == delivery.PhaseRunning
@@ -600,7 +466,12 @@ func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, runnin
if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == delivery.JudgedUnchanged {
return nil, false
}
if _, barred := m.busBar(before, had, running); barred {
// The fingerprint the bus ran before: its want's, or with no readable want the one its have names (ADR 0539 §15 bars a new one).
prev := before.FP
if !had {
prev = running
}
if catalogue.ProvidesBus(m.Manifest) && prev != m.FP {
return nil, true // the bus is never moved by the assigner; the operator writes its want (§15)
}
if !m.recorded(d) || d.Judgement == delivery.JudgedNeither || d.Judgement == delivery.JudgedUnchanged {
@@ -612,16 +483,6 @@ func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, runnin
return &RecordNeed{Node: node, Digest: d.Digest, Judgement: d.Judgement, Words: d.Words}, true
}
// busBar says whether m is the bus and the move would take it off the fingerprint it ran before, prev: its want's, or with no
// readable want the one its have names (ADR 0539 §15 bars a new one).
func (m Module) busBar(before delivery.Want, had bool, running string) (prev string, barred bool) {
prev = before.FP
if !had {
prev = running
}
return prev, catalogue.ProvidesBus(m.Manifest) && prev != m.FP
}
// recorded says whether the module is of policy record: as the controller holds it beside the composition's digest, the
// operator's own choice included (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it.
func (m Module) recorded(d delivery.ComposedDigest) bool {
@@ -87,96 +87,6 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
}
}
// 1126/AC-SW1, AC-SW2, AC-SW3: a want written Self while g14 was placed, kept by a rule after the placement is gone, names
// the controller's composition of the fingerprint the rule keeps (on 2026-10-27 g14's systemd-resolved, held, and forticlient,
// rule 5, kept their Self wants, which g14 could no longer compose); with no such composition, or while a record ask waits, it
// stays and is said; a placed node's want is Self with no composition.
selfAt := func(fp string) map[string]delivery.Want {
return map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fp, Self: true, Yes: dig(fp)}}
}
atF := func(_ *Module, n *Nodes) {
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Composed: dig(fNew), Phase: delivery.PhaseFailed, Health: delivery.HealthOK}
}
for _, c := range []struct {
name string
set func(*Module, *Nodes)
before string
want delivery.Want // the want of g14 written; zero: none written
rule int // the rule g14's holding names
asked bool
}{
{name: "rule 5 at F", before: fNew, set: func(m *Module, n *Nodes) {
atF(m, n)
n.Haves["ace"]["store"] = delivery.Have{FP: fpOf('s'), Phase: delivery.PhaseRunning, Health: delivery.HealthWaits}
}, want: delivery.Want{FP: fNew, Composed: dig(fNew)}, rule: delivery.RuleProviders},
{name: "rule 3 at F", before: fNew, set: func(m *Module, n *Nodes) {
atF(m, n)
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
}, want: delivery.Want{FP: fNew, Composed: dig(fNew)}, rule: delivery.RuleCanary},
{name: "rule 3 held back at an older fingerprint", before: fOld, set: func(_ *Module, n *Nodes) {
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
}, want: delivery.Want{FP: fOld, Composed: dig(fOld), Newest: fNew, Rule: delivery.RuleCanary,
Words: "the canary ace does not run bbbbbbbbbbbb healthy yet"}, rule: delivery.RuleCanary},
{name: "held at F with nothing to set back to", before: fNew, set: func(m *Module, n *Nodes) { atF(m, n); m.Held = true },
want: delivery.Want{FP: fNew, Composed: dig(fNew), Words: "mail is held at bbbbbbbbbbbb and g14 ran no other fingerprint healthy to set it back to"},
rule: delivery.RuleNotHeld},
{name: "held data module", before: fNew, set: func(m *Module, n *Nodes) {
atF(m, n)
m.Held = true
m.Manifest.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Class: catalogue.ClassIrreplaceable}}}
}, want: delivery.Want{FP: fNew, Composed: dig(fNew),
Words: "mail holds data, so it is not set back from bbbbbbbbbbbb: the operator is asked whether to restore it"}, rule: delivery.RuleNotHeld},
{name: "a record ask waits", before: fNew, set: func(m *Module, n *Nodes) {
atF(m, n)
recording(m)
m.Held = true
m.Composed = func(_, fp string) (delivery.ComposedDigest, bool) {
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedRecreates, Against: dig(fOld)}, true
}
}, rule: delivery.RuleNotHeld, asked: true},
{name: "no composition of F", before: fNew, set: func(m *Module, n *Nodes) {
atF(m, n)
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
if node == "g14" {
return delivery.ComposedDigest{}, false
}
return composedEverywhere(node, fp)
}
}, rule: delivery.RuleComposed},
{name: "placed", before: fNew, set: func(m *Module, n *Nodes) {
atF(m, n)
m.Self = func(node string) bool { return node == "g14" }
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
if node == "g14" {
return delivery.ComposedDigest{Judgement: delivery.JudgedNeither}, true
}
return composedEverywhere(node, fp)
}
}, want: delivery.Want{FP: fNew, Self: true}},
} {
m, n := ready()
c.set(&m, &n)
before := selfAt(c.before)
if c.name == "placed" {
before["g14"] = delivery.Want{FP: fNew, Composed: dig(fNew)}
}
out := Wants(m, n, before, nil)
if got := out.Wants["g14"]; got != c.want {
t.Errorf("%s: g14's want = %+v, want %+v", c.name, got, c.want)
}
if got := out.Rules["g14"].Rule; got != c.rule {
t.Errorf("%s: g14 is held by rule %d (%s), want %d", c.name, got, out.Rules["g14"].Words, c.rule)
}
if asked := len(out.Record) == 1 && out.Record[0].Node == "g14"; asked != c.asked {
t.Errorf("%s: record asks %+v", c.name, out.Record)
}
if w, ok := out.Wants["g14"]; ok {
if _, err := delivery.EncodeWant(w); err != nil {
t.Errorf("%s: g14's want does not encode: %v", c.name, err)
}
}
}
// Rule 6 per node: a grant written for the module's other nodes does not grant a node whose account it does not name
// (issue 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus).
m, n := ready()
@@ -240,72 +150,6 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
if w, ok := out.Wants["g14"]; ok && w.FP == fNew {
t.Errorf("g14 moved to a held fingerprint: %+v", w)
}
// 1496, ADR 0557 §1, §2: the same failure while F runs healthy on another node is that node's: F is not held, nothing is set
// back, the failing node keeps its want, and the outcome names it (on ace a newly assigned module's first check held a version healthy for an hour on three nodes).
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
before := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fNew, Composed: dig(fNew)}}
out = Wants(m, n, before, map[string]string{"ace": fOld, "g14": fOld})
if out.Held != nil || len(out.Wants) != 0 {
t.Fatalf("held %+v, wants %+v: want no hold and no want changed", out.Held, out.Wants)
}
if f := out.NodeFailed["ace"]; !slices.Equal(f.Others, []string{"g14"}) || f.Words != "health failed, phase running: connection refused" ||
len(out.NodeFailed) != 1 {
t.Errorf("node failed = %+v", out.NodeFailed)
}
// ADR 0557 §2 (version 9): while one node fails F alone, novox, whose want names fOld, gets no new want naming F; the wants
// at F stay. The canary ace runs F healthy here and g14 fails it, so the canary rule (3) would let novox move: only §2 holds it.
three := m
three.Nodes = []string{"ace", "g14", "novox"}
failedHave := n.Haves["ace"]["mail"]
n.Haves["ace"]["mail"], n.Haves["g14"]["mail"] = n.Haves["g14"]["mail"], failedHave
n.Online["novox"] = true
n.Haves["novox"] = map[string]delivery.Have{"mail": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}}
before["novox"] = delivery.Want{FP: fOld, Composed: dig(fOld)}
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
if _, alone := out.NodeFailed["g14"]; !alone {
t.Errorf("g14 does not fail alone: %+v", out.NodeFailed)
}
if w, written := out.Wants["novox"]; out.Held != nil || len(out.NodeFailed) != 1 || (written && w.FP == fNew) || out.Rules["novox"].Rule != delivery.RuleNotHeld {
t.Errorf("held %+v, node failed %+v, novox's want %+v (%+v): want novox kept off F", out.Held, out.NodeFailed, out.Wants["novox"], out.Rules["novox"])
}
if _, written := out.Wants["g14"]; written {
t.Errorf("g14's want at F was rewritten: %+v", out.Wants["g14"])
}
// A node that runs F healthy with no want, while g14 fails it alone, gets its want at F and is on no neither list (issue 1945).
n.Haves["novox"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
delete(before, "novox")
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
if w := out.Wants["novox"]; w.FP != fNew || len(out.Neither) != 0 {
t.Errorf("novox running F with no want: want %+v, neither %+v; want its want at F and no neither", w, out.Neither)
}
n.Haves["ace"]["mail"], n.Haves["g14"]["mail"] = failedHave, n.Haves["ace"]["mail"]
// ADR 0557 §2: a witness that is offline keeps the have it last wrote and witnesses nothing (D1); F failing on a second node is
// not one node's failure (D3). Either way F is held and ace is set back.
for _, c := range []struct {
name string
online map[string]bool
novox delivery.Health
}{
{"the witness sleeps", map[string]bool{"ace": true, "g14": false, "novox": true}, delivery.HealthOK},
{"F fails on a second node", map[string]bool{"ace": true, "g14": true, "novox": true}, delivery.HealthFailed},
} {
n.Online = c.online
n.Haves["novox"] = map[string]delivery.Have{"mail": {FP: fNew, Phase: delivery.PhaseRunning, Health: c.novox}}
if c.novox == delivery.HealthOK {
n.Haves["novox"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
}
before["novox"] = delivery.Want{FP: fNew, Composed: dig(fNew)}
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
if out.Held == nil || out.Held.Node != "ace" || len(out.NodeFailed) != 0 {
t.Errorf("%s: held %+v, node failed %+v: want F held at ace", c.name, out.Held, out.NodeFailed)
}
if w := out.Wants["ace"]; w.FP != fOld {
t.Errorf("%s: ace's want = %+v, want set back", c.name, w)
}
}
}
// 1297/B4: a module holding data is not set back; its want stays and says why.
@@ -583,23 +427,6 @@ func TestTheBusIsNotMovedButItsUserListRevisionIsWrittenAtOnce(t *testing.T) {
t.Errorf("a held bus did not take its user-list revision as a put: %+v", w)
}
m.Puts = nil
// 1945 (the bus bar made loud): a node that runs the bus at an old-form build and holds no want for it gets none, since only
// the operator's put moves the bus, and the pass says so: a holding naming the bar, and the node on neither list so its
// condition stays raised. A node with a live want keeps it and is not on neither list.
m.Composed = composedEverywhere
n.Haves["ace"]["mail"] = delivery.Have{FP: "b-1700000000-oldform", Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
out = Wants(m, n, map[string]delivery.Want{"g14": {FP: fOld, Composed: dig(fOld)}}, nil)
if w, wrote := out.Wants["ace"]; wrote {
t.Errorf("the bus bar wrote ace's want %+v", w)
}
h, on := out.Neither["ace"]
if !on || !h.Bus || out.Rules["ace"] != h || !strings.Contains(h.Words, "only your put moves the bus") || !strings.Contains(h.Words, "ace runs b-1700000000-oldform") ||
!strings.Contains(h.Words, "assigner_put") {
t.Errorf("the barred bus on ace: neither %+v (%v), rules %+v", h, on, out.Rules["ace"])
}
if _, on := out.Neither["g14"]; on || out.Wants["g14"].FP == fNew {
t.Errorf("g14 has a live want: neither %+v, its want %+v", out.Neither, out.Wants["g14"])
}
stepsOfAPut(t)
}
@@ -34,7 +34,7 @@ func TestManifest(t *testing.T) {
slices.Sort(reads)
want := []string{"mesh-build-go.steps", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants",
"mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests",
"mesh-delivery-indexer.modules", "mesh-delivery-indexer.placement", "mesh-delivery-indexer.placement-candidates", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "node-engine.have"}
"mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "node-engine.have"}
if !reflect.DeepEqual(reads, want) {
t.Errorf("reads = %v, want exactly %v", reads, want)
}
@@ -12,8 +12,6 @@
"mesh-delivery-indexer.candidates",
"mesh-delivery-indexer.manifests",
"mesh-delivery-indexer.modules",
"mesh-delivery-indexer.placement",
"mesh-delivery-indexer.placement-candidates",
"mesh-delivery-teller.answers",
"mesh-delivery-watchman.presence",
"node-engine.have"
@@ -15,7 +15,6 @@ import (
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"mesh-delivery-builder/internal/build"
)
@@ -32,8 +31,7 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q
Description: "Every build key that waits (building or queued), oldest first: its module, fingerprint, step words and since when, and each ask it " +
"waits on by language with its id, when it was put, its asking (1 for the first; the builder asks an ask again when it can no longer be " +
"answered, at most 3 times) and what its build module's steps hold of it (none, or its state on its node since its start). The " +
"seat workers' delivery counts are not readable by the builder. dead_letters says whether the builder hears the asks the build seats give up " +
"on (issue 734), and why not: heard, a given-up ask is asked again at once; deaf, at its check bound. With module, that module's keys only. Read-only.",
"seat workers' delivery counts are not readable by the builder. With module, that module's keys only. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "only this module's keys"}}},
Run: func(args map[string]any) (any, error) { return r.Stuck(str(args, "module")) }},
@@ -168,8 +166,7 @@ func main() {
steps[bm] = read(stdio.State(bm + ".steps"))
}
reasks := stdio.State(build.ReAsksState)
r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps,
Hearing: read(stdio.State(buildrun.HearingState))}
r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps}
put := func(key string, value []byte) error {
_, err := reasks.Put(key, json.RawMessage(value))
return err
@@ -24,7 +24,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-builder/internal/build"
)
@@ -346,7 +345,7 @@ func (b *bed) archive(t *testing.T, name, path, at string) string {
func (b *bed) start(t *testing.T, node string, env map[string]string) {
t.Helper()
broker := filepath.Join(t.TempDir(), "broker")
cred, _ := json.Marshal(process.Account{URL: b.srvURL, Node: node, Module: "mesh-delivery-builder", User: node + ".mesh-delivery-builder", Password: "pw"})
cred, _ := json.Marshal(credential{URL: b.srvURL, Node: node, Module: "mesh-delivery-builder", User: node + ".mesh-delivery-builder", Password: "pw"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
@@ -6,6 +6,12 @@ package main
import (
"context"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"log"
@@ -45,18 +51,80 @@ func main() {
}
}
// credential is the broker file the controller issues to the module: JSON of the bus address, the certificate
// fingerprint to pin, the principal and its password (as the indexer reads it; issue 1242 moves this into the SDK).
type credential struct {
URL string `json:"url"`
Fingerprint string `json:"fingerprint"`
Node string `json:"node"`
Module string `json:"module"`
User string `json:"user"`
Password string `json:"password"`
}
func readCredential(path string) (credential, error) {
raw, err := os.ReadFile(path)
if err != nil {
return credential{}, err
}
var c credential
if err := json.Unmarshal(raw, &c); err != nil {
return credential{}, fmt.Errorf("the broker credential is not one: %w", err)
}
if c.URL == "" || c.User == "" {
return credential{}, errors.New("the broker credential names no bus address or user")
}
return c, nil
}
// pinned accepts exactly the certificate with this SHA-256 and no other.
func pinned(want string) *tls.Config {
return &tls.Config{
InsecureSkipVerify: true, //nolint:gosec // replaced by the pin below, which is stricter
MinVersion: tls.VersionTLS12,
VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
if len(rawCerts) == 0 {
return errors.New("the bus presented no certificate")
}
sum := sha256.Sum256(rawCerts[0])
if got := "sha256:" + hex.EncodeToString(sum[:]); got != want {
return fmt.Errorf("the bus presented a certificate this mesh does not know (%.23s…)", got)
}
return nil
},
}
}
// run is the program: it returns when ctx ends or the bus cannot be used.
func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
logger := log.New(logOut, "mesh-delivery-builder: ", log.LstdFlags)
cred, err := process.ReadBrokerAccount(getenv)
path := strings.TrimSpace(getenv("MESH_BROKER_FILE"))
if path == "" {
return errors.New("MESH_BROKER_FILE is not set, so there is no credential to hear the bus with")
}
cred, err := readCredential(path)
if err != nil {
return err
}
conn, js, err := cred.Dial()
opts := []nats.Option{
nats.Name(cred.Node + "." + cred.Module),
nats.UserInfo(cred.User, cred.Password),
nats.CustomInboxPrefix("_INBOX." + cred.User),
nats.MaxReconnects(-1),
nats.RetryOnFailedConnect(true),
}
if strings.TrimSpace(cred.Fingerprint) != "" {
opts = append(opts, nats.Secure(pinned(cred.Fingerprint)))
}
conn, err := nats.Connect(cred.URL, opts...)
if err != nil {
return err
return fmt.Errorf("connecting to the bus: %w", err)
}
defer conn.Close()
js, err := jetstream.New(conn)
if err != nil {
return err
}
builds, err := process.EssentialBucket(ctx, js, "builder", delivery.BuilderBuildsBucket, "claim or write a build key")
if err != nil {
return err
@@ -81,28 +149,20 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// Each build module's part of a build is an ask on its build seat's stream, pulled by a build node with a free slot
// (novox/hq ADR 0525): the builder chooses no node, so it reads neither presence nor have and holds no capacity.
asks := buildrun.JetStreamAsks{JS: js}
// The asks the build seats gave up on (issue 734): heard from DEAD_LETTERS, each ends the wait for its step at once,
// and the lane asks its part again (Builder.waitOnce) instead of waiting for the check bound.
// Heard in the background, and again at a reconnect or an ask while it is not; whether it is heard is kept in the
// state hearing, which builder_stuck says (dead_letters).
dead := &buildrun.DeadLetters{}
// The state may not be on the bus yet when a delivery started the builder (the grants part creates it, issue 1790): it
// is bound when the bus says it was created, and the latest hearing is written then.
hearingKV := process.AwaitBucket(ctx, js, conn, delivery.BuilderModule+"_"+buildrun.HearingState, "builder_stuck cannot say whether DEAD_LETTERS is heard", logger.Printf)
hearing := buildrun.NewDeadLetterHearing(ctx, buildrun.JetStreamDeadLetters{JS: js}, delivery.AskLanguages, dead, logger.Printf,
buildrun.SayHearing(ctx, hearingKV, logger.Printf))
defer hearing.Stop()
conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again() })
// The builder asks the controller, as its own account, whether the artifact store still holds a done key's blobs
// (the seat verb held, novox/hq issue 1937) before it leaves the key done.
calls, err := process.Connect(cred, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
account, err := process.ReadAccount(path)
if err != nil {
return err
}
calls, err := process.Connect(account, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
if err != nil {
return err
}
defer calls.Close()
b := &build.Builder{
Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
Asks: buildrun.Asker{Publisher: asks, Waiter: buildrun.LostAskWaiter{Inner: asks, Dead: dead, Hearing: hearing}, Bound: bound},
Asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: bound},
Held: heldOnTheBus{js},
// An ask that can no longer be answered is asked again (issue 1945), read at the step bound: a context's bound.
Gone: heldOnTheBus{js},
@@ -160,12 +220,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// module.
for _, module := range []string{buildrun.GoBuildModule, buildrun.NodeBuildModule} {
stream := toolchainStreams[module]
cons, err := cred.EventConsumer(ctx, js, stream)
cons, err := js.Consumer(ctx, stream, cred.Node+"_"+cred.Module)
if err != nil {
logger.Printf("the consumer %s_%s on %s does not exist yet, so %s's reports are not heard until the builder starts again: %v", cred.Node, cred.Module, stream, module, err)
continue
}
cc, err := process.HearEvents(cons, func(m jetstream.Msg) { watch.Reported(m) }, logger.Printf)
cc, err := cons.Consume(func(m jetstream.Msg) { watch.Reported(m) },
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
if err != nil {
return err
}
@@ -174,12 +235,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// The controller says artifacts-gone when its take-in found a key's blob gone (issue 1937): the builder's durable
// consumer on EVENTS, made by the controller from the manifest's consumes. Where it is not there yet the builder says
// so; it still asks held at its start and at every module value.
switch cons, err := cred.EventConsumer(ctx, js, eventsStream); {
switch cons, err := js.Consumer(ctx, eventsStream, cred.Node+"_"+cred.Module); {
case err != nil:
logger.Printf("the consumer %s_%s on %s does not exist yet, so %s is not heard until the builder starts again: %v",
cred.Node, cred.Module, eventsStream, delivery.ArtifactsGoneEvent, err)
default:
cc, err := process.HearEvents(cons, func(m jetstream.Msg) { heardGone(ctx, b, m, logger.Printf) }, logger.Printf)
cc, err := cons.Consume(func(m jetstream.Msg) { heardGone(ctx, b, m, logger.Printf) },
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
if err != nil {
return err
}
@@ -218,25 +280,24 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
// A person asks a key's build again with builder_ask_again, which writes the reasks state (issue 1945): watched, as
// every other input. The builder asks again by itself without it.
// A state not on the bus yet at this start is watched once it appears (issue 1790).
process.AwaitBucket(ctx, js, conn, build.ReAsksBucket, "builder_ask_again is not heard; an ask that can no longer be answered is still asked again by itself", logger.Printf).Ready(func(kv jetstream.KeyValue) {
if kv := process.OptionalBucket(ctx, js, build.ReAsksBucket, "builder_ask_again is not heard until the builder starts again; an ask that can no longer be answered is still asked again by itself", logger.Printf); kv != nil {
eachReAsk := func(e jetstream.KeyValueEntry) { reAsked(ctx, b, kv, e, logger.Printf) }
if rw, err := settle(ctx, kv, eachReAsk); err != nil {
process.NotWatched(logger.Printf, build.ReAsksBucket, err, "builder_ask_again is not heard until the builder starts again")
} else {
go func() { errs <- follow(ctx, rw, eachReAsk) }()
}
})
}
// A person reads a build seat's stream with builder_seat_queue, which writes the queues state (issue 1945 note 62): the
// builder alone may read the stream, so it answers there.
process.AwaitBucket(ctx, js, conn, build.QueuesBucket, "builder_seat_queue is not answered", logger.Printf).Ready(func(kv jetstream.KeyValue) {
if kv := process.OptionalBucket(ctx, js, build.QueuesBucket, "builder_seat_queue is not answered until the builder starts again", logger.Printf); kv != nil {
eachRead := func(e jetstream.KeyValueEntry) { queueRead(ctx, heldOnTheBus{js}, kv, e, logger.Printf) }
if qw, err := settle(ctx, kv, eachRead); err != nil {
process.NotWatched(logger.Printf, build.QueuesBucket, err, "builder_seat_queue is not answered until the builder starts again")
} else {
go func() { errs <- follow(ctx, qw, eachRead) }()
}
})
}
logger.Printf("started on %s", cred.Node)
go func() { errs <- follow(ctx, w, each) }()
select {
@@ -8,7 +8,6 @@ import (
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
@@ -54,12 +53,12 @@ func reAsked(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstre
// emitAskedAgain says the builder's event build.asked-again (issue 1945): mesh.mod.<module>.event.build.asked-again,
// with the SDK's envelope headers and the new ask's id as the bus's dedup id.
func emitAskedAgain(ctx context.Context, js jetstream.JetStream, cred process.Account, e build.AskedAgain) error {
func emitAskedAgain(ctx context.Context, js jetstream.JetStream, cred credential, e build.AskedAgain) error {
data, err := json.Marshal(e)
if err != nil {
return err
}
msg := nats.NewMsg(cred.EventSubject(build.AskedAgainEvent))
msg := nats.NewMsg("mesh.mod." + cred.Module + ".event." + build.AskedAgainEvent)
id := "asked-again-" + e.New
msg.Header.Set("x-event-id", id)
msg.Header.Set("x-source", cred.Module)
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-builder
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+4 -2
View File
@@ -1,5 +1,7 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952 h1:TmveT3Bija643RGWkilvkzP/h8UWlVvHFZZtQfJK5oU=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1 h1:fzOkrz9CSp3mEjeV7NfZx8yoivI1sD1AOtIPtyyF8RI=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -12,7 +12,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// **An ask that can no longer be answered is asked again** (novox/hq issue 1945). A build seat's stream gives an ask up
@@ -21,8 +20,7 @@ import (
// restart take over the same id and wait again. The builder notices it at the check bound (CheckAfter, the step bound),
// a context's bound and no timer of its own: it asks Gone, which reads the bus, and asks such an ask again under a new
// id, written into the key, said in its log and as the event build.asked-again, at most Askings times; then the key is
// failed with why. An ask DEAD_LETTERS says the seat gave up (buildrun.LostAskWaiter, issue 734) is asked again at once.
// A person asks the same by hand with the tool builder_ask_again (AskAgain).
// failed with why. A person asks the same by hand with the tool builder_ask_again (AskAgain).
// DefaultAskings is how many times a build module's part of a build is asked, the first included, when Builder.Askings is 0.
const DefaultAskings = 3
@@ -317,11 +315,6 @@ func (b *Builder) waitOnce(ctx context.Context, ask delivery.StepAsk, again <-ch
defer stop()
}
step, err := b.Asks.Wait(wait, ask)
if err == nil && buildrun.IsLost(step) {
// The build seat gave the ask up (issue 734): it can no longer be answered, so it is asked again now, with the
// words of why, not at the check bound.
return step, step.Reason, errAgain
}
if err == nil || ctx.Err() != nil {
return step, "", err
}
@@ -690,14 +690,12 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
step string
byHand bool
always bool // Gone says every ask cannot be answered, not only the first
dead bool // DEAD_LETTERS keeps the first ask: the seat gave it up (issue 734)
puts int
done bool
asking int // the asking the event says, 0 when none is said
}{
{name: "given up by its stream: asked again under a new id and done from it", puts: 2, done: true, asking: 2},
{name: "asked again by hand: the same, with no check bound", byHand: true, puts: 2, done: true, asking: 2},
{name: "given up into DEAD_LETTERS: its wait ends failed with the words and it is asked again at once", dead: true, puts: 2, done: true, asking: 2},
{name: "never answerable: failed after its askings", askings: 2, always: true, puts: 2},
{name: "taken over at its last asking: failed, not asked again", askings: 3, always: true, step: "asked mesh-build-go again (asking 3): x", puts: 0},
} {
@@ -716,11 +714,7 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
r.st.vals[r.key] = data
}
gone := &goneAsks{}
dead := &buildrun.DeadLetters{}
if c.dead {
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: buildrun.LostAskWaiter{Inner: r.asks, Dead: dead}, Bound: 2 * time.Second}
}
if !c.byHand && !c.dead {
if !c.byHand {
gone.why = func(a delivery.StepAsk) string {
if c.always || a.ID == first || first == "" {
return "it waits first in line while a node took a later ask"
@@ -730,10 +724,6 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
r.b.Gone, r.b.CheckAfter = gone, 5*time.Millisecond
}
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
if c.dead {
dead.Heard(buildrun.GivenUp{AskID: waitForAsks(t, r, 1)[0].ID, Stream: "SEAT_NODE_BUILD_GO", Deliveries: 5,
At: time.Date(2026, 10, 26, 12, 26, 40, 0, time.UTC)})
}
if c.byHand {
waitForAsks(t, r, 1)
for deadline := time.Now().Add(2 * time.Second); len(r.b.AskAgain(r.key, "stuck since 19:06")) == 0; time.Sleep(time.Millisecond) {
@@ -767,16 +757,13 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
mu.Lock()
defer mu.Unlock()
// Note 62: the old message of an ask asked again because its stream gave it up is dropped from the stream.
if gone.mu.Lock(); c.done && !c.byHand && !c.dead && (len(gone.dropped) != 1 || gone.dropped[0] != first) {
if gone.mu.Lock(); c.done && !c.byHand && (len(gone.dropped) != 1 || gone.dropped[0] != first) {
t.Errorf("dropped %v from the stream, want the old ask %s", gone.dropped, first)
}
gone.mu.Unlock()
if c.asking > 0 && (len(said) != 1 || said[0].Asking != c.asking || said[0].New == said[0].Old || said[0].Key != r.key || said[0].Why == "") {
t.Errorf("said %+v, want once at asking %d", said, c.asking)
}
if c.dead && (len(said) != 1 || !strings.Contains(said[0].Why, buildrun.LostWords+" after 5 deliveries at 2026-10-26T12:26:40Z")) {
t.Errorf("said %+v, want the ask asked again saying the seat gave it up", said)
}
})
}
t.Run("what the bus says of an ask", func(t *testing.T) { // 1945
@@ -849,17 +836,12 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
running, _ := json.Marshal(delivery.Step{V: delivery.Version, ID: "mvo53jcm70b5f8", Tool: "delivery ask", Node: "g14", State: delivery.StepRunning, Started: at})
_, reask, _ := NewReAsk("m", fp1, "stuck", "shanks.lead", at)
r := StuckReader{Builds: state(map[string]string{key: string(building), done: string(finished)}), ReAsks: state(map[string]string{key: string(reask)}),
Steps: map[string]StateRead{buildrun.GoBuildModule: state(map[string]string{}), ImageBuildModule: state(map[string]string{"g14.mvo53jcm70b5f8": string(running)})},
Hearing: state(map[string]string{buildrun.HearingKey: `{"heard":false,"streams":["SEAT_NODE_BUILD_GO"],"why":"DEAD_LETTERS is not heard: SEAT_NODE_BUILD_GO: permissions violation","at":"2026-10-26T13:00:00Z"}`}),
Now: func() time.Time { return at.Add(time.Hour) }}
Steps: map[string]StateRead{buildrun.GoBuildModule: state(map[string]string{}), ImageBuildModule: state(map[string]string{"g14.mvo53jcm70b5f8": string(running)})},
Now: func() time.Time { return at.Add(time.Hour) }}
out, err := r.Stuck("")
if err != nil {
t.Fatal(err)
}
// Issue 734, review round 1: a builder that does not hear DEAD_LETTERS says so, with why.
if dl := out["dead_letters"].(map[string]any); dl["heard"] != false || !strings.Contains(dl["why"].(string), "permissions violation") {
t.Errorf("dead_letters = %v, want not heard, saying why", dl)
}
stuck := out["stuck"].([]Stuck)
if len(stuck) != 1 || stuck[0].Key != key || stuck[0].Age != "1h0m0s" || stuck[0].NoneStep != 1 || stuck[0].ReAsk == nil || stuck[0].ReAsk.Why != "stuck" {
t.Fatalf("stuck = %+v, want the building key alone, an hour old, one ask with no step, with its ask again", stuck)
@@ -106,9 +106,7 @@ type StuckReader struct {
ReAsks StateRead
// Steps are the build modules' steps states, by build module.
Steps map[string]StateRead
// Hearing is the builder's state hearing: whether it hears DEAD_LETTERS (issue 734); without it none is said.
Hearing StateRead
Now func() time.Time
Now func() time.Time
// Logf, when set, gets the one line a call logs: the values it read and the time it took.
Logf func(format string, args ...any)
}
@@ -151,20 +149,6 @@ func parallel(n int, read func(i int) error) error {
}
// Stuck is every key building or queued with its asks, oldest first; with module, that module's only.
// deadLetters says whether the builder hears the asks the build seats give up on (issue 734), and why not: deaf, a lane
// whose ask is given up waits for its check bound (buildrun.HearingHealth over the kept hearing).
func (r StuckReader) deadLetters() map[string]any {
if r.Hearing.Get == nil {
return map[string]any{"heard": false, "why": "the builder's state hearing is not readable here"}
}
raw, _, err := r.Hearing.Get(buildrun.HearingKey)
if err != nil {
return map[string]any{"heard": false, "why": "the builder's state hearing cannot be read: " + err.Error()}
}
a := buildrun.HearingHealth(raw)
return map[string]any{"heard": a.Healthy, "why": a.Why}
}
func (r StuckReader) Stuck(module string) (map[string]any, error) {
started := time.Now()
reads := 0
@@ -284,7 +268,6 @@ func (r StuckReader) Stuck(module string) (map[string]any, error) {
})
answer := map[string]any{"stuck": out, "count": len(out),
"delivery_counts": "not readable by the builder: the build seats' workers' information is granted to the delivery view alone (its backlog); an ask given up on is told by the builder at its check bound and asked again"}
answer["dead_letters"] = r.deadLetters()
if cut {
answer["cut"], answer["note"] = true, fmt.Sprintf("read %d of %d build keys in key order and stopped; give a module to read its keys alone", MaxReads, total)
}
@@ -87,7 +87,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
if m.Module != "mesh-delivery-builder" {
t.Errorf("module = %q", m.Module)
}
if want := []state{{Name: "builds", History: 5}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "queues", History: 1}, {Name: "reasks", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
if want := []state{{Name: "builds", History: 5}, {Name: "instances", History: 1}, {Name: "queues", History: 1}, {Name: "reasks", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
t.Errorf("state = %+v, want %+v", m.State, want)
}
// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and
@@ -20,7 +20,6 @@
],
"state": [
{ "name": "builds", "history": 5 },
{ "name": "hearing", "history": 1 },
{ "name": "instances", "history": 1 },
{ "name": "queues", "history": 1 },
{ "name": "reasks", "history": 1 }
@@ -12,7 +12,6 @@ import (
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"mesh-delivery-checker/internal/check"
)
@@ -54,36 +53,10 @@ func read(s stdio.KeptState) check.StateRead {
}}
}
// runsOf says which commit the mesh runs of a module from the indexer's modules state, as the checker's own record of it does
// (the commit the module's record was computed at); "" when the record is not readable.
type runsOf struct{ modules check.StateRead }
func (r runsOf) CommitOf(module string) string {
if r.modules.Get == nil {
return ""
}
raw, err := r.modules.Get(module)
if err != nil || len(raw) == 0 {
return ""
}
m, err := delivery.DecodeModule(raw)
if err != nil {
return ""
}
return m.Commit
}
// stateStore is a state the tools write: the tries.
type stateStore struct {
check.StateRead
create func(key string, value []byte) error
entry func(key string) (json.RawMessage, uint64, error)
update func(key string, value []byte, revision uint64) error
}
func (s stateStore) Entry(key string) (json.RawMessage, uint64, error) { return s.entry(key) }
func (s stateStore) Update(key string, value []byte, revision uint64) error {
return s.update(key, value, revision)
}
func (s stateStore) Keys() ([]string, error) { return s.StateRead.Keys() }
@@ -93,21 +66,19 @@ func (s stateStore) Create(key string, value []byte) error { return s.create(k
// Trying is what the try's tools work on: the tries, and the gate's trunk heads.
type Trying struct {
Tries check.TryStore
Ender check.TryEnder
Trunks func() (map[string]string, error)
Now func() time.Time
}
// TryTools are the try's tools (ADR 0501 §5): ask one, read one, and end one by hand (issue 734). mesh-controller's try calls them.
// TryTools are the try's two tools (ADR 0501 §5): ask one, read one. mesh-controller's try calls them.
func TryTools(tr Trying) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return s }
return []stdio.Tool{
{Name: check.TryTool,
Description: "Try a pushed head before its pull request is opened: the checker judges it as it judges a pull request's head, merged onto its trunk's head, " +
"its unit and integration levels through the build modules, and owes the forge nothing. Answers the try's id at once; " + check.TryReadTool + " reads " +
"where it stands. A try of the same head that waits or runs is the answer to a second ask; a new try is refused while 4 tries run on a build node or 8 wait for one, mesh-wide; tries let in while waiting may later all run at once, and each node's build capacity bounds what runs on it. A repository " +
"the checker checks no level of (one whose trunk the gate does not name) is refused. A try whose ask no build node took within the step " +
"bound ends as error; its ask stays in the build seat's stream and may still run later, judged by nobody (issue 734).",
"where it stands. A try of the same head that waits or runs is the answer to a second ask; at most 4 wait or run at once, mesh-wide. A repository " +
"the checker checks no level of (one whose trunk the gate does not name) is refused.",
Input: map[string]any{"type": "object", "required": []string{"repository", "head"}, "properties": map[string]any{
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
"head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits"},
@@ -135,8 +106,7 @@ func TryTools(tr Trying) []stdio.Tool {
}},
{Name: check.TryReadTool,
Description: "Where a try stands, as stored: queued, running, or its end (pass, fail, or error when it could not run), with what was said of it, " +
"the head and the trunk commit it was merged onto, the time it took and what failed first; while it runs, each ask it put as soon as it is put, " +
"with the node that took it, its state and the phase its step says (fetch and merge, a level's digest, lint, tests) and since when. Read-only.",
"the head and the trunk commit it was merged onto, the time it took and what failed first. Read-only.",
Input: map[string]any{"type": "object", "required": []string{"id"}, "properties": map[string]any{
"id": map[string]any{"type": "string", "description": "the try's id, try-<digits>"}}},
Run: func(args map[string]any) (any, error) {
@@ -145,28 +115,6 @@ func TryTools(tr Trying) []stdio.Tool {
}
return check.ReadTry(tr.Tries, str(args, "id"))
}},
{Name: check.TryEndTool,
Description: "End a try that waits or runs, as error, by hand: say why (kept in what it says). Its place among the 4 is free at once, " +
"the checker stops waiting for its asks, and the next " + check.TryTool + " of its head asks anew with a new id. For a try whose ask " +
"was lost (no step runs for it in checker_status); the checker ends one by itself when the build seat gives its ask up, or no node " +
"takes it within the step bound. A try that ended already is refused.",
Input: map[string]any{"type": "object", "required": []string{"id", "why"}, "properties": map[string]any{
"id": map[string]any{"type": "string", "description": "the try's id, try-<digits>"},
"why": map[string]any{"type": "string", "description": "why it is ended, in words"}}},
Run: func(args map[string]any) (any, error) {
if tr.Ender == nil {
return nil, fmt.Errorf("the tries state cannot be written here")
}
now := time.Now
if tr.Now != nil {
now = tr.Now
}
t, err := check.EndTry(tr.Ender, str(args, "id"), str(args, "why"), now())
if err != nil {
return nil, err
}
return map[string]any{"id": t.ID, "state": t.State, "said": t.Said}, nil
}},
}
}
@@ -226,10 +174,7 @@ func Tools(r check.Reader) []stdio.Tool {
"and the candidates whose check is owed by kind (waiting_by_kind: new, nodes, and trunk, a re-judge pulled after the other two). A key a " +
"build module left running (started past the step bound, or its build module not running on its node) is listed apart under " +
"left_running with why, and not counted. Nothing holds the checker and it has no capacity of its own: a check is an ask on the " +
"build seats' streams, whose backlog the delivery view shows (novox/hq ADR 0525). held_by_turn names the candidates whose check an approved " +
"merge's turn holds off (novox/hq ADR 0548 §3), with the repository and the merge that waits; the gate's turns tool reads every turn and its " +
"hand-now tool steers one. dead_letters says whether the checker hears the asks the build seats give up on (issue 734), and why not: " +
"deaf, a try or check whose ask is given up waits until its bound. Read-only.",
"build seats' streams, whose backlog the delivery view shows (novox/hq ADR 0525). Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) { return r.Status() }},
}
@@ -248,9 +193,7 @@ func main() {
r := check.Reader{VerdictState: read(stdio.State("verdicts")), KeptState: read(stdio.State("kept")), CandidateState: read(stdio.State("mesh-delivery-indexer.candidates")),
PreviewState: read(stdio.State("previews")),
StepStates: map[string]check.StateRead{"mesh-build-go": read(stdio.State("mesh-build-go.steps")), "mesh-build-node": read(stdio.State("mesh-build-node.steps"))},
HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound(),
TurnState: read(stdio.State("mesh-delivery-gate.turns")), HearingState: read(stdio.State(buildrun.HearingState))}
r.Runs = runsOf{read(stdio.State("mesh-delivery-indexer.modules"))} // the node-engine at the commit the mesh runs, as the checker reads it
HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound()}
tries := stdio.State(check.TriesState)
source := stdio.State("mesh-delivery-gate.source")
trunks := func() (map[string]string, error) {
@@ -262,21 +205,13 @@ func main() {
return s.Heads, err
}
r.Trunks = trunks
store := stateStore{StateRead: read(tries), create: func(key string, value []byte) error {
_, err := tries.Create(key, json.RawMessage(value))
return err
}, entry: func(key string) (json.RawMessage, uint64, error) {
reads.Add(1)
e, err := tries.Get(key)
if err != nil || e == nil {
return nil, 0, err
}
return e.Value, e.Revision, nil
}, update: func(key string, value []byte, revision uint64) error {
_, err := tries.Update(key, json.RawMessage(value), revision)
return err
}}
trying := Trying{Tries: store, Ender: store, Trunks: trunks}
trying := Trying{
Tries: stateStore{StateRead: read(tries), create: func(key string, value []byte) error {
_, err := tries.Create(key, json.RawMessage(value))
return err
}},
Trunks: trunks,
}
if err := stdio.Serve("", logged(append(Tools(r), TryTools(trying)...))); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err)
os.Exit(1)
@@ -53,19 +53,13 @@ func (m *memStore) Create(key string, value []byte) error {
return nil
}
func (m *memStore) Entry(key string) (json.RawMessage, uint64, error) { return m.vals[key], 1, nil }
func (m *memStore) Update(key string, value []byte, revision uint64) error {
m.vals[key] = value
return nil
}
func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed interface with mesh-controller's try
store := &memStore{vals: map[string]json.RawMessage{}}
tools := TryTools(Trying{Tries: store, Ender: store, Now: func() time.Time { return time.Unix(1, 5) },
tools := TryTools(Trying{Tries: store, Now: func() time.Time { return time.Unix(1, 5) },
Trunks: func() (map[string]string, error) {
return map[string]string{"novox/mesh-delivery": strings.Repeat("a", 40)}, nil
}})
if len(tools) != 3 || tools[0].Name != "checker_try" || tools[1].Name != "checker_try_read" || tools[2].Name != "checker_try_end" {
if len(tools) != 2 || tools[0].Name != "checker_try" || tools[1].Name != "checker_try_read" {
t.Fatalf("tools = %+v", tools)
}
out, err := tools[0].Run(map[string]any{"repository": "novox/mesh-delivery", "head": strings.Repeat("1", 40), "trunk": "main"})
@@ -80,14 +74,6 @@ func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed i
if err != nil || !strings.Contains(string(raw.(json.RawMessage)), `"state":"queued"`) {
t.Errorf("read = %s, %v", raw, err)
}
// Issue 734: a try whose ask was lost is ended by hand with why, and its head is asked anew.
ended, err := tools[2].Run(map[string]any{"id": id, "why": "its ask was lost"})
if err != nil || ended.(map[string]any)["state"] != "error" {
t.Errorf("end = %v, %v", ended, err)
}
if _, err := tools[2].Run(map[string]any{"id": id, "why": "again"}); err == nil {
t.Error("an ended try was ended again")
}
if _, err := tools[0].Run(map[string]any{"repository": "novox/hq", "head": strings.Repeat("1", 40)}); err == nil ||
err.Error() != "no level of novox/hq is checked by the checker" {
t.Errorf("hq: %v", err)
@@ -24,8 +24,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/announce"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-checker/internal/check"
)
@@ -81,8 +79,6 @@ type fakeBuildModules struct {
js jetstream.JetStream
ctx context.Context
conflict atomic.Bool
// hold is a commit whose fetch never answers: its ask stays taken, as one lost in a bus reload (issue 734).
hold atomic.Value
mu sync.Mutex
calls map[string]int // by `<module>.<tool>`
testTool *testing.T
@@ -102,9 +98,6 @@ func (f *fakeBuildModules) result(t *testing.T, tool string, args map[string]any
var v any
switch {
case strings.HasSuffix(tool, "_fetch"):
if h, _ := f.hold.Load().(string); h != "" && args["commit"] == h {
<-f.ctx.Done()
}
name := "go-fetch-clean"
if f.conflict.Load() {
name = "go-fetch-conflict"
@@ -305,10 +298,6 @@ func startBed(t *testing.T) *bed {
}
b.tools[module] = kv
}
// What the controller keeps of the messages a consumer gave up on (issue 734).
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buildrun.DeadLettersStream, Subjects: []string{"mesh.events.dead.>"}}); err != nil {
t.Fatal(err)
}
// The checker's stream of events, which the indexer and the forge bridge hear.
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "CHECKS", Subjects: []string{"mesh.stream.mesh-delivery-checker.event.>"}}); err != nil {
t.Fatal(err)
@@ -316,8 +305,7 @@ func startBed(t *testing.T) *bed {
for name, history := range map[string]int{delivery.CheckerVerdictsBucket: delivery.CheckerVerdictsHistory, delivery.CheckerKeptBucket: delivery.CheckerKeptHistory,
delivery.CheckerOwedBucket: delivery.CheckerOwedHistory, delivery.PreviewsBucket: delivery.PreviewsHistory, delivery.CandidatesBucket: delivery.CandidatesHistory,
delivery.ModulesBucket: delivery.ModulesHistory, delivery.ManifestsBucket: delivery.ManifestsHistory, delivery.SourceBucket: delivery.SourceHistory,
delivery.PresenceBucket: delivery.PresenceHistory, delivery.HaveBucket: delivery.HaveHistory, check.TriesBucket: 1,
delivery.TurnsBucket: delivery.TurnsHistory} {
delivery.PresenceBucket: delivery.PresenceHistory, delivery.HaveBucket: delivery.HaveHistory, check.TriesBucket: 1} {
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: uint8(history)})
if err != nil {
t.Fatal(err)
@@ -424,12 +412,11 @@ func pull(repository string, number int, head string) delivery.CandidatePull {
func (b *bed) start(t *testing.T, env map[string]string) {
t.Helper()
broker := filepath.Join(t.TempDir(), "broker")
cred, _ := json.Marshal(process.Account{URL: b.srvURL, Node: "n1", Module: "mesh-delivery-checker", User: "n1.mesh-delivery-checker", Password: "pw"})
cred, _ := json.Marshal(credential{URL: b.srvURL, Node: "n1", Module: "mesh-delivery-checker", User: "n1.mesh-delivery-checker", Password: "pw"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
// The settle window of a group's heads (check.DefaultSettle) is cut short: each bed's candidate is written whole.
e := map[string]string{"MESH_BROKER_FILE": broker, "MESH_CHECKER_SETTLE": "10ms"}
e := map[string]string{"MESH_BROKER_FILE": broker}
for k, v := range env {
e[k] = v
}
@@ -520,12 +507,9 @@ func TestCheckerJudgesACandidateAndOwesItsStatus(t *testing.T) { // N6/AC5, 1300
}
}
// TestACheckOrACarryOfEqualInputsRunsNothing: a second candidate (another head, the same digests) is checked with every
// package taken from the first's kept verdicts: no test step runs and no second kept key appears. Then, through the real bus
// (novox/hq ADR 0548 §3, §5): a third candidate is not claimed while the gate's merge turn on its repository stands, and is
// claimed when the turn's key is deleted; and a move of the trunk carries the first verdict on the recheck stream by its
// unchanged digests, running no test, the verdict naming the trunk it was carried from.
func TestACheckOrACarryOfEqualInputsRunsNothing(t *testing.T) { // 1287/AC5, 1288/AC6, 1929/AC2, ADR 0548 §3, §5
// TestACheckOfEqualInputsRunsNothing: a second candidate (another head, the same digests) is checked with every package
// taken from the first's kept verdicts: no test step runs and no second kept key appears.
func TestACheckOfEqualInputsRunsNothing(t *testing.T) { // 1287/AC5, 1288/AC6, 1929/AC2
b := startBed(t)
first := b.candidate(t, "feat/x", pull(repo, 7, head1))
b.start(t, nil)
@@ -542,42 +526,6 @@ func TestACheckOrACarryOfEqualInputsRunsNothing(t *testing.T) { // 1287/AC5, 128
if got := keptCount(t, b); got != kept {
t.Errorf("%d kept keys, was %d", got, kept)
}
// §3: the gate's turn on the repository holds a new check off; its delete claims it.
at := time.Now().UTC()
turn, err := delivery.EncodeTurn(delivery.Turn{Repository: repo, Waiting: []delivery.TurnWait{{Group: "fix/other", Heads: "0123456789ab", Approved: at, Since: at}}})
if err != nil {
t.Fatal(err)
}
tk, _ := delivery.TurnsKey(repo)
if _, err := b.buckets[delivery.TurnsBucket].Put(b.ctx, tk, turn); err != nil {
t.Fatal(err)
}
time.Sleep(300 * time.Millisecond) // the checker hears the turn
third := b.candidate(t, "feat/z", pull(repo, 9, "3333333333333333333333333333333333333333"))
time.Sleep(time.Second)
if _, err := b.buckets[delivery.CheckerVerdictsBucket].Get(b.ctx, third); !errors.Is(err, jetstream.ErrKeyNotFound) {
t.Fatalf("verdicts.%s exists (%v) while a merge turn holds its repository", third, err)
}
if err := b.buckets[delivery.TurnsBucket].Delete(b.ctx, tk); err != nil {
t.Fatal(err)
}
b.verdict(t, third, ended)
// §5: the trunk moves; the first verdict is carried by its digests, nothing runs.
tests = b.fake.count("mesh-build-go.build_go_test")
const moved = "abababababababababababababababababababab"
b.setTrunk(t, map[string]string{repo: moved})
c := b.verdict(t, first, func(v delivery.Verdict) bool { return ended(v) && v.Trunk[repo] == moved })
if c.State != delivery.VerdictPass || c.CarriedFrom[repo] != trunk || c.Levels[delivery.LevelUnit].Run != 0 || c.Levels[delivery.LevelIntegration].Run != 0 {
t.Fatalf("the carried verdict = %+v, want a pass carried from %s with no test run", c, trunk[:12])
}
if got := b.fake.count("mesh-build-go.build_go_test"); got != tests {
t.Errorf("build_go_test called %d times on the carry, was %d: a carry of unchanged digests runs nothing", got, tests)
}
if o := b.owed(t, 7, head1, "success"); !strings.HasPrefix(o.Text, "pass 111111111111 on abababababab | carried from aaaaaaaaaaaa") {
t.Errorf("the carried status says %q", o.Text)
}
}
func keptCount(t *testing.T, b *bed) int {
@@ -728,49 +676,6 @@ func TestATryIsJudgedAndItsEndWrittenWithNoStatusOwed(t *testing.T) {
t.Errorf("a try wrote %s: %v", name, keys)
}
}
// Issue 734: a try whose ask the build seat gives up on (the controller keeps its copy in DEAD_LETTERS with the ask's
// message id) ends as error saying so, and its head asked again is a new try.
b.fake.hold.Store(head2)
lost, err := check.AskTry(store, map[string]string{repo: trunk}, check.TryAsk{Repository: repo, Head: head2}, time.Now())
if err != nil {
t.Fatal(err)
}
steps, err := b.js.KeyValue(b.ctx, delivery.StepsBucket("mesh-build-go"))
if err != nil {
t.Fatal(err)
}
askID := ""
for deadline := time.Now().Add(20 * time.Second); askID == "" && time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
keys, _ := steps.Keys(b.ctx)
for _, k := range keys {
if e, err := steps.Get(b.ctx, k); err == nil && strings.Contains(string(e.Value()), `"running"`) {
askID = strings.TrimPrefix(k, "n1.")
}
}
}
if askID == "" {
t.Fatal("the held try's ask was never taken")
}
dead := &nats.Msg{Subject: "mesh.events.dead.SEAT_NODE_BUILD_GO.SEAT_NODE_BUILD_GO_CHECK_worker", Header: nats.Header{}}
dead.Header.Set(buildrun.DeadHeaderMsgID, askID)
dead.Header.Set(buildrun.DeadHeaderDeliveries, "5")
dead.Header.Set(buildrun.DeadHeaderStream, "SEAT_NODE_BUILD_GO")
dead.Header.Set(buildrun.DeadHeaderGaveUp, time.Now().UTC().Format(time.RFC3339Nano))
if _, err := b.js.PublishMsg(b.ctx, dead); err != nil {
t.Fatal(err)
}
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
raw, _ := store.Get(lost)
if last, err = check.DecodeTry(raw); err == nil && last.Over() {
break
}
}
if last.State != check.TryError || !strings.Contains(last.Said, buildrun.LostWords+" after 5 deliveries") {
t.Fatalf("the try whose ask was given up = %+v, want error saying so", last)
}
if again, err := check.AskTry(store, map[string]string{repo: trunk}, check.TryAsk{Repository: repo, Head: head2}, time.Now()); err != nil || again == lost {
t.Errorf("its head asked again answered %q, %v, want a new try", again, err)
}
}
// TestTheCheckerSaysOwedReadableOnTheBus: started, the checker says on its own states subject what it keeps, owed among
@@ -797,7 +702,7 @@ func TestTheCheckerSaysOwedReadableOnTheBus(t *testing.T) { // R2-1
break
}
if said.Module != "mesh-delivery-checker" || said.Node != "n1" || !said.Keeps("owed") || !said.Readable("owed") ||
!said.Readable("previews") || !said.Readable("verdicts") || !said.Readable("kept") || !said.Readable("tries") || said.Readable("hearing") || len(said.States) != 6 {
t.Fatalf("the checker said %+v, want its six states with kept, owed, previews, tries and verdicts readable, tries by the delivery view (1929/AC4; 1887 A5b; build_usage)", said)
!said.Readable("previews") || !said.Readable("verdicts") || !said.Readable("kept") || said.Readable("tries") || len(said.States) != 5 {
t.Fatalf("the checker said %+v, want its five states with kept, owed, previews and verdicts readable (1929/AC4; 1887 A5b)", said)
}
}
@@ -37,8 +37,8 @@ func TestTheCheckerAnnouncesEveryStateItKeepsAndOwedReadable(t *testing.T) { //
if !equal(manifest, announced) {
t.Errorf("announced %v, the manifest keeps %v", announced, manifest)
}
// kept is read by the Go build module for a check ask (issue 1887 design limit 1); tries by the delivery view (build_usage).
if want := []string{"kept", "owed", "previews", "tries", "verdicts"}; !equal(readable, want) {
// kept is read by the Go build module for a check ask (issue 1887 design limit 1).
if want := []string{"kept", "owed", "previews", "verdicts"}; !equal(readable, want) {
t.Errorf("readable %v, want %v", readable, want)
}
}
@@ -1,50 +0,0 @@
package main
import (
"sync"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-checker/internal/check"
)
// lazyTurns is the gate's merge turns as the checker reads them (novox/hq ADR 0548 §3), opened when the gate's bucket is
// there: before each claim it is opened again until it is, so a checker that started before the gate made the bucket holds
// checks from the gate's first turn on, with no timer and no restart. Until then it holds nothing, and says so once, loudly,
// and once again when it opens.
type lazyTurns struct {
turns *check.Turns
open func() error
logf func(format string, args ...any)
mu sync.Mutex
opened bool
said bool
}
// ensure opens the bucket once it is there; a failure is said once.
func (l *lazyTurns) ensure() {
l.mu.Lock()
defer l.mu.Unlock()
if l.opened || l.open == nil {
return
}
if err := l.open(); err != nil {
if !l.said {
l.said = true
l.logf("NO MERGE TURN HOLDS A CHECK: the gate's bucket %s cannot be read (%v); every check is claimed as if no approved merge waited, "+
"and the bucket is opened again before each claim until it is there", delivery.TurnsBucket, err)
}
return
}
l.opened = true
if l.said {
l.logf("the gate's merge turns are read from %s from now on: an approved merge's turn holds checks off its repositories again", delivery.TurnsBucket)
}
}
// Holding opens the bucket when it is not yet, then answers what the turns hold.
func (l *lazyTurns) Holding(repositories []string, heads string) (string, delivery.TurnWait, bool) {
l.ensure()
return l.turns.Holding(repositories, heads)
}
@@ -7,7 +7,12 @@ package main
import (
"context"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"log"
@@ -31,7 +36,7 @@ import (
// eventSubject is the subject of an event of the checker's stream checks (the bus names the stream CHECKS): the indexer's
// watch of the previews bucket and the forge bridge's watch of the owed bucket are woken by them.
func eventSubject(event string) string {
return process.ModuleStreamSubject(delivery.CheckerModule, event)
return "mesh.stream." + delivery.CheckerModule + ".event." + event
}
const (
@@ -48,6 +53,50 @@ func main() {
}
}
// credential is the broker file the controller issues to the module: JSON of the bus address, the certificate
// fingerprint to pin, the principal and its password (as the indexer reads it; issue 1242 moves this into the SDK).
type credential struct {
URL string `json:"url"`
Fingerprint string `json:"fingerprint"`
Node string `json:"node"`
Module string `json:"module"`
User string `json:"user"`
Password string `json:"password"`
}
func readCredential(path string) (credential, error) {
raw, err := os.ReadFile(path)
if err != nil {
return credential{}, err
}
var c credential
if err := json.Unmarshal(raw, &c); err != nil {
return credential{}, fmt.Errorf("the broker credential is not one: %w", err)
}
if c.URL == "" || c.User == "" {
return credential{}, errors.New("the broker credential names no bus address or user")
}
return c, nil
}
// pinned accepts exactly the certificate with this SHA-256 and no other.
func pinned(want string) *tls.Config {
return &tls.Config{
InsecureSkipVerify: true, //nolint:gosec // replaced by the pin below, which is stricter
MinVersion: tls.VersionTLS12,
VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
if len(rawCerts) == 0 {
return errors.New("the bus presented no certificate")
}
sum := sha256.Sum256(rawCerts[0])
if got := "sha256:" + hex.EncodeToString(sum[:]); got != want {
return fmt.Errorf("the bus presented a certificate this mesh does not know (%.23s…)", got)
}
return nil
},
}
}
// durationEnv reads a duration setting from the environment, def when it is not set.
func durationEnv(getenv func(string) string, name string, def time.Duration) (time.Duration, error) {
v := strings.TrimSpace(getenv(name))
@@ -63,31 +112,46 @@ func durationEnv(getenv func(string) string, name string, def time.Duration) (ti
// run is the program: it returns when ctx ends or the bus cannot be used.
// keptStates is every state the checker keeps and which others may read (ADR 0495 §2-§3): owed by the forge module,
// which writes mesh/check from it (ADR 0521), and by the forge bridge; previews by the indexer; verdicts by the gate; tries by the delivery view.
// which writes mesh/check from it (ADR 0521), and by the forge bridge; previews by the indexer; verdicts by the gate.
var keptStates = []announce.State{
// The kept verdicts are read by the Go build module, which takes them for a check ask's unchanged packages (issue 1887's
// design limit 1); the checker alone writes them (ADR 0444 clause 5).
// Whether DEAD_LETTERS is heard (issue 734): read by the checker's own status tool alone.
{Name: "hearing"},
{Name: "kept", Readable: true},
{Name: "owed", Readable: true},
{Name: "previews", Readable: true},
// The tries are read by the delivery view, which tells a try's asks from a candidate's by the asks each names (build_usage).
{Name: "tries", Readable: true},
{Name: "tries"},
{Name: "verdicts", Readable: true},
}
func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
logger := log.New(logOut, "mesh-delivery-checker: ", log.LstdFlags)
cred, err := process.ReadBrokerAccount(getenv)
path := strings.TrimSpace(getenv("MESH_BROKER_FILE"))
if path == "" {
return errors.New("MESH_BROKER_FILE is not set, so there is no credential to hear the bus with")
}
cred, err := readCredential(path)
if err != nil {
return err
}
conn, js, err := cred.Dial()
opts := []nats.Option{
nats.Name(cred.Node + "." + cred.Module),
nats.UserInfo(cred.User, cred.Password),
nats.CustomInboxPrefix("_INBOX." + cred.User),
nats.MaxReconnects(-1),
nats.RetryOnFailedConnect(true),
}
if strings.TrimSpace(cred.Fingerprint) != "" {
opts = append(opts, nats.Secure(pinned(cred.Fingerprint)))
}
conn, err := nats.Connect(cred.URL, opts...)
if err != nil {
return err
return fmt.Errorf("connecting to the bus: %w", err)
}
defer conn.Close()
js, err := jetstream.New(conn)
if err != nil {
return err
}
// Its own states, the candidates it checks, and the modules and manifests that say which build modules judge a repository:
// a check without one of them would be wrong or unwritten, so each stops the start, naming its grant (issue 1945).
buckets := map[string]jetstream.KeyValue{}
@@ -99,7 +163,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
// Calls to the build modules go out as this module's own account, naming the machine (mesh-sdk process.CallOn); the
// buckets are read and written on the connection above.
caller, err := process.Connect(cred, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
account, err := process.ReadAccount(path)
if err != nil {
return err
}
caller, err := process.Connect(account, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
if err != nil {
return err
}
@@ -118,10 +186,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil {
return err
}
settleWindow, err := durationEnv(getenv, "MESH_CHECKER_SETTLE", check.DefaultSettle)
if err != nil {
return err
}
link := strings.TrimSpace(getenv("MESH_CHECKER_LINK"))
if link == "" {
link = defaultLink
@@ -129,32 +193,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
tools := &buildrun.ToolchainSet{}
presence := &buildrun.PresenceSet{}
trunks := &check.Trunks{}
turns := &check.Turns{}
lazy := &lazyTurns{turns: turns, logf: logger.Printf}
// A check is one ask per pull request and build module on the build seats' check stream, pulled by a build node with a
// free slot (novox/hq ADR 0525 §1, §3): the checker chooses no node and starts no step.
asks := buildrun.JetStreamAsks{JS: js}
// The asks the build seats gave up on (issue 734): heard from DEAD_LETTERS, each ends the step its asker waits for.
// Heard in the background, and again at a reconnect or an ask while it is not; whether it is heard is kept in the
// state hearing, which checker_status says (dead_letters).
dead := &buildrun.DeadLetters{}
// The state may not be on the bus yet when a delivery started the checker (the grants part creates it, issue 1790): it
// is bound when the bus says it was created, and the latest hearing is written then.
hearingKV := process.AwaitBucket(ctx, js, conn, check.HearingBucket, "checker_status cannot say whether DEAD_LETTERS is heard", logger.Printf)
hearing := buildrun.NewDeadLetterHearing(ctx, buildrun.JetStreamDeadLetters{JS: js}, []string{delivery.AskGo, delivery.AskNode}, dead,
logger.Printf, buildrun.SayHearing(ctx, hearingKV, logger.Printf))
defer hearing.Stop()
conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again() })
waiter := buildrun.LostAskWaiter{Inner: asks, Dead: dead, Hearing: hearing}
ch := &check.Checker{
Verdicts: kvStore{buckets[delivery.CheckerVerdictsBucket]},
Owed: kvStore{buckets[delivery.CheckerOwedBucket]},
Trunks: trunks,
Turns: lazy,
Bound: checkBound,
Link: link,
Logf: logger.Printf,
Settle: settleWindow,
}
// The builder's builds say how the candidates' fingerprints stand (issue 1684); where its bucket is not there yet the line
// says nothing of the builds, until the checker starts again.
@@ -180,7 +228,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return cw, nil
}}
// The tries (ADR 0501 §5) are judged by the same judge, held by the same deliveries.
tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, NotTaken: stepBound, Logf: logger.Printf}
tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, Logf: logger.Printf}
announce := func(ctx context.Context, event, key string) {
body, _ := json.Marshal(map[string]string{"key": key})
if _, err := js.Publish(ctx, eventSubject(event), body); err != nil {
@@ -188,9 +236,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
}
repos := &repoModules{manifests: kvStore{buckets[delivery.ManifestsBucket]}, logf: logger.Printf}
ch.Runs = repos // the node-engine a check reads beside at its running commit (ADR 0548 §6)
ch.Judge = newJudge(judgeDeps{
asks: buildrun.Asker{Publisher: asks, Waiter: waiter, Bound: stepBound}, tools: tools, trunks: trunks,
asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: stepBound}, tools: tools, trunks: trunks,
kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]},
repos: repos, builds: builds, forge: forgeReads{call: caller.Call}, logf: logger.Printf,
announce: announce, compositions: compositions,
@@ -216,12 +263,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// module.
for _, module := range []string{buildrun.GoBuildModule, buildrun.NodeBuildModule} {
stream := toolchainStreams[module]
cons, err := cred.EventConsumer(ctx, js, stream)
cons, err := js.Consumer(ctx, stream, cred.Node+"_"+cred.Module)
if err != nil {
logger.Printf("the consumer %s_%s on %s does not exist yet, so %s's reports are not heard until the checker starts again: %v", cred.Node, cred.Module, stream, module, err)
continue
}
cc, err := process.HearEvents(cons, func(m jetstream.Msg) { watch.Reported(m) }, logger.Printf)
cc, err := cons.Consume(func(m jetstream.Msg) { watch.Reported(m) },
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
if err != nil {
return err
}
@@ -265,42 +313,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}()
}
}
// The gate's merge turns (novox/hq ADR 0548 §3): no check is claimed of a repository an approved merge waits on, and the
// delete of its turn claims it. Read before the candidates, so no check starts past a turn written before this start.
// Where the gate's bucket is not there yet it is read as empty, and opened again before each claim until it is there
// (lazyTurns), so a gate delivered after the checker started holds checks from its first turn on.
lazy.open = func() error {
kv, err := js.KeyValue(ctx, delivery.TurnsBucket)
if err != nil {
return err
}
tw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { turns.Put(e.Key(), e.Value(), !opOf(e)) })
if err != nil {
return err
}
go func() {
errs <- follow(ctx, tw, func(e jetstream.KeyValueEntry) {
if turns.Put(e.Key(), e.Value(), !opOf(e)) {
ch.Reconsider(ctx) // a turn went or changed: a check it held is claimed now
}
})
}()
return nil
}
lazy.ensure()
// The modules say which repository holds a TypeScript module, so the Node build module judges it too.
mw, err := settle(ctx, buckets[delivery.ModulesBucket], func(e jetstream.KeyValueEntry) { repos.Put(ctx, e.Key(), e.Value(), !opOf(e)) })
if err != nil {
return err
}
go func() {
errs <- follow(ctx, mw, func(e jetstream.KeyValueEntry) {
before := repos.CommitOf(check.NodeEngineModule)
repos.Put(ctx, e.Key(), e.Value(), !opOf(e))
if after := repos.CommitOf(check.NodeEngineModule); after != before {
ch.TrunkMoved(ctx) // the node-engine the mesh runs moved: a verdict that read it beside is carried (ADR 0548 §6)
}
})
errs <- follow(ctx, mw, func(e jetstream.KeyValueEntry) { repos.Put(ctx, e.Key(), e.Value(), !opOf(e)) })
}()
each := func(e jetstream.KeyValueEntry) {
if opOf(e) {
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-checker
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026095702-79649c3fa9f4
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952 h1:TmveT3Bija643RGWkilvkzP/h8UWlVvHFZZtQfJK5oU=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026095702-79649c3fa9f4 h1:05sVel/UOvz4/g0dxrDOeVUL7BGakVrxPBlqRq02ZhM=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026095702-79649c3fa9f4/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -1,355 +0,0 @@
package check
import (
"context"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// One check per delivery (novox/hq ADR 0548): the checker's part. A merge turn holds new checks off its repository (§3);
// a trunk or sibling move never ends a running check and carries its verdict once it ended (§1, §5, §6); only a head git
// refused gets a whole check after a move (§7); the line of a carried verdict says so (§8).
// turnOf is a turn's value on a repository for a delivery of the heads hash.
func turnOf(t testing.TB, repo, group, heads string) []byte {
t.Helper()
at := time.Date(2026, 10, 26, 9, 0, 0, 0, time.UTC)
data, err := delivery.EncodeTurn(delivery.Turn{Repository: repo, Waiting: []delivery.TurnWait{{Group: group, Heads: heads, Approved: at, Since: at}}})
if err != nil {
t.Fatal(err)
}
return data
}
func turnKey(t testing.TB, repo string) string {
t.Helper()
k, err := delivery.TurnsKey(repo)
if err != nil {
t.Fatal(err)
}
return k
}
// waitFor polls a condition the checker's goroutines make true, failing after two seconds.
func waitFor(t testing.TB, what string, ok func() bool) {
t.Helper()
for deadline := time.Now().Add(2 * time.Second); !ok(); time.Sleep(time.Millisecond) {
if time.Now().After(deadline) {
t.Fatalf("%s did not happen", what)
}
}
}
// pullWithFP is a pull request touching the module m at the fingerprint `src2:` and 64 times the hex digit d.
func pullWithFP(repo string, number int, head string, d string) delivery.CandidatePull {
p := pullOf(repo, number, head)
p.Modules = []string{"m"}
p.Fps = map[string]delivery.CandidateModule{"m": {FP2: "src2:" + strings.Repeat(d, 64)}}
return p
}
func TestOneCheckPerDelivery(t *testing.T) {
ctx := context.Background()
for name, run := range map[string]func(t *testing.T){
// §3: no claim while a turn names one of its repositories; a claim on the turn's delete event.
"a turn on its repository holds a new check off until the turn goes": func(t *testing.T) {
j := passJudge()
r := newRig(t, j)
turns := &Turns{}
r.ch.Turns = turns
turns.Put(turnKey(t, repoA), turnOf(t, repoA, "fix/other", "0123456789ab"), false)
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
if j.count() != 0 || len(r.verdicts.vals) != 0 {
t.Fatalf("judged %d times with a turn on %s, verdicts %v", j.count(), repoA, r.verdicts.vals)
}
if held := r.ch.Held()[keyOf(t, "feat/x", cand)]; !strings.Contains(held, repoA) || !strings.Contains(held, "fix/other") {
t.Errorf("the hold is said as %q, want the turn's repository and group", held)
}
if !turns.Put(turnKey(t, repoA), nil, true) {
t.Fatal("the turn's delete changed nothing")
}
r.ch.Reconsider(ctx) // what the watch of turns does on the delete
r.ch.Wait()
if j.count() != 1 || r.verdict(t, keyOf(t, "feat/x", cand)).State != delivery.VerdictPass {
t.Errorf("judged %d times after the turn went, want once", j.count())
}
},
"a turn on another repository, or of its own heads, holds nothing": func(t *testing.T) {
j := passJudge()
r := newRig(t, j)
turns := &Turns{}
r.ch.Turns = turns
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
c, _ := delivery.DecodeCandidate(cand)
turns.Put(turnKey(t, repoSDK), turnOf(t, repoSDK, "fix/other", "0123456789ab"), false)
turns.Put(turnKey(t, repoA), turnOf(t, repoA, "feat/x", GateHeads(c)), false)
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
if j.count() != 1 {
t.Errorf("judged %d times, want once: the turns hold another repository and its own merge", j.count())
}
},
// §3, the race: a check claimed before the gate's turn is judged on the commit it read, never restarted.
"a check claimed before a turn was written runs to its verdict": func(t *testing.T) {
release, started := make(chan struct{}), make(chan struct{})
j := &judgeFunc{f: func(context.Context, *Case, int) (Outcome, error) {
close(started)
<-release
return Outcome{Verdict: passing()}, nil
}}
r := newRig(t, j)
turns := &Turns{}
r.ch.Turns = turns
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
<-started
turns.Put(turnKey(t, repoA), turnOf(t, repoA, "fix/other", "0123456789ab"), false)
r.ch.Reconsider(ctx)
close(release)
r.ch.Wait()
if j.count() != 1 || r.verdict(t, keyOf(t, "feat/x", cand)).State != delivery.VerdictPass {
t.Errorf("judged %d times; the running check is not ended by the turn", j.count())
}
},
// §1, §5: a trunk move during a running check calls the judge once, writes its verdict, keeps the status pending (a pass
// on an old trunk is never shown), then asks one carry, on the recheck stream, and no check.
"a trunk move during a check ends nothing and asks one carry after it": func(t *testing.T) {
var r *rig
var kinds []string
var from []map[string]string
judged := make(chan struct{})
j := &judgeFunc{f: func(ctx context.Context, c *Case, call int) (Outcome, error) {
kinds, from = append(kinds, c.Kind), append(from, c.CarriedFrom)
if call == 1 {
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) // the trunk moves while it runs
r.ch.TrunkMoved(ctx) // as the watch of the source says it
close(judged)
} else if s := owedState(t, r, repoA, 7, head1); s != "pending" {
t.Errorf("the status during the carry is %q, want pending: the pass on the old trunk is never shown", s)
}
return Outcome{Verdict: passing()}, nil
}}
r = newRig(t, j)
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
<-judged
r.ch.Wait()
v := r.verdict(t, keyOf(t, "feat/x", cand))
if j.count() != 2 || kinds[0] != KindNew || kinds[1] != KindTrunk || from[0] != nil || from[1][repoA] != trunkA {
t.Fatalf("judged %d times as %v carried from %v; want a whole check, then one carry from %s", j.count(), kinds, from, trunkA[:12])
}
if v.State != delivery.VerdictPass || v.Trunk[repoA] != trunkA2 || v.CarriedFrom[repoA] != trunkA {
t.Errorf("the verdict is %s on %v carried from %v; want a pass on the new trunk carried from the old", v.State, v.Trunk, v.CarriedFrom)
}
if s := owedState(t, r, repoA, 7, head1); s != "success" {
t.Errorf("the status after the carry is %q, want success", s)
}
if r.verdicts.creates[keyOf(t, "feat/x", cand)] != 1 {
t.Error("the verdict key was made again: a carry rewrites it")
}
},
// §5: the composition is kept when no fingerprint of a module it composed changed, and runs again otherwise.
"a carry keeps a passed composition while the fingerprints stay the same": func(t *testing.T) {
var kept []bool
j := &judgeFunc{f: func(_ context.Context, c *Case, _ int) (Outcome, error) {
kept = append(kept, c.Composition != nil)
v := passing()
v.Composition = &delivery.Composition{State: delivery.CompositionPass, SDK: "v0.1.14-x", FPSet: c.FPSet}
return Outcome{Verdict: v}, nil
}}
r := newRig(t, j)
cand := candidateOf(t, "feat/x", pullWithFP(repoA, 7, head1, "a"))
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS})
r.ch.TrunkMoved(ctx) // the indexer's fingerprints of the candidate stay the same
r.ch.Wait()
r.ch.Candidate(ctx, "feat%2Fx", candidateOf(t, "feat/x", pullWithFP(repoA, 7, head1, "b"))) // a fingerprint changed with the move
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS})
r.ch.TrunkMoved(ctx)
r.ch.Wait()
if len(kept) != 3 || kept[0] || !kept[1] || kept[2] {
t.Errorf("the composition was kept in the checks %v; want composed, kept on the carry, composed again once a fingerprint changed", kept)
}
},
// §6: a sibling's move puts the verdict out of date, its status pending saying it is carried, and the carry's verdict
// names the sibling's old commit; the status follows the carry's verdict. Which packages run is the digest's (only
// those that name MESH_CHECK_BESIDE change with a sibling), the build module's part.
"a sibling's move is carried and the status follows the carry": func(t *testing.T) {
var r *rig
var from map[string]string
j := &judgeFunc{f: func(_ context.Context, c *Case, call int) (Outcome, error) {
if call == 2 {
from = c.CarriedFrom
if s := owedState(t, r, repoA, 7, head1); s != "pending" {
t.Errorf("the status during the sibling's carry is %q, want pending", s)
}
}
return Outcome{Verdict: passing(), Beside: map[string]string{catalogueRepository: r.trunks.Heads()[catalogueRepository]}}, nil
}}
r = newRig(t, j)
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS, catalogueRepository: trunkS})
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
if b := r.verdict(t, keyOf(t, "feat/x", cand)).Beside; b[catalogueRepository] != trunkS {
t.Fatalf("the verdict names the siblings %v, want the catalogue at %s", b, trunkS[:12])
}
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS, catalogueRepository: trunkS2})
r.ch.TrunkMoved(ctx)
r.ch.Wait()
v := r.verdict(t, keyOf(t, "feat/x", cand))
if j.count() != 2 || from[catalogueRepository] != trunkS || v.CarriedFrom[catalogueRepository] != trunkS || v.Beside[catalogueRepository] != trunkS2 {
t.Errorf("judged %d times, carried from %v, verdict beside %v carried from %v; want one carry over the catalogue's move", j.count(), from, v.Beside, v.CarriedFrom)
}
if s := owedState(t, r, repoA, 7, head1); s != "success" {
t.Errorf("the status after the carry is %q, want success", s)
}
},
// §7: a carry whose merge conflicts fails naming the files and asks no check; a later move onto which the head merges
// cleanly asks a whole check, not a carry.
"a carry git refuses fails naming the files, and the next move asks a whole check": func(t *testing.T) {
var kinds []string
var from []map[string]string
j := &judgeFunc{f: func(_ context.Context, c *Case, call int) (Outcome, error) {
kinds, from = append(kinds, c.Kind), append(from, c.CarriedFrom)
if call == 2 {
return Outcome{Verdict: delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "ace",
Words: repoA + "#7 " + ConflictWords + "go.mod"}}}, nil
}
return Outcome{Verdict: passing()}, nil
}}
r := newRig(t, j)
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS})
r.ch.TrunkMoved(ctx)
r.ch.Wait()
r.ch.Reconsider(ctx) // anything else that looks again asks nothing of a conflict
r.ch.Wait()
if v := r.verdict(t, keyOf(t, "feat/x", cand)); j.count() != 2 || !IsConflictFailure(v) || !strings.Contains(v.Failed.Words, "go.mod") {
t.Fatalf("judged %d times, verdict %+v; want the carry failed naming go.mod and nothing asked after it", j.count(), v.Failed)
}
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS})
r.ch.TrunkMoved(ctx)
r.ch.Wait()
if j.count() != 3 || kinds[1] != KindTrunk || from[1] == nil || kinds[2] != KindNew || from[2] != nil {
t.Errorf("checks %v carried from %v; want a whole check, a carry, then a whole check after the conflict", kinds, from)
}
},
// §8, §6: the line of a carried verdict names the head on the new trunk commit first, as the gate and the merge tool
// read a line, then where it was carried from and what the carry ran and kept; while it is carried it is pending and says so.
"the line of a carry says where it was carried from": func(t *testing.T) {
cs, p := lineCase(t, pullOf(repoA, 7, head1))
v := passVerdict()
v.Trunk = map[string]string{repoA: trunkA}
v.Beside = map[string]string{catalogueRepository: trunkS2}
v.CarriedFrom = map[string]string{repoA: trunkA2, catalogueRepository: trunkS}
line := Line(v, cs, p, nil, nil, nil)
if !strings.HasPrefix(line, "pass 111111111111 on aaaaaaaaaaaa | carried from abababababab, mesh-catalog 555555555555 | unit 12 run") {
t.Errorf("the carried line = %q", line)
}
if head, trunk := checkWordsOf(line); head != head1[:12] || trunk != trunkA[:12] {
t.Errorf("the line names %s on %s first, want the head on the new trunk", head, trunk)
}
cs.CarriedFrom = map[string]string{repoA: trunkA2}
pending := Line(delivery.Verdict{State: delivery.VerdictRunning}, cs, p, nil, nil, nil)
if !strings.HasPrefix(pending, "being carried to 111111111111 on aaaaaaaaaaaa | from abababababab") {
t.Errorf("the pending line of a carry = %q", pending)
}
},
// §3 (review round 1): two approved merges in one turn whose trunk moved each get their carry; neither holds the other's.
"two merges waiting in one turn each get their carry": func(t *testing.T) {
var r *rig
j := &judgeFunc{f: func(context.Context, *Case, int) (Outcome, error) { return Outcome{Verdict: passing()}, nil }}
r = newRig(t, j)
candA, candC := candidateOf(t, "feat/a", pullOf(repoA, 7, head1)), candidateOf(t, "feat/c", pullOf(repoA, 8, head2))
r.ch.Candidate(ctx, "feat%2Fa", candA)
r.ch.Candidate(ctx, "feat%2Fc", candC)
r.ch.Wait()
a, _ := delivery.DecodeCandidate(candA)
c, _ := delivery.DecodeCandidate(candC)
at := time.Date(2026, 10, 26, 9, 0, 0, 0, time.UTC)
turn, err := delivery.EncodeTurn(delivery.Turn{Repository: repoA, Waiting: []delivery.TurnWait{
{Group: "feat/a", Heads: GateHeads(a), Approved: at, Since: at}, {Group: "feat/c", Heads: GateHeads(c), Approved: at.Add(time.Minute), Since: at}}})
if err != nil {
t.Fatal(err)
}
turns := &Turns{}
turns.Put(turnKey(t, repoA), turn, false)
r.ch.Turns = turns
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) // a third merge moves the trunk under both
r.ch.TrunkMoved(ctx)
r.ch.Wait()
for _, cand := range [][]byte{candA, candC} {
group := map[bool]string{true: "feat/a", false: "feat/c"}[string(cand) == string(candA)]
if v := r.verdict(t, keyOf(t, group, cand)); v.Trunk[repoA] != trunkA2 || v.CarriedFrom[repoA] != trunkA {
t.Errorf("%s: verdict on %v carried from %v; want its carry claimed and done", group, v.Trunk, v.CarriedFrom)
}
}
},
// §6 (review round 1): the node-engine read at the commit the mesh runs is named, and a new running commit is carried.
"a new running node-engine is carried": func(t *testing.T) {
runs := runsAt{nodeEngineModule: trunkS}
j := &judgeFunc{f: func(_ context.Context, c *Case, _ int) (Outcome, error) {
return Outcome{Verdict: passing(), Beside: map[string]string{nodeEngineRepo: runs[nodeEngineModule]}}, nil
}}
r := newRig(t, j)
r.ch.Runs = runs
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
r.ch.TrunkMoved(ctx) // nothing moved: nothing is carried
r.ch.Wait()
runs[nodeEngineModule] = trunkS2
r.ch.TrunkMoved(ctx) // what the indexer's watch does when mesh-host's running commit changes
r.ch.Wait()
v := r.verdict(t, keyOf(t, "feat/x", cand))
if j.count() != 2 || v.CarriedFrom[nodeEngineRepo] != trunkS || v.Beside[nodeEngineRepo] != trunkS2 {
t.Errorf("judged %d times, verdict beside %v carried from %v; want one carry over the node-engine's new running commit", j.count(), v.Beside, v.CarriedFrom)
}
},
// §5: a carry that fails names the move it failed on.
"a carry that fails names the move": func(t *testing.T) {
j := &judgeFunc{f: func(_ context.Context, c *Case, call int) (Outcome, error) {
if call == 1 {
return Outcome{Verdict: passing()}, nil
}
return Outcome{Verdict: delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Test: "unit: m/a.TestX", Words: "want 4"}}}, nil
}}
r := newRig(t, j)
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS})
r.ch.TrunkMoved(ctx)
r.ch.Wait()
v := r.verdict(t, keyOf(t, "feat/x", cand))
if v.State != delivery.VerdictFail || v.CarriedFrom != nil || !strings.HasPrefix(v.Failed.Words, "want 4") ||
!strings.Contains(v.Failed.Words, repoA+" "+trunkA[:12]+" to "+trunkA2[:12]) {
t.Errorf("the failed carry is %s carried from %v with words %q; want the test's words, then the move", v.State, v.CarriedFrom, v.Failed.Words)
}
},
} {
t.Run(name, run)
}
}
// checkWordsOf is the head and trunk a line names first, as the gate reads it (mesh-delivery-gate onestatus.go checkLine).
func checkWordsOf(line string) (string, string) {
for _, part := range strings.Split(line, " | ") {
f := strings.Fields(part)
for i := 0; i+2 < len(f); i++ {
if f[i+1] == "on" && len(f[i]) == 12 && len(f[i+2]) == 12 {
return f[i], f[i+2]
}
}
}
return "", ""
}
@@ -4,7 +4,6 @@ import (
"context"
"errors"
"fmt"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -97,19 +96,9 @@ func notJudged(v delivery.Verdict) bool {
return v.Composition != nil && v.Composition.State == delivery.CompositionNotRun && v.Composition.Words == delivery.AssignerNotRunning
}
// unanswered says whether a verdict failed only on its composition, and that only because nodes the change places did
// not answer its placement in time (delivery.CompositionUnanswered, ADR 0559 §5): its tests passed (a composition is read
// only into a passing verdict), so a later composition of the same heads is taken in on its event, with no new check
// (ADR 0548 §1: a part of the mesh not heard is judged again when it is heard).
func unanswered(v delivery.Verdict) bool {
return v.State == delivery.VerdictFail && v.Composition != nil && v.Composition.State == delivery.CompositionFail &&
strings.Contains(v.Composition.Words, delivery.CompositionUnanswered) && v.Failed.Words == v.Composition.Words
}
// Composed takes in a composition the assigner wrote after the check's verdict said it was not judged because the
// assigner was not running (the assigner started since), or after it failed only because placed nodes had not answered
// (unanswered): the verdict and its status are written again with it. A composition of heads no candidate holds now, or
// of a verdict that judged its composition otherwise, is left.
// assigner was not running (the assigner started since): the verdict and its status are written again with it. A
// composition of heads no candidate holds now, or of a verdict that judged its composition, is left.
func (ch *Checker) Composed(ctx context.Context, comp delivery.CompositionVerdict) {
ch.init()
if comp.State == delivery.CompositionWaits {
@@ -134,21 +123,10 @@ func (ch *Checker) Composed(ctx context.Context, comp delivery.CompositionVerdic
return
}
v, err := delivery.DecodeVerdict(cur)
if err != nil {
return
}
switch {
case v.State == delivery.VerdictPass && notJudged(v):
case unanswered(v) && comp.Words != v.Composition.Words:
// The nodes answered since (or another failure stands now): the verdict is its tests' pass with this composition.
v.State, v.Failed = delivery.VerdictPass, delivery.FailedFirst{}
default:
if err != nil || v.State != delivery.VerdictPass || !notJudged(v) {
return
}
withComposition(&v, comp)
if v.Composition != nil && v.Composition.State != delivery.CompositionNotRun {
v.Composition.FPSet = cs.FPSet
}
data, err := delivery.EncodeVerdict(v)
if err != nil {
ch.logf("%s: the verdict with its composition cannot be written: %v", cs.Key, err)
@@ -113,27 +113,6 @@ func TestTheAssignerStartingBringsTheComposition(t *testing.T) {
t.Errorf("the status's line says %q", o.Text)
}
}
// ADR 0559 §5: a verdict that failed only because a placed node had not answered takes the same heads' later
// composition in on its event, a pass making it pass with no new check; a composition failing the same way is left.
ch, cs := checkerWithNotJudged(t)
silent := delivery.CompositionVerdict{Group: cs.Group, Heads: cs.HeadsHash, State: delivery.CompositionFail, SDK: "x", Machine: "g14",
Words: "g14 (offline) " + delivery.CompositionUnanswered}
ch.Composed(context.Background(), silent)
for _, row := range []struct {
name string
comp delivery.CompositionVerdict
state string
}{
{"the same failure again", silent, delivery.VerdictFail},
{"the node's answer", delivery.CompositionVerdict{Group: cs.Group, Heads: cs.HeadsHash, State: delivery.CompositionPass, SDK: "x"}, delivery.VerdictPass},
} {
ch.Composed(context.Background(), row.comp)
raw, _, _, _ := ch.Verdicts.Get(context.Background(), cs.Key)
if v, err := delivery.DecodeVerdict(raw); err != nil || v.State != row.state || v.Composition == nil || v.Composition.State != row.comp.State {
t.Errorf("%s: the verdict is %+v, %v", row.name, v, err)
}
}
}
// checkerWithNotJudged is a checker holding a candidate whose passing verdict said its composition was not judged.
@@ -12,18 +12,8 @@ type BesideRef struct {
Repository string `json:"repository"`
Commit string `json:"commit"`
Ref string `json:"ref,omitempty"`
// atTrunk says the sibling was read at its trunk's head, so a move of that trunk puts the verdict out of date and is
// carried (novox/hq ADR 0548 §6). Never sent: the node-engine at the commit the mesh runs, mesh-sdk at a pin and a
// group's member at its head do not move with a trunk.
atTrunk bool
// runs says the sibling was read at the commit the mesh runs (the node-engine, the indexer's record of mesh-host): a new
// running commit puts the verdict out of date and is carried (ADR 0548 §6). Never sent.
runs bool
}
// NodeEngineModule is the module whose running commit the node-engine sibling is read at.
const NodeEngineModule = nodeEngineModule
const (
catalogueRepository = "novox/mesh-catalog"
nodeEngineRepo = "novox/mesh-host"
@@ -61,21 +51,18 @@ func (j *Examiner) besideFor(ctx context.Context, c *Case, p Pull) ([]BesideRef,
switch {
case repo == catalogueRepository:
if head := heads[repo]; head != "" {
out = append(out, BesideRef{Repository: repo, Commit: head, atTrunk: c.member(repo) == nil})
out = append(out, BesideRef{Repository: repo, Commit: head})
}
case repo == nodeEngineRepo:
commit := ""
if j.Runs != nil {
commit = j.Runs.CommitOf(nodeEngineModule)
}
atTrunk := false
if commit == "" {
commit, atTrunk = heads[repo], true
commit = heads[repo]
}
if commit != "" {
// Named in the verdict either way (ADR 0548 §6, issue 568's own example): at the commit the mesh runs, a new
// running commit puts the verdict out of date; at the trunk, a trunk move does (Checker.besideNow).
out = append(out, BesideRef{Repository: repo, Commit: commit, atTrunk: atTrunk, runs: !atTrunk})
out = append(out, BesideRef{Repository: repo, Commit: commit})
}
case repo == SDKRepository && p.Repository == controllerRepo && j.Forge != nil:
text, err := j.Forge.File(ctx, p.Repository, "go.mod", p.Head)
@@ -95,7 +82,7 @@ func (j *Examiner) besideFor(ctx context.Context, c *Case, p Pull) ([]BesideRef,
if member := c.member(repo); member != nil {
out = append(out, BesideRef{Repository: repo, Commit: member.Head, Ref: refOf(member.Number)})
} else if head := heads[repo]; head != "" {
out = append(out, BesideRef{Repository: repo, Commit: head, atTrunk: true})
out = append(out, BesideRef{Repository: repo, Commit: head})
} else if repo == controllerRepo && p.Repository == nodeEngineRepo {
// The node-engine's step-nine test judges the controller's own manifest and fails without it (issue 1295): a
// checker that cannot place it says so, where placing none left that test to skip.
@@ -121,13 +108,3 @@ func refOf(number int) string {
}
return "refs/pull/" + strconv.Itoa(number) + "/head"
}
// besideAtTrunk are the siblings of refs read at their trunk's head that the case does not merge, by repository: what a
// verdict names as Beside (ADR 0548 §6).
func besideAtTrunk(c *Case, refs []BesideRef, into map[string]string) {
for _, r := range refs {
if _, merged := c.Trunk[r.Repository]; (r.atTrunk || r.runs) && !merged {
into[r.Repository] = r.Commit
}
}
}
@@ -1,42 +0,0 @@
package check
import (
"fmt"
"strings"
)
// A change that adds or modifies a compiled program fails (issue 1577): it is build output, `go build` in a module's folder
// writes one beside go.mod, and `git add -A` takes it. The build module's fetch step, which holds the merged clone, reads the
// first bytes of each changed file and reports the programs (mesh-sdk buildstep.MergeInto); the checker judges the answer.
// NoProgramsReported is the warning of a build module whose fetch answered no programs field: an older module, which cannot
// tell, so the check does not refuse a program it did not look for.
const NoProgramsReported = "the build module did not report programs: a compiled program in the change is not refused"
// checkBinariesOf fails a pull request whose change adds or modifies a compiled program, naming each file. It judges the
// first fetch answer of the pull request, whichever build module gave it; a build module that reports no programs leaves a
// warning on the check and passes it.
func (st *state) checkBinariesOf(t *target) verdictPtr {
key := fmt.Sprintf("%s#%d", t.pull.Repository, t.pull.Number)
if t.fetched.Merge == "" {
return nil // no merge was made: nothing was changed to judge
}
if t.fetched.Programs == nil {
st.c.note(fmt.Sprintf("%s: %s (%s)", key, NoProgramsReported, t.module))
st.c.warn(key, fmt.Sprintf("a compiled program is not refused: %s reported no programs", t.module))
st.j.logf("%s: %s (%s)", key, NoProgramsReported, t.module)
return nil
}
var found []string
for _, p := range *t.fetched.Programs {
if p.Commit != "" {
found = append(found, fmt.Sprintf("%s: a compiled program added in %s and removed later: it stays in history; squash the branch (issue 1577)", p.Path, short12(p.Commit)))
continue
}
found = append(found, p.Path+": a compiled program is build output: remove it and add it to .gitignore (issue 1577)")
}
if len(found) > 0 {
return failed(t.node, fmt.Sprintf("%s: %s", key, strings.Join(found, "; ")))
}
return nil
}
@@ -1,63 +0,0 @@
package check
import (
"slices"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
func TestAChangeWhoseFetchReportsProgramsFailsNamingEachFile(t *testing.T) { // 1577
prog := func(p ...string) *[]ProgramFile {
out := []ProgramFile{}
for _, f := range p {
path, commit, _ := strings.Cut(f, "@") // `path@commit`: added by a commit of the branch and removed later
out = append(out, ProgramFile{Path: path, Kind: "ELF", Commit: commit})
}
return &out
}
for _, c := range []struct {
name string
programs *[]ProgramFile
fail []string // paths named; none passes
warned bool // the build module reported no programs field
}{
{"none found", prog(), nil, false},
{"one program", prog("modules/m/m"), []string{"modules/m/m"}, false},
{"two programs", prog("a/x", "b/y"), []string{"a/x", "b/y"}, false},
{"added and removed in the branch", prog("gone/prog@0123456789abcdef0123"), []string{"gone/prog: a compiled program added in 0123456789ab and removed later: it stays in history; squash the branch (issue 1577)"}, false},
{"an older build module reports nothing", nil, nil, true},
} {
b := newBench(t)
b.fetch.Merge, b.fetch.Programs = "mmmm", c.programs
cs := caseOf(t, pullOf(repoA, 7, head1))
v := b.judge(t, cs).Verdict
if got := slices.ContainsFunc(cs.Notes(), func(n string) bool { return strings.Contains(n, NoProgramsReported) }); got != c.warned {
t.Errorf("%s: warned = %v, want %v (%v)", c.name, got, c.warned, cs.Notes())
}
line := Line(v, cs, cs.Pulls[0], nil, nil, nil)
if got := strings.Contains(line, "a compiled program is not refused: mesh-build-go reported no programs"); got != c.warned {
t.Errorf("%s: the status line holds the warning = %v, want %v: %s", c.name, got, c.warned, line)
}
if len(c.fail) == 0 {
if v.State != delivery.VerdictPass {
t.Errorf("%s: verdict = %+v", c.name, v)
}
continue
}
if v.State != delivery.VerdictFail {
t.Errorf("%s: verdict = %+v, want a failure", c.name, v)
continue
}
for _, p := range c.fail {
want := p
if !strings.Contains(p, " added in ") {
want = p + ": a compiled program is build output: remove it and add it to .gitignore (issue 1577)"
}
if !strings.Contains(v.Failed.Words, want) {
t.Errorf("%s: words %q do not name %s", c.name, v.Failed.Words, p)
}
}
}
}
@@ -43,43 +43,12 @@ type Case struct {
Try *TryOptions
notesMu sync.Mutex
// warnings are, by pull request, what part of its check did not run (Case.warn).
warnings map[string]string
notes []string
// asks are the ids of the asks the check put on the build seats' streams, in order: a try names them (Try.Asks).
asks []string
// Watch, when set, hears each ask the check puts, as it is put (step nil), and each value its step key takes while
// the check waits for it: the node that took it, the phase it says, its end. A try writes them into its record, so a
// running try names its asks and where each stands (issue 1890). Called from the check's goroutines; it must not block
// for long.
Watch func(askID string, step *delivery.Step)
notes []string
// Progress says what the check is doing, so a check that ran past its bound names the step it was stuck at.
Progress *Progress
// Kind is what asked the check (KindNew, KindNodes, KindTrunk): a carry after a trunk or sibling move is asked on the
// build seats' recheck stream, every other check on check (ADR 0536 §4, ADR 0548 §5).
// Kind is what asked the check (KindNew, KindNodes, KindTrunk): a re-judge after a trunk move is asked on the build
// seats' recheck stream, every other check on check (ADR 0536 §4).
Kind string
// CarriedFrom are, for the carry of a verdict that passed, the trunk and sibling commits it was judged on (ADR 0548 §5);
// nil for a whole check.
CarriedFrom map[string]string
// FPSet is the candidate's fingerprint set (Candidate.FPSet): a composition names the set it judged.
FPSet string
// Composition is, for a carry, the composition that passed on the same fingerprint set: it is kept and not asked again,
// since no fingerprint of a module it composed changed (ADR 0548 §5). Nil composes.
Composition *delivery.Composition
}
// Asked records the id of an ask the check put.
func (c *Case) Asked(id string) {
c.notesMu.Lock()
c.asks = append(c.asks, id)
c.notesMu.Unlock()
}
// Asks are the ids of the asks the check put, in order.
func (c *Case) Asks() []string {
c.notesMu.Lock()
defer c.notesMu.Unlock()
return append([]string(nil), c.asks...)
}
// Progress is the step a check is at, written by the check and read when its bound passes.
@@ -111,7 +80,7 @@ func CaseOf(c delivery.Candidate, trunk map[string]string) *Case {
if err != nil {
return nil
}
cs := &Case{Key: key, Group: c.Group, HeadsHash: c.HeadsHash, Heads: GateHeads(c), Trunk: map[string]string{}, Progress: &Progress{}, FPSet: c.FPSet}
cs := &Case{Key: key, Group: c.Group, HeadsHash: c.HeadsHash, Heads: GateHeads(c), Trunk: map[string]string{}, Progress: &Progress{}}
for _, p := range c.Pulls {
switch head, known := trunk[p.Repository]; {
case !known || head == "":
@@ -193,23 +162,6 @@ func (c *Case) note(s string) {
}
}
// warn says of a pull request (pullKey) that a part of its check did not run, in words for its status line (issue 1577).
func (c *Case) warn(key, words string) {
c.notesMu.Lock()
defer c.notesMu.Unlock()
if c.warnings == nil {
c.warnings = map[string]string{}
}
c.warnings[key] = words
}
// Warning is what warn said of a pull request, or "".
func (c *Case) Warning(key string) string {
c.notesMu.Lock()
defer c.notesMu.Unlock()
return c.warnings[key]
}
// Notes are the words noted on the case, in the order they were noted.
func (c *Case) Notes() []string {
c.notesMu.Lock()
@@ -4,7 +4,6 @@ import (
"context"
"errors"
"fmt"
"sort"
"strings"
"sync"
"time"
@@ -43,9 +42,6 @@ type Outcome struct {
// Judged says, by pull request (pullKey), which program judged a change to a build module (ADR 0535 §5); a pull request
// that changes none is not named.
Judged map[string]string
// Beside are the siblings the check read at their trunk's head, by repository (ADR 0548 §6): the verdict names them, and
// a move of one is carried.
Beside map[string]string
}
// BuildNote is the state of a candidate's module build, by module: built, failed or waiting (ADR 0484 clause 7).
@@ -69,14 +65,6 @@ type Checker struct {
Owed KV
Judge Judge
Trunks interface{ Heads() map[string]string }
// Runs says which commit the mesh runs of a module (the indexer's modules): the node-engine a verdict read beside at its
// running commit is out of date when that commit moves (ADR 0548 §6); nil reads every sibling at its trunk.
Runs interface{ CommitOf(module string) string }
// Turns are the gate's merge turns (ADR 0548 §3): no check is claimed of a repository a turn names, but the check of the
// waiting delivery's own heads; nil holds none.
Turns interface {
Holding(repositories []string, heads string) (string, delivery.TurnWait, bool)
}
// Bound is how long one check may run; none when 0 (DefaultBound).
Bound time.Duration
// Link is the base of the address a status points at; the verdict key follows.
@@ -85,10 +73,6 @@ type Checker struct {
Announce Announcer
Now func() time.Time
Logf func(format string, args ...any)
// Settle is how long a group's heads must stay the same after the candidate last changed them before its check starts
// (issue 1890): the members of a group are announced one after another, and a check of each partial set is thrown
// away at the next. None when 0; DefaultSettle in the module.
Settle time.Duration
once sync.Once
wg sync.WaitGroup
@@ -99,78 +83,15 @@ type Checker struct {
tried map[string]bool // by verdict key: looked at since this process started
epoch uint64 // changes of the nodes seen since this process started
nodesFail map[string]uint64 // by verdict key: the epoch a failure of the nodes was written at
held map[string]string // by verdict key: the turn that holds its check off, as last said
live map[string]liveCheck // by verdict key: the check running in this process, ended when its heads go
settling map[string]*window // by candidates key: the window its heads settle in
}
// liveCheck is a check running in this process: the candidates key of its group and the end of its run.
type liveCheck struct {
group string
end context.CancelCauseFunc
}
// window is a group's settle window: aborted with errUnsettled when its heads change again or the group goes, and with
// errSettled to end it at once.
type window struct{ abort context.CancelCauseFunc }
var (
errUnsettled = errors.New("the group's heads changed again")
errSettled = errors.New("the group's heads settled")
)
// settleNow ends the settle window of every group at once, as if Settle had passed.
func (ch *Checker) settleNow() {
ch.mu.Lock()
defer ch.mu.Unlock()
for _, w := range ch.settling {
w.abort(errSettled)
}
}
// DefaultSettle is the module's Settle (issue 1890). On 2026-10-26 the three members of a group were announced within 2.8
// seconds of each other (20:31:52.3, :53.7, :55.1 UTC), a session opening or pushing them one API call after another;
// fifteen seconds holds five times that, and costs a check of 10 to 18 minutes under 2.5 % of its time. A member that comes
// later still is no waste beyond its asks already taken: the check of the heads it replaced is ended at once.
const DefaultSettle = 15 * time.Second
// EndedWords open the words of a verdict the checker ended without judging it, its heads no candidate's any more (issue
// 1890): never a failure of the change, and judged whole if a candidate holds those heads again.
const EndedWords = "ended: "
// IsEnded says whether a verdict was ended without being judged (EndedWords).
func IsEnded(v delivery.Verdict) bool {
return v.State == delivery.VerdictFail && v.Failed.Machine == "checker" && strings.HasPrefix(v.Failed.Words, EndedWords)
}
// superseded is the cause a running check is ended with: the heads that replaced its own, or none when its group went.
type superseded struct{ by string }
func (s superseded) Error() string {
if s.by == "" {
return EndedWords + "no candidate holds these heads any more: the group was closed or merged"
}
return EndedWords + "superseded by " + s.by
}
func (ch *Checker) init() {
ch.once.Do(func() {
ch.known, ch.inflight, ch.again, ch.tried = map[string]delivery.Candidate{}, map[string]bool{}, map[string]bool{}, map[string]bool{}
ch.nodesFail, ch.held = map[string]uint64{}, map[string]string{}
ch.live, ch.settling = map[string]liveCheck{}, map[string]*window{}
ch.nodesFail = map[string]uint64{}
})
}
// endOthers ends every check of the group running in this process whose verdict key is not keep: no candidate holds its
// heads any more (issue 1890). Called with ch.mu held.
func (ch *Checker) endOthers(group, keep, by string) {
for k, l := range ch.live {
if l.group == group && k != keep {
l.end(superseded{by: by})
}
}
}
func (ch *Checker) logf(format string, args ...any) {
if ch.Logf != nil {
ch.Logf(format, args...)
@@ -203,62 +124,17 @@ func (ch *Checker) Candidate(ctx context.Context, key string, value []byte) {
ch.logf("candidates.%s holds what is not a candidate, left: %v", key, err)
return
}
vkey, _ := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
ch.mu.Lock()
prev, had := ch.known[key]
ch.known[key] = c
// A check of heads this candidate no longer holds is ended at once (issue 1890): a member's new head, a member joining
// or leaving.
ch.endOthers(key, vkey, GateHeads(c))
moved := !had || prev.HeadsHash != c.HeadsHash
if ch.Settle > 0 && (moved || ch.settling[key] != nil) {
// The group's heads changed: its check waits until they have been the same for Settle, so a group whose members
// are announced one after another is checked once, whole. A later change starts the window again. The wait is a
// context's bound, no timer (ADR 0437 §8).
if w := ch.settling[key]; w != nil {
w.abort(errUnsettled)
}
base, abort := context.WithCancelCause(ctx)
w := &window{abort: abort}
ch.settling[key] = w
ch.wg.Add(1)
go func() {
defer ch.wg.Done()
wait, done := context.WithTimeout(base, ch.Settle)
defer done()
<-wait.Done()
settled := errors.Is(wait.Err(), context.DeadlineExceeded) || errors.Is(context.Cause(wait), errSettled)
ch.mu.Lock()
if !settled || ch.settling[key] != w {
ch.mu.Unlock()
return // changed again, forgotten, or the checker stops
}
delete(ch.settling, key)
now, ok := ch.known[key]
ch.mu.Unlock()
abort(nil)
if ok {
ch.consider(ctx, key, now)
}
}()
ch.mu.Unlock()
return
}
ch.mu.Unlock()
ch.consider(ctx, key, c)
}
// Forget takes a candidate out (its key was deleted): a check of its heads still running is ended, as no candidate holds
// them any more.
// Forget takes a candidate out (its key was deleted).
func (ch *Checker) Forget(key string) {
ch.init()
ch.mu.Lock()
delete(ch.known, key)
if w := ch.settling[key]; w != nil {
w.abort(errUnsettled)
delete(ch.settling, key)
}
ch.endOthers(key, "", "")
ch.mu.Unlock()
}
@@ -301,49 +177,32 @@ func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidat
}
cs.Queued = ch.now()
ch.mu.Lock()
if ch.settling[key] != nil {
ch.mu.Unlock()
return // its heads are settling: the window's end considers it (Settle)
}
if ch.inflight[cs.Key] {
ch.again[cs.Key] = true // looked at once more when the check ends, so a change of the trunk is never lost
ch.mu.Unlock()
return
}
ch.inflight[cs.Key] = true
run, end := context.WithCancelCause(ctx)
ch.live[cs.Key] = liveCheck{group: key, end: end}
ch.mu.Unlock()
ch.wg.Add(1)
go func() {
defer ch.wg.Done()
vkey := cs.Key
defer func() {
ch.mu.Lock()
delete(ch.inflight, vkey)
delete(ch.again, vkey)
delete(ch.live, vkey)
ch.mu.Unlock()
end(nil)
}()
for {
// Every check is asked at once: the build seats' streams hold the backlog, a new head's check and a check after a
// failure of the nodes on check, a re-judge after a trunk move on recheck, pulled after them (ADR 0525 §4, §9;
// ADR 0536 §4). The checker holds no place and no capacity of its own.
ch.prepare(ctx, cs)
ch.try(ctx, run, key, cs)
cs.Kind = ch.kindOf(ctx, cs)
ch.try(ctx, key, cs)
ch.mu.Lock()
more := ch.again[vkey]
delete(ch.again, vkey)
ch.mu.Unlock()
if !more || run.Err() != nil {
more := ch.again[cs.Key]
delete(ch.again, cs.Key)
if !more || ctx.Err() != nil {
delete(ch.inflight, cs.Key)
ch.mu.Unlock()
return
}
// Looked at again for the same heads only: a candidate that moved to other heads has its own check (issue 1890).
ch.mu.Unlock()
if next := CaseOf(ch.candidate(key, c), ch.heads()); next != nil {
if next.Key != vkey {
return
}
cs = next
}
cs.Queued = ch.now()
@@ -352,132 +211,29 @@ func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidat
}
// The kinds of a check (ADR 0536 §4): a head never judged and a head asked again after a failure of the nodes are asked on
// the build seats' check stream; a carry, a verdict moved to a newer trunk or sibling commit by its package digests (ADR 0548
// §5), on the recheck stream, pulled after.
// the build seats' check stream, a head asked again only because its trunk moved on the recheck stream, pulled after.
const (
KindNew = "new"
KindNodes = "nodes"
KindTrunk = "trunk"
)
// prepare reads the case's verdict key and says what its check is (ADR 0548 §1, §5, §7): none (or one left running) is a
// head never judged, a whole check; a failure of the nodes is asked again after the nodes changed; a verdict judged on trunk
// or sibling commits that moved since is carried, its packages run only where the move changed their digest, and a pass's
// commits are named as the carry's CarriedFrom. A verdict that stands is ranked with the carries: its claim finds nothing
// to do.
func (ch *Checker) prepare(ctx context.Context, cs *Case) {
cs.Kind, cs.CarriedFrom, cs.Composition = KindNew, nil, nil
// kindOf is the kind of a case's check, read from its verdict key: none (or one left running) is a head never judged; a
// failure of the nodes is asked again after the nodes changed; a verdict on other trunk heads is a re-judge after a trunk
// move. A verdict that stands is ranked with the re-judges: its claim finds nothing to do.
func (ch *Checker) kindOf(ctx context.Context, cs *Case) string {
value, _, found, err := ch.Verdicts.Get(ctx, cs.Key)
if err != nil || !found {
return
return KindNew
}
v, err := delivery.DecodeVerdict(value)
switch {
case err != nil || v.State == delivery.VerdictRunning:
return
return KindNew
case IsNodesFailure(v):
cs.Kind = KindNodes
return
case IsConflictFailure(v):
return // a head git refused: its next check is whole (ADR 0548 §7)
case IsEnded(v):
return // ended unjudged when its heads went (issue 1890): held again, they are checked whole
return KindNodes
}
cs.Kind = KindTrunk
if v.State == delivery.VerdictPass && ch.stale(v, cs) {
cs.CarriedFrom = judgedOn(v)
if c := v.Composition; c != nil && c.State == delivery.CompositionPass && c.FPSet != "" && c.FPSet == cs.FPSet {
kept := *c
cs.Composition = &kept
}
}
}
// stale says whether a verdict was judged on commits that are no longer the case's: a trunk it merged moved, or a sibling it
// read at its trunk's head (ADR 0548 §6).
func (ch *Checker) stale(v delivery.Verdict, cs *Case) bool {
return !sameTrunk(v.Trunk, cs.Trunk) || besideMoved(v.Beside, ch.besideNow())
}
// besideNow are the commits a sibling is read at now, by repository: its trunk head, and for the node-engine the commit the
// mesh runs when the indexer names one, as besideFor places them.
func (ch *Checker) besideNow() map[string]string {
return besideAt(ch.heads(), ch.Runs)
}
// besideAt places the commits a sibling is read at: the trunk heads, with the node-engine at the commit the mesh runs when
// runs names one (ADR 0548 §6). The checker's decision and the status read the same commits through it.
func besideAt(now map[string]string, runs interface{ CommitOf(module string) string }) map[string]string {
if runs != nil {
if c := runs.CommitOf(nodeEngineModule); c != "" {
out := make(map[string]string, len(now)+1)
for r, h := range now {
out[r] = h
}
out[nodeEngineRepo] = c
return out
}
}
return now
}
// besideMoved says whether a sibling read at these commits moved: the commit it is read at now is known and another.
func besideMoved(beside, now map[string]string) bool {
for r, c := range beside {
if h := now[r]; h != "" && h != c {
return true
}
}
return false
}
// judgedOn are the trunk and sibling commits a verdict was judged on, by repository.
func judgedOn(v delivery.Verdict) map[string]string {
out := make(map[string]string, len(v.Trunk)+len(v.Beside))
for r, c := range v.Beside {
out[r] = c
}
for r, c := range v.Trunk {
out[r] = c
}
return out
}
// carried is what a carry's verdict names as carried from (ADR 0548 §5): the commits it was judged on before, of the
// repositories the new verdict still judges or reads; nil when none of them moved.
func carried(from map[string]string, v delivery.Verdict) map[string]string {
out, moved := map[string]string{}, false
for r, c := range from {
now, ok := v.Trunk[r]
if !ok {
now, ok = v.Beside[r]
}
if ok && now != "" && c != "" {
out[r] = c
moved = moved || now != c
}
}
if !moved {
return nil
}
return out
}
// moveWords say a carry's moves, for the words of a failure: "carried from <old12> to <new12> of <repository>", each moved
// repository, sorted.
func moveWords(from map[string]string, v delivery.Verdict) string {
var moves []string
for r, c := range from {
now, ok := v.Trunk[r]
if !ok {
now = v.Beside[r]
}
if now != "" && now != c {
moves = append(moves, fmt.Sprintf("%s %s to %s", r, short12(c), short12(now)))
}
}
sort.Strings(moves)
return strings.Join(moves, ", ")
return KindTrunk
}
func (ch *Checker) candidate(key string, fallback delivery.Candidate) delivery.Candidate {
@@ -509,9 +265,6 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64
return nil, fmt.Errorf("%s: %w", cs.Key, err)
}
if !found {
if ch.heldByTurn(cs) {
return nil, nil
}
rev, err := ch.Verdicts.Create(ctx, cs.Key, data)
if errors.Is(err, ErrConflict) {
return nil, nil // another checker made it first
@@ -532,15 +285,15 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64
return &claimed{rev}, nil
}
return nil, nil
case IsEnded(cur), ch.stale(cur, cs):
retake = true // heads held again after their check was ended unjudged (issue 1890), or a trunk that moved
case !sameTrunk(cur.Trunk, cs.Trunk):
retake = true
case cur.State == delivery.VerdictFail && IsNodesFailure(cur):
ch.mu.Lock()
at, ok := ch.nodesFail[cs.Key]
ch.mu.Unlock()
retake = !ok || epoch > at
}
if !retake || ch.heldByTurn(cs) {
if !retake {
return nil, nil
}
rev, err = ch.Verdicts.Update(ctx, cs.Key, data, rev)
@@ -553,59 +306,6 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64
return &claimed{rev}, nil
}
// heldByTurn says whether a merge turn holds the case's check off (ADR 0548 §3): an approved merge waits on one of the
// repositories it merges heads into, and it is not the merge of the case's own heads. The check is claimed when the turn's
// key goes (Reconsider on the turns watch). Said once per turn in the log. A check already claimed is never held: one
// claimed in the moment before the gate wrote its turn is judged on the commit it read and carried once after.
func (ch *Checker) heldByTurn(cs *Case) bool {
if ch.Turns == nil {
return false
}
repos := make([]string, 0, len(cs.Trunk))
for r := range cs.Trunk {
repos = append(repos, r)
}
repo, w, held := ch.Turns.Holding(repos, cs.Heads)
why := ""
if held {
why = repo + " " + w.Group + " " + w.Heads
}
ch.mu.Lock()
said := ch.held[cs.Key]
if why == "" {
delete(ch.held, cs.Key)
} else {
ch.held[cs.Key] = why
}
ch.mu.Unlock()
if held && said != why {
ch.logf("%s: not claimed: the approved merge of %s (heads %s) holds a turn on %s, so its check waits for that merge (ADR 0548 §3)",
cs.Key, w.Group, w.Heads, repo)
}
return held
}
// Held is every check a merge turn holds off now, by verdict key, with the turn's repository, group and heads.
func (ch *Checker) Held() map[string]string {
ch.init()
ch.mu.Lock()
defer ch.mu.Unlock()
out := make(map[string]string, len(ch.held))
for k, v := range ch.held {
out[k] = v
}
return out
}
// ConflictWords are what the words of a head that git would not merge onto its trunk say before the files in conflict.
const ConflictWords = "does not merge into the trunk: files in conflict: "
// IsConflictFailure says whether a verdict failed because its head did not merge onto its trunk (ADR 0548 §7): it has no
// pass to carry, so its next check, at a push or at a later trunk move, is a whole check of its head.
func IsConflictFailure(v delivery.Verdict) bool {
return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, ConflictWords)
}
// IsNodesFailure says whether a verdict failed because a part of the mesh the check reads was not heard (UnheardWords):
// it is judged again when the nodes change. No check fails for want of a build node any more: its asks wait in the build
// seats' streams (ADR 0525 §6, issue 1912).
@@ -613,13 +313,9 @@ func IsNodesFailure(v delivery.Verdict) bool {
return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, UnheardWords)
}
// try claims the check, judges it within its bound under run and writes the verdict and the owed statuses. A check that ends
// because ctx did is left running: the checker takes it again when it starts. A check whose run was ended because no
// candidate holds its heads any more (issue 1890) is ended: its verdict says so, and no status is written.
func (ch *Checker) try(ctx, run context.Context, key string, cs *Case) {
if run.Err() != nil {
return // ended before it was claimed: nothing to write
}
// try claims the check, judges it within its bound and writes the verdict and the owed statuses. A check that ends
// because ctx did is left running: the checker takes it again when it starts.
func (ch *Checker) try(ctx context.Context, key string, cs *Case) {
ch.mu.Lock()
first := !ch.tried[cs.Key]
ch.tried[cs.Key] = true
@@ -635,47 +331,12 @@ func (ch *Checker) try(ctx, run context.Context, key string, cs *Case) {
}
began := ch.now()
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
out := ch.judge(run, cs)
// An ask the build seat gave up on judged nothing (issue 734): the check is asked again under the same claim, its status
// pending, and after LostAskTries it fails as not heard, judged again when the nodes change.
for try := 1; IsLostAsk(out.Verdict) && run.Err() == nil; try++ {
if try >= LostAskTries {
out.Verdict.Failed.Words = strings.TrimSpace(out.Verdict.Failed.Words + "\nasked " + fmt.Sprint(LostAskTries) +
" times and given up each time: " + UnheardWords)
break
}
ch.logf("%s: %s; asked again (%d of %d)", cs.Key, out.Verdict.Failed.Words, try+1, LostAskTries)
out = ch.judge(run, cs)
}
out := ch.judge(ctx, cs)
if ctx.Err() != nil {
return
}
var why superseded
if errors.As(context.Cause(run), &why) {
ch.end(ctx, cs, cl, why)
return
}
v := out.Verdict
v.Group, v.Heads, v.Trunk, v.Beside = cs.Group, cs.Heads, cs.Trunk, nil
for r, c := range out.Beside {
if _, merged := cs.Trunk[r]; !merged && c != "" {
if v.Beside == nil {
v.Beside = map[string]string{}
}
v.Beside[r] = c
}
}
if cs.CarriedFrom != nil {
// A carry (ADR 0548 §5): a pass names the commits it was carried from; a failure names the move it failed on.
switch v.State {
case delivery.VerdictPass:
v.CarriedFrom = carried(cs.CarriedFrom, v)
case delivery.VerdictFail:
if moves := moveWords(cs.CarriedFrom, v); moves != "" {
v.Failed.Words = strings.TrimSpace(v.Failed.Words + "\nfailed on the carry over the move of " + moves + ": the move changed what these tests read")
}
}
}
v.Group, v.Heads, v.Trunk = cs.Group, cs.Heads, cs.Trunk
if v.Took.Queue == 0 && !cs.Queued.IsZero() && began.After(cs.Queued) {
v.Took.Queue = began.Sub(cs.Queued).Milliseconds()
}
@@ -697,37 +358,17 @@ func (ch *Checker) try(ctx, run context.Context, key string, cs *Case) {
ch.nodesFail[cs.Key] = epoch
ch.mu.Unlock()
}
if ch.movedSince(cs) || besideMoved(v.Beside, ch.besideNow()) {
if ch.movedSince(cs) {
// Judged on a trunk that moved while it ran: the verdict is kept as judged, and the status stays pending until the
// check of the new trunk, which Reconsider starts, ends.
ch.owePending(ctx, cs, v)
ch.logf("%s: %s on a trunk that moved while it ran; the status stays pending until it is carried", cs.Key, v.State)
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
ch.logf("%s: %s on a trunk that moved while it ran; the status stays pending", cs.Key, v.State)
return
}
ch.writeOwed(ctx, cs, v, out)
ch.logf("%s: %s", cs.Key, v.State)
}
// end writes the verdict of a check whose heads no candidate holds any more (issue 1890), by compare-and-set on its claim,
// and says it once in the log with the asks it had put: they stay in the build seats' streams, since an ask once put cannot
// be taken back (buildrun.Asker has no withdraw), and are no longer waited for. No status is written: the heads that
// replaced these have their own check, whose pending status stands.
func (ch *Checker) end(ctx context.Context, cs *Case, cl *claimed, why superseded) {
v := delivery.Verdict{Group: cs.Group, Heads: cs.Heads, Trunk: cs.Trunk, State: delivery.VerdictFail,
Failed: delivery.FailedFirst{Machine: "checker", Words: why.Error()}}
data, err := delivery.EncodeVerdict(v)
if err == nil {
_, err = ch.Verdicts.Update(ctx, cs.Key, data, cl.rev)
}
if err != nil {
ch.logf("%s: %s, but the verdict could not say so: %v", cs.Key, why.Error(), err)
return
}
stage, _ := cs.Progress.Now()
asks := cs.Asks()
ch.logf("%s: %s; it was at %q with %d asks put %v, which are no longer waited for", cs.Key, why.Error(), stage, len(asks), asks)
}
// judge runs the case within the bound. A judge that fails to judge, and a check that ran past its bound, are verdicts of
// failure naming the machine and the step.
func (ch *Checker) judge(ctx context.Context, cs *Case) Outcome {
@@ -826,7 +467,7 @@ func (ch *Checker) Sweep(ctx context.Context) error {
continue
}
cur.State = delivery.VerdictFail
cur.Failed = delivery.FailedFirst{Machine: "checker", Words: EndedWords + "closed at start: left running, and no candidate holds these heads any more, so nothing will check them"}
cur.Failed = delivery.FailedFirst{Machine: "checker", Words: "closed at start: left running, and no candidate holds these heads any more, so nothing will check them"}
data, err := delivery.EncodeVerdict(cur)
if err != nil {
return err
@@ -861,37 +502,14 @@ func (ch *Checker) TrunkMoved(ctx context.Context) {
if err != nil || !found {
continue // never judged: no status to put out of date
}
v, err := delivery.DecodeVerdict(value)
if err == nil && !ch.stale(v, cs) {
if v, err := delivery.DecodeVerdict(value); err == nil && sameTrunk(v.Trunk, cs.Trunk) {
continue
}
if err == nil && v.State == delivery.VerdictPass {
cs.CarriedFrom = judgedOn(v) // the status says it is being carried to the new commit (ADR 0548 §6)
}
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
}
ch.Reconsider(ctx)
}
// owePending owes mesh/check pending on the heads of a verdict judged on commits that moved while it ran: a pass says it is
// being carried to the trunk's head now (ADR 0548 §6), anything else that it is checked again.
func (ch *Checker) owePending(ctx context.Context, judged *Case, v delivery.Verdict) {
next := judged
ckey, err := delivery.CandidatesKey(v.Group)
ch.mu.Lock()
c, ok := ch.known[ckey]
ch.mu.Unlock()
if err == nil && ok {
if now := CaseOf(c, ch.heads()); now != nil && now.Key == judged.Key {
next = now
}
}
if v.State == delivery.VerdictPass {
next.CarriedFrom = judgedOn(v)
}
ch.writeOwed(ctx, next, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
}
// movedSince says whether a trunk the case was judged on is no longer the trunk's head.
func (ch *Checker) movedSince(cs *Case) bool {
now := ch.heads()
@@ -3,7 +3,6 @@ package check
import (
"context"
"encoding/json"
"fmt"
"strings"
"sync"
"testing"
@@ -334,10 +333,7 @@ func TestTheStatusIsPendingWhileTheCheckRunsAndEndsSuccessOrFailure(t *testing.T
}
}
// No check of heads no candidate holds is left running (issue 1890): at start the sweep ends a key left running; while the
// checker runs, a check whose group moved to other heads is ended at once, unjudged, and writes no status; heads held again
// are checked whole; and a group's check starts only once its heads settled, so a partial set of members is never checked.
func TestHeadsNoCandidateHoldsAreNeverLeftRunning(t *testing.T) {
func TestSweepClosesARunningKeyNoCandidateHoldsAndLeavesOneItDoes(t *testing.T) {
ctx, j := context.Background(), passJudge()
r := newRig(t, j)
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
@@ -355,91 +351,12 @@ func TestHeadsNoCandidateHoldsAreNeverLeftRunning(t *testing.T) {
if err := r.ch.Sweep(ctx); err != nil {
t.Fatal(err)
}
if v := r.verdict(t, dead); !IsEnded(v) || !strings.Contains(v.Failed.Words, "closed at start") {
if v := r.verdict(t, dead); v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "closed at start") {
t.Errorf("the dead key = %+v", v)
}
if v := r.verdict(t, live); v.State != delivery.VerdictRunning {
t.Errorf("the live key = %+v, want it left for the check to take over", v)
}
// A member joins while the check of the lone member's heads runs: that check is ended, its judge's run cancelled.
started, ended := make(chan struct{}), make(chan error, 1)
j = &judgeFunc{f: func(run context.Context, _ *Case, call int) (Outcome, error) {
if call == 1 {
close(started)
<-run.Done()
ended <- context.Cause(run)
return Outcome{}, run.Err()
}
return Outcome{Verdict: passing()}, nil
}}
r = newRig(t, j)
var logged []string
var logMu sync.Mutex
r.ch.Logf = func(f string, a ...any) { logMu.Lock(); logged = append(logged, fmt.Sprintf(f, a...)); logMu.Unlock() }
alone, both := cand, candidateOf(t, "feat/x", pullOf(repoA, 7, head1), pullOf(repoSDK, 3, head2))
r.ch.Candidate(ctx, "feat%2Fx", alone)
<-started
r.ch.Candidate(ctx, "feat%2Fx", both)
var bothC delivery.Candidate
_ = json.Unmarshal(both, &bothC)
select {
case err := <-ended:
if err == nil || !strings.Contains(err.Error(), "superseded by "+GateHeads(bothC)) {
t.Errorf("the old check's run ended with %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("the check of the heads the candidate left still runs")
}
r.ch.Wait()
if v := r.verdict(t, live); !IsEnded(v) || v.Failed.Words != EndedWords+"superseded by "+GateHeads(bothC) {
t.Errorf("the superseded verdict = %+v", v.Failed)
}
if v := r.verdict(t, keyOf(t, "feat/x", both)); v.State != delivery.VerdictPass {
t.Errorf("the group's verdict = %s", v.State)
}
okey, _ := delivery.CheckerOwedKey(repoA, 7, head1)
if raw, _, _, _ := r.owed.Get(ctx, okey); !strings.Contains(string(raw), `"success"`) {
t.Errorf("the head's status = %s, want the group's success, never the ended check's", raw)
}
logMu.Lock()
said := 0
for _, l := range logged {
if strings.Contains(l, "ended: superseded") {
said++
}
}
logMu.Unlock()
if said != 1 {
t.Errorf("the end was said %d times: %v", said, logged)
}
// The member leaves again: the ended heads are held again and checked whole.
r.ch.Candidate(ctx, "feat%2Fx", alone)
r.ch.Wait()
if v := r.verdict(t, live); v.State != delivery.VerdictPass || j.count() != 3 {
t.Errorf("the heads held again = %s after %d judgings, want a pass of a third", v.State, j.count())
}
// Members announced one after another: only the settled, whole set is checked.
j = passJudge()
r = newRig(t, j)
r.ch.Settle = time.Hour
sdk := candidateOf(t, "feat/y", pullOf(repoSDK, 4, head2))
whole := candidateOf(t, "feat/y", pullOf(repoSDK, 4, head2), pullOf(repoA, 8, head1))
r.ch.Candidate(ctx, "feat%2Fy", sdk)
r.ch.Candidate(ctx, "feat%2Fy", whole)
r.ch.Reconsider(ctx) // a trunk move or a turn while the heads settle starts nothing
if j.count() != 0 {
t.Fatalf("judged %d times before the heads settled", j.count())
}
r.ch.settleNow()
r.ch.Wait()
if _, _, found, _ := r.verdicts.Get(ctx, keyOf(t, "feat/y", sdk)); found || j.count() != 1 {
t.Errorf("the partial set has a verdict (%v) or the judge ran %d times, want the whole set alone", found, j.count())
}
if v := r.verdict(t, keyOf(t, "feat/y", whole)); v.State != delivery.VerdictPass {
t.Errorf("the whole set = %s", v.State)
}
}
func TestTheForgeBridgeIsToldOnceForEachChangeOfAStatus(t *testing.T) { // 1683 (the checker's half of AC5)
@@ -28,7 +28,6 @@ func TestAPassingCheckTakesTheAssignersComposition(t *testing.T) {
f := &fakeCompositions{v: delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionPass, SDK: "v0.1.14-x", Machines: []string{"g14"}, Took: 40}}
b.ex.Compositions = f
cs := caseOf(t, pullOf(repoA, 7, head1))
cs.FPSet = "fps-of-the-candidate"
v := b.judge(t, cs).Verdict
if v.State != delivery.VerdictPass || v.Composition == nil || v.Composition.State != delivery.CompositionPass || v.Composition.SDK != "v0.1.14-x" || v.Took.Composition != 40 {
t.Fatalf("verdict = %+v, composition %+v", v, v.Composition)
@@ -39,15 +38,6 @@ func TestAPassingCheckTakesTheAssignersComposition(t *testing.T) {
if !compositionMentions(v, "passed") {
t.Errorf("the line says %q", compositionWords(v.Composition, v.Took.Composition))
}
if v.Composition.FPSet == "" || v.Composition.FPSet != cs.FPSet {
t.Errorf("the composition names the fingerprint set %q, want the candidate's %q (ADR 0548 §5)", v.Composition.FPSet, cs.FPSet)
}
// ADR 0548 §5: a carry whose fingerprints did not change keeps the composition it passed and asks the assigner nothing.
carry := caseOf(t, pullOf(repoA, 7, head1))
carry.CarriedFrom, carry.Composition = map[string]string{repoA: trunkA2}, v.Composition
if kept := b.judge(t, carry).Verdict; kept.Composition == nil || kept.Composition.SDK != "v0.1.14-x" || len(f.asked) != 1 {
t.Errorf("the carry's composition is %+v and the assigner was asked %d times; want the kept one and no second ask", kept.Composition, len(f.asked))
}
}
func compositionMentions(v delivery.Verdict, word string) bool {
@@ -116,9 +116,6 @@ type bench struct {
fetch Fetched
digest map[string]Digested // by level
test func(c stepCall) (any, error)
// declines is how many asks naming needs a node lacking them declines before shanks, which has every need, takes
// one (issue 890); an ask naming none runs on shanks, which then has none.
declines int
}
// refsOf reads the siblings of a fetch's arguments, as they arrive in an ask (JSON) or as the checker gave them.
@@ -154,17 +151,8 @@ func (b *bench) answer(a delivery.StepAsk) delivery.Step {
if module == buildrun.GoBuildModule {
steps.Digest = tool("digest")
}
began := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC)
if len(a.Needs) > 0 {
if b.declines > 0 {
b.declines--
result, _ := json.Marshal(delivery.StepDeclined{Declined: true, Needs: a.Needs, Lacks: a.Needs})
return delivery.Step{ID: a.ID, Tool: a.Kind + " ask", Node: "ace", State: delivery.StepRefused, Started: began, Ended: began,
Reason: "declined", Result: result}
}
steps.Has = func(string) bool { return true }
}
out, err := buildstep.RunAsk(context.Background(), steps, a)
began := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC)
step := delivery.Step{ID: a.ID, Tool: a.Kind + " ask", Node: node, State: delivery.StepPassed, Started: began, Ended: began.Add(2 * time.Second)}
if err != nil {
step.State, step.Reason = delivery.StepFailed, err.Error()
@@ -288,49 +276,6 @@ func TestOnlyThePackageWithoutAKeptVerdictRuns(t *testing.T) { // 1288/AC1
if u := out.Verdict.Levels[delivery.LevelUnit]; u.Run != 2 || u.Kept != 40 {
t.Errorf("unit = %+v, want 2 run and 40 kept", u)
}
// Issue 890/AC5: a package needing node-build-node is left unmet where the node lacks it; the check keeps the passes
// and asks the integration level again for a node with it, puts it again when a node declines it, and that node tests
// the package alone, its kept passes not counted twice.
// fixture/outside has no kept verdict after its pass (it reads outside itself: its digest moves every call), so a
// placed ask that ran more than its unmet packages would run and count it again.
b = newBench(t)
b.declines = 1
outside := 0
b.steps.handlers["build_go_digest"] = func(c stepCall) (any, error) {
d := echoBeside(b, b.digest[c.args["level"].(string)])
if d.Level == "integration" {
outside++
d.Packages = map[string]string{"fixture/slow": digB, "fixture/node": digC, "fixture/outside": fmt.Sprintf("%064x", outside)}
}
return d, nil
}
b.digest[delivery.LevelIntegration] = Digested{Level: "integration",
Needs: []delivery.PackageNeeds{{Package: "fixture/node", Needs: []string{delivery.NeedNodeBuildNode}}}}
out = b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
asks := b.asks.Asks()
if out.Verdict.State != delivery.VerdictPass || len(asks) != 3 || len(asks[0].Needs) != 0 {
t.Fatalf("verdict %+v, asks %+v: want a pass of three asks, the first naming no need", out.Verdict, asks)
}
for _, a := range asks[1:] {
if strings.Join(a.Needs, ",") != delivery.NeedNodeBuildNode || strings.Join(a.Check.Levels, ",") != delivery.LevelIntegration || strings.Join(a.Check.Packages, ",") != "fixture/node" {
t.Errorf("a placed ask %+v, want the integration level needing node-build-node", a)
}
}
tests = b.steps.tool("test")
if last := tests[len(tests)-1]; len(tests) != 3 || strings.Join(packagesOf(last), ",") != "fixture/node" || last.args["level"] != "integration" {
t.Errorf("test calls = %+v, want the unit level, the integration level of fixture/slow and fixture/outside, then fixture/node alone", tests)
}
slow := captured[Tested](t, "go-test-slow")
if i := out.Verdict.Levels[delivery.LevelIntegration]; i.Run != 2*slow.Tests || i.Kept != 0 {
t.Errorf("integration = %+v, want %d run and nothing counted kept", i, 2*slow.Tests)
}
// And where no node has the need, the check fails once the declines outlast the bound, naming the need.
b.kept, b.declines = newMemKV(), 1000
b.ex.Kept, b.ex.DeclineBound = Kept{Store: b.kept}, 200*time.Millisecond
if v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict; v.State != delivery.VerdictFail || !strings.Contains(fmt.Sprint(v), "needs a node with node-build-node") {
t.Errorf("with no node having the need: %+v", v)
}
}
func TestAReAskRunsOnlyWhatFailedAndWhatChanged(t *testing.T) { // 1288/AC2, AC3
@@ -675,8 +620,9 @@ func TestNothingIsKeptForAWorkspaceGivenSiblingsWhenTheDigestDoesNotSayItCoversT
// Beside): every check now places the change's own repository at its trunk (issue 1887 A12).
func echoBeside(b *bench, d Digested) Digested {
var lines []string
if fetches := b.steps.tool("fetch"); len(fetches) > 0 { // the workspace digested is the last fetch's (issue 890: a placed ask fetches again)
for _, r := range refsOf(fetches[len(fetches)-1].args["beside"]) {
for _, f := range b.steps.tool("fetch") {
refs, _ := refsOf(f.args["beside"]), true
for _, r := range refs {
lines = append(lines, r.Repository+" "+r.Commit)
}
}
@@ -42,7 +42,7 @@ func TestTheStatusLineNamesTheFailureNotTheFirstLoggedLine(t *testing.T) { // 19
f := fatalAfterLog()
f.Cause = ""
v = levelFailure(&target{}, "unit", Tested{Failed: []Failure{f}})
if !strings.HasPrefix(afterHid(v.Failed.Words), "agreement_catalogue_test.go:130") {
if !strings.HasPrefix(v.Failed.Words, "agreement_catalogue_test.go:130") {
t.Errorf("words = %q", v.Failed.Words)
}
}
@@ -56,45 +56,18 @@ func TestWordsLongerThanTheBoundKeepTheirLastLines(t *testing.T) { // 1901/AC1
_ = delivery.VerdictFail
}
// An older build module sends no "cause": its answer is still accepted and the verdict's words are its output, as before
// (1901/AC3). A failure that hides its cause (none, or a passing line) is said loudly first in the words and in the status
// line, naming the build module and the version it says it is, and the level still fails; a failure that keeps the
// contract is worded as before (1901/AC8).
func TestAnOlderBuildModulesFailureIsAcceptedWithItsWholeOutput(t *testing.T) { // 1901/AC3, AC8
// An older build module sends no "cause": its answer is still accepted and the verdict's words are its output, as before.
func TestAnOlderBuildModulesFailureIsAcceptedWithItsWholeOutput(t *testing.T) { // 1901/AC3
old := `{"package":"p","test":"TestX","output":"x_test.go:1: logged\nx_test.go:2: the old kept lines"}`
var f Failure
if err := json.Unmarshal([]byte(old), &f); err != nil || f.Cause != "" {
t.Fatalf("1901/AC3: err = %v, cause = %q", err, f.Cause)
}
v := levelFailure(&target{}, "unit", Tested{Failed: []Failure{f}})
if afterHid(v.Failed.Words) != "x_test.go:1: logged\nx_test.go:2: the old kept lines" {
if v.Failed.Words != "x_test.go:1: logged\nx_test.go:2: the old kept lines" {
t.Errorf("1901/AC3: words = %q", v.Failed.Words)
}
if got := firstFailed(*v); !strings.Contains(got, "the build module hid the failure") || strings.Contains(got, "\n") {
t.Errorf("1901/AC3, AC8: status = %q", got)
}
by := "mesh-build-node with mesh-sdk v0.1.14-0.20261026184535-e40dd0f56a78"
for _, c := range []struct {
name, module, by, cause, output, loud string
}{
{"no cause", "mesh-build-node", by, "", "test/b.test.ts: something", "the build module hid the failure (mesh-build-node at mesh-sdk v0.1.14-0.20261026184535-e40dd0f56a78): it sent no cause"},
{"a passing line as its cause", "mesh-build-node", by, "\u2714 the same merge has the same id (8.006913ms)", "", "the build module hid the failure (mesh-build-node at mesh-sdk v0.1.14-0.20261026184535-e40dd0f56a78): its cause is a passing line"},
{"from before the contract", "mesh-build-node", "", "", "\u2714 the same merge has the same id (8.006913ms)", "the build module hid the failure (mesh-build-node at a version before the failure contract): it sent no cause"},
{"a real cause", "mesh-build-node", by, "one plus one is not three", "not ok 3 - fails", ""},
} {
t.Run(c.name, func(t *testing.T) {
v := levelFailure(&target{module: c.module}, "unit", Tested{By: c.by, Failed: []Failure{{Package: "modules/gitea", Test: "fails", Cause: c.cause, Output: c.output}}})
if v.State != delivery.VerdictFail {
t.Errorf("1901/AC8: the verdict is %q, want it failed", v.State)
}
line := firstFailed(*v)
switch {
case c.loud == "" && strings.Contains(v.Failed.Words, "hid the failure"):
t.Errorf("1901/AC8: a failure that keeps the contract is said hidden: %q", v.Failed.Words)
case c.loud != "" && (!strings.HasPrefix(v.Failed.Words, c.loud) || !strings.Contains(line, c.loud) || !strings.Contains(v.Failures[0].Words, "hid the failure")):
t.Errorf("1901/AC8: words %q, status %q, want both to start with %q", v.Failed.Words, line, c.loud)
}
})
if got := firstFailed(*v); !strings.Contains(got, "logged") || strings.Contains(got, "\n") {
t.Errorf("1901/AC3: status = %q", got)
}
}
@@ -4,6 +4,7 @@ import (
"context"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -87,3 +88,29 @@ func TestATrunkMoveMarksEveryOpenPullRequestIntoItPendingAtOnce(t *testing.T) {
t.Errorf("mesh-sdk#13, alone into a trunk that did not move, is %q, want its success to stand", s)
}
}
func TestACheckThatEndsAfterItsTrunkMovedOwesPendingNotItsVerdict(t *testing.T) { // gate (a): a pass on an old trunk is never shown
ctx := context.Background()
var r *rig
release, judged := make(chan struct{}), make(chan struct{})
j := &judgeFunc{f: func(ctx context.Context, c *Case, call int) (Outcome, error) {
if call == 1 {
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) // the trunk moves while it runs
defer close(judged)
} else {
<-release // the check asked again for the new trunk waits in the stream meanwhile
}
return Outcome{Verdict: passing()}, nil
}}
r = newRig(t, j)
r.ch.Candidate(ctx, "feat%2Fx", candidateOf(t, "feat/x", pullOf(repoA, 7, head1)))
<-judged
for deadline := time.Now().Add(2 * time.Second); owedState(t, r, repoA, 7, head1) == "" && time.Now().Before(deadline); time.Sleep(time.Millisecond) {
}
time.Sleep(20 * time.Millisecond)
if s := owedState(t, r, repoA, 7, head1); s != "pending" {
t.Errorf("the status is %q, want pending: the pass was judged on a trunk that moved", s)
}
close(release)
r.ch.Wait()
}
@@ -80,9 +80,6 @@ var askLanguage = map[string]string{buildrun.GoBuildModule: delivery.AskGo, buil
// stopping at the first failure; it judges the step's answer as it judged the tools' answers.
type Examiner struct {
Asks Asks
// DeclineBound is how long a placed ask may go on being declined, from its first decline, before the check fails
// saying no node with its need took it (issue 890, ADR 0552); DefaultDeclineBound when zero.
DeclineBound time.Duration
// Toolchains are the nodes' toolchains: a kept verdict names the toolchain the nodes agree on.
Toolchains ToolchainEntries
Kept Kept
@@ -206,15 +203,12 @@ type state struct {
nodes map[string]bool
clone time.Duration
slow []string
// beside are the siblings read at their trunk's head, by repository (ADR 0548 §6).
beside map[string]string
}
// Judge runs the case (it is the Judge of a Checker). A failure of the change, of a merge or of a step is a verdict; an error is only a case that cannot
// be judged at all.
func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) {
st := &state{j: j, c: c, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{},
beside: map[string]string{}}
st := &state{j: j, c: c, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{}}
if v := st.judgeAll(ctx); v != nil {
return st.finish(ctx, *v), nil
}
@@ -236,13 +230,6 @@ func (st *state) finish(ctx context.Context, v delivery.Verdict) Outcome {
// fails the check, naming the machine it failed at. A try is not composed (it has no candidate); a composition that
// cannot be had leaves it not run, said; with no assigner running it is not judged, said at once.
func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
if st.c.Composition != nil && st.c.Try == nil {
// A carry whose candidate's fingerprints did not change keeps the composition it passed (ADR 0548 §5).
kept := *st.c.Composition
v.Composition = &kept
st.c.note("the composition is kept: no fingerprint of a module it composed changed")
return
}
if st.j.Compositions == nil || st.c.Try != nil {
return
}
@@ -256,9 +243,6 @@ func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
return
}
withComposition(v, comp)
if v.Composition != nil && v.Composition.State != delivery.CompositionNotRun {
v.Composition.FPSet = st.c.FPSet // a carry keeps it while the fingerprints stay the same (ADR 0548 §5)
}
}
// outcome fills the verdict's levels, times, machines and composition from what ran.
@@ -298,9 +282,6 @@ func (st *state) outcome(v delivery.Verdict) Outcome {
}
}
out := Outcome{Verdict: v, Slowest: slow}
if len(st.beside) > 0 {
out.Beside = st.beside
}
for _, t := range st.targets {
if t.judged != "" {
if out.Judged == nil {
@@ -362,7 +343,6 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
if len(beside) > 0 {
args["beside"] = beside
t.beside = besideLines(beside)
besideAtTrunk(st.c, beside, st.beside)
}
if p.Repository == catalogueRepository {
t.runner = RunnerWorkspace // used by the build module when the merge changes it (ADR 0535 §1)
@@ -384,25 +364,11 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
return nil
}
// DefaultDeclineBound is an Examiner's DeclineBound when it names none: longer than a check runs, so a node with the
// need that is busy is waited for, and short enough that a mesh where no node has it says so.
const DefaultDeclineBound = 30 * time.Minute
// ask puts the target's check ask, waits for the step of the node that pulled it and judges its answer: the fetch, the
// pins of its go.mod files, then each level's digest, kept verdicts and tests. When the answer leaves packages unmet,
// their tests needing what that node lacks (issue 890), the passes are kept and the levels from the first such one are
// asked again of a node that has it (placed).
// pins of its go.mod files, then each level's digest, kept verdicts and tests.
func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *delivery.Verdict {
return st.askNeeding(ctx, t, merged, st.levelsOf(), nil, nil)
}
// askNeeding is ask with the levels to run, what the node must have and the packages it tests: none for the first ask
// of a target; for the placed one the unmet packages' needs and those packages alone, so no package the first ask judged
// is run or counted again (ADR 0552 §4).
func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, levels, needs, packages []string) *delivery.Verdict {
p := t.pull
placed := len(needs) > 0
job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: levels, Kept: true, Packages: packages}
job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: st.levelsOf(), Kept: true}
if t.runner != "" {
job.Runner, job.RunnerWhen = t.runner, buildGoDir
}
@@ -414,53 +380,17 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
kind = delivery.AskRecheck // a re-judge after a trunk move goes after every new head's check (ADR 0536 §4)
}
a := delivery.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: askLanguage[t.module], Asker: delivery.CheckerModule,
At: st.j.now().UTC(), Check: job, Needs: needs}
var step delivery.Step
var firstDecline time.Time
for {
what := fmt.Sprintf("the %s ask %s of %s#%d", kind, a.ID, p.Repository, p.Number)
if placed {
what += " for a node with " + strings.Join(needs, ", ")
At: st.j.now().UTC(), Check: job}
st.c.Progress.Set(fmt.Sprintf("the %s ask %s of %s#%d", kind, a.ID, p.Repository, p.Number), "")
if err := st.j.Asks.Put(ctx, a); err != nil {
return fail(t.module, fmt.Sprintf("the check of %s#%d could not be asked: %v", p.Repository, p.Number, err))
}
step, err := st.j.Asks.Wait(ctx, a)
if err != nil {
if ctx.Err() != nil {
return fail(t.module, ctx.Err().Error())
}
st.c.Progress.Set(what, "")
st.c.Asked(a.ID)
if err := st.j.Asks.Put(ctx, a); err != nil {
return fail(t.module, fmt.Sprintf("the check of %s#%d could not be asked: %v", p.Repository, p.Number, err))
}
waiting := ctx
if watch := st.c.Watch; watch != nil {
id := a.ID
watch(id, nil) // put: a try names it at once, not only when it ends (issue 1890)
waiting = buildrun.WithStepSeen(ctx, func(s delivery.Step) { watch(id, &s) })
}
var err error
step, err = st.j.Asks.Wait(waiting, a)
if err != nil {
if ctx.Err() != nil {
return fail(t.module, ctx.Err().Error())
}
return fail(t.module, fmt.Sprintf("the check of %s#%d: %v", p.Repository, p.Number, err))
}
d, declined := delivery.DeclinedOf(step)
if !declined {
break
}
// A node that lacks the need dropped the ask (issue 890): it is put again, for a node that has it to take; the
// node that declined looks at the stream again only after its next pull's wait.
now := st.j.now()
if firstDecline.IsZero() {
firstDecline = now
}
bound := st.j.DeclineBound
if bound <= 0 {
bound = DefaultDeclineBound
}
if now.Sub(firstDecline) > bound {
return fail(step.Node, fmt.Sprintf("the check of %s#%d needs a node with %s, and none took it in %s: every node that pulled it lacks it (last %s)",
p.Repository, p.Number, strings.Join(needs, ", "), bound, step.Node))
}
st.j.logf("%s#%d: the ask %s was declined on %s, which lacks %s; asked again", p.Repository, p.Number, a.ID, step.Node, strings.Join(d.Lacks, ", "))
a.ID, a.At = buildstep.NewID(), now.UTC()
return fail(t.module, fmt.Sprintf("the check of %s#%d: %v", p.Repository, p.Number, err))
}
// The fetch and the digests are what of the ask's time its levels' running did not take (Took.Clone).
took := time.Duration(0)
@@ -484,31 +414,18 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
if st.c.Try == nil {
st.j.writePreview(ctx, p, append(*merged, t.fetched))
}
return fail(step.Node, fmt.Sprintf("%s#%d %s%s", p.Repository, p.Number, ConflictWords, strings.Join(t.fetched.Conflicts, ", ")))
return fail(step.Node, fmt.Sprintf("%s#%d does not merge into the trunk: files in conflict: %s", p.Repository, p.Number, strings.Join(t.fetched.Conflicts, ", ")))
}
t.changed = t.fetched.Changed
if !changesBuildGo(p.Repository, t.changed) {
t.runner = "" // the build module used no runner: the merge does not change it
}
if !placed { // the placed ask fetched the same merge: its preview and pins were read from the first
*merged = append(*merged, t.fetched)
if v := st.checkPinsOf(ctx, t); v != nil {
return v
}
if v := st.checkBinariesOf(t); v != nil {
return v
}
*merged = append(*merged, t.fetched)
if v := st.checkPinsOf(ctx, t); v != nil {
return v
}
ran := 0.0
for _, la := range answer.Levels {
if placed {
if len(la.Unmet) > 0 {
st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
return fail(step.Node, fmt.Sprintf("the %s level of %s#%d left %s untested on %s, which was asked for a node with %s",
la.Level, p.Repository, p.Number, unmetNames(la.Unmet), step.Node, strings.Join(needs, ", ")))
}
la.Kept = nil // it names its packages and takes no other's kept verdict; this is only a guard against an older module
}
v, elapsed := st.level(ctx, t, la)
ran += elapsed
if v != nil {
@@ -517,48 +434,9 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
}
}
st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
if unmet := delivery.UnmetNeeds(answer.Levels); len(unmet) > 0 && !placed {
// Every level passed, its passes kept, and some packages were left to a node with what their tests need: the levels
// from the first that left one are asked of such a node, which takes the kept passes and tests those (issue 890).
var again, packages []string
seen := map[string]bool{}
for _, la := range answer.Levels {
if len(again) > 0 || len(la.Unmet) > 0 {
again = append(again, la.Level)
}
for _, u := range la.Unmet {
if !seen[u.Package] {
seen[u.Package] = true
packages = append(packages, u.Package)
}
}
}
st.j.logf("%s#%d: %s on %s left untested; asked of a node with %s", p.Repository, p.Number, unmetOf(answer.Levels), step.Node, strings.Join(unmet, ", "))
return st.askNeeding(ctx, t, merged, again, unmet, packages)
}
return nil
}
// unmetNames names the unmet packages and their needs.
func unmetNames(unmet []delivery.PackageNeeds) string {
var out []string
for _, u := range unmet {
out = append(out, u.Package+" (needs "+strings.Join(u.Needs, ", ")+")")
}
return strings.Join(out, ", ")
}
// unmetOf names the unmet packages of every level.
func unmetOf(levels []delivery.LevelAnswer) string {
var out []string
for _, la := range levels {
if len(la.Unmet) > 0 {
out = append(out, "at "+la.Level+" "+unmetNames(la.Unmet))
}
}
return strings.Join(out, "; ")
}
// level judges one level of a check ask's answer as the tools' answers were judged: the digest keys the kept verdicts
// of the packages that passed, the kept ones taken are counted, and every test result is read and counted, also after
// one that did not pass, so the verdict names every failed test of the level (issue 1921). It answers the level's
@@ -663,12 +541,6 @@ const MaxWords = 4096
// no cause told apart (an older build module) the cause is the output's first "panic:", "fatal error:" or "signal:"
// line, read before the output is cut, or the words say the tests passed and the program failed after them.
func failureWords(f Failure) string {
return failureWordsAfter(f, "")
}
// failureWordsAfter is failureWords with the line loud put first, within the same MaxWords: what the checker itself says of
// the failure (that the build module hid it), which the status line then names.
func failureWordsAfter(f Failure, loud string) string {
out, passed := dropSummary(strings.TrimSpace(f.Output))
cause := strings.TrimSpace(f.Cause)
if cause == "" {
@@ -684,14 +556,11 @@ func failureWordsAfter(f Failure, loud string) string {
if len(cause) > MaxCause {
cause = strings.ToValidUTF8(cause[:MaxCause], "") + "..."
}
// The words keep MaxWords exactly: the loud line, the cause and their newlines, the marker and the kept lines.
// The words keep MaxWords exactly: the cause and its newline, the marker and the kept lines (review round 1).
budget := MaxWords
if cause != "" {
budget -= len(cause) + 1
}
if loud != "" {
budget -= len(loud) + 1
}
if len(out) > budget {
const marker = "... "
tail := out[len(out)-(budget-len(marker)):]
@@ -704,31 +573,9 @@ func failureWordsAfter(f Failure, loud string) string {
if cause != "" && !strings.HasPrefix(firstLine(out), cause) {
out = strings.TrimRight(cause+"\n"+out, "\n")
}
if loud != "" {
out = strings.TrimRight(loud+"\n"+out, "\n")
}
return out
}
// hidWords is the checker's loud line for a failure the build module sent hiding its cause (issue 1901, its thorough
// fix): with no cause, or a passing line as its cause, under the failure contract (buildstep.Hidden). It names the build
// module and the version it says it is (Tested.By, "<module> with mesh-sdk <version>"); one from before the contract
// says nothing of itself. "" when the failure keeps the contract. The check fails either way: only its words change.
func hidWords(module, by string, f Failure) string {
why := buildstep.Hidden(f)
if why == "" {
return ""
}
version := "a version before the failure contract"
if module == "" {
module = "an unnamed build module"
}
if m, v, ok := strings.Cut(by, " with "); ok && m != "" {
module, version = m, v
}
return fmt.Sprintf("the build module hid the failure (%s at %s): %s", module, version, why)
}
// MaxCause is the most a cause in a failure's words holds: one long line (the 1901 review: a cause of a kilobyte could
// push the words past MaxWords).
const MaxCause = 1024
@@ -815,7 +662,7 @@ func levelFailure(t *target, level string, results ...Tested) *delivery.Verdict
if t.module == buildrun.NodeBuildModule && f.Package == "" {
name = res.Module
}
all = append(all, delivery.FailedFirst{Test: level + ": " + name, Words: failureWordsAfter(f, hidWords(t.module, res.By, f))})
all = append(all, delivery.FailedFirst{Test: level + ": " + name, Words: failureWords(f)})
}
}
if len(all) == 0 { // no result: a level said failed with nothing to name
@@ -56,22 +56,12 @@ func (k Kept) Keep(ctx context.Context, level, digest string, v delivery.Kept) (
// KeepPasses keeps, for the packages of a test result that passed with tests, a verdict under the digest the build module
// named for the package at the level. A package that failed, has no tests or has no digest is never kept (ADR 0444 clause
// 1): it runs every check. A package with skipped tests is not kept either: a pass that skipped (a test needing a
// toolchain the node lacks) would be reused for its digest and never run anywhere (issue 890). A build module that
// counts skips only per result, not per package, keeps nothing of a result in which a test skipped. It answers the
// packages it kept now.
// 1): it runs every check. It answers the packages it kept now.
func (k Kept) KeepPasses(ctx context.Context, repository, level, toolchain string, res Tested, digests map[string]string, at time.Time) ([]string, error) {
var kept []string
perPackage := false
for _, p := range res.Packages {
perPackage = perPackage || p.Skipped > 0
}
if res.Skipped > 0 && !perPackage {
return kept, nil // an older build module: the skip cannot be placed in a package
}
for _, p := range res.Packages {
digest, ok := digests[p.Package]
if !ok || !p.Passed || p.NoTests || p.Tests == 0 || p.Skipped > 0 {
if !ok || !p.Passed || p.NoTests || p.Tests == 0 {
continue
}
if failedIn(res, p.Package) {
@@ -74,32 +74,6 @@ func TestOnlyPassingPackagesWithADigestAreKept(t *testing.T) { // 1287/AC3, AC4
}
}
func TestAResultWithSkippedTestsKeepsNothing(t *testing.T) { // issue 890: a pass that skipped proved nothing
ctx, st := context.Background(), newMemKV()
k := Kept{Store: st}
res := Tested{Level: "integration", Passed: true, Skipped: 2, Packages: []GoPackage{{Package: "m/a", Passed: true, Tests: 4}}}
kept, err := k.KeepPasses(ctx, "novox/mesh-catalog", "integration", "go1.26.0", res, map[string]string{"m/a": digA}, at0)
if err != nil || len(kept) != 0 {
t.Fatalf("kept %v, %v, want nothing", kept, err)
}
if keys, _ := st.Keys(ctx); len(keys) != 0 {
t.Errorf("the bucket holds %v, want none", keys)
}
}
func TestOnlyTheSkippingPackageOfAResultIsWithheld(t *testing.T) { // issue 890
ctx, st := context.Background(), newMemKV()
k := Kept{Store: st}
res := Tested{Level: "integration", Passed: true, Skipped: 2, Packages: []GoPackage{
{Package: "m/a", Passed: true, Tests: 4, Skipped: 2},
{Package: "m/b", Passed: true, Tests: 3},
}}
kept, err := k.KeepPasses(ctx, "novox/mesh-catalog", "integration", "go1.26.0", res, map[string]string{"m/a": digA, "m/b": digB}, at0)
if err != nil || strings.Join(kept, ",") != "m/b" {
t.Fatalf("kept %v, %v, want only m/b", kept, err)
}
}
func TestAKeptValueOfAnotherDigestIsNotRead(t *testing.T) { // 1287/AC7
ctx, st := context.Background(), newMemKV()
k := Kept{Store: st}
@@ -85,22 +85,18 @@ type pieces struct {
head, commits, counts, shortCounts, times, shortTimes, over, builds, composition, shortComposition, group string
// judged names the program that judged a change to a build module (ADR 0535 §5); never dropped.
judged string
// carried says, of a carried verdict, the commits it was carried from (ADR 0548 §8); never dropped.
carried string
// warning says that a part of the check did not run, for instance that an older build module could not refuse a compiled program; never dropped.
warning string
}
// pullKey names a pull request of a case: `<owner/repo>#<number>`.
func pullKey(p Pull) string { return fmt.Sprintf("%s#%d", p.Repository, p.Number) }
func (p pieces) join(drop int) string {
parts := []string{p.head, p.commits, p.carried, p.warning, p.counts, p.judged}
parts := []string{p.head, p.commits, p.counts, p.judged}
switch {
case drop < 3 && p.times != "":
parts = append(parts, p.times)
case drop == 3 && p.shortTimes != "":
parts = []string{p.head, p.commits, p.carried, p.warning, p.shortCounts, p.judged, p.shortTimes}
parts = []string{p.head, p.commits, p.shortCounts, p.judged, p.shortTimes}
}
if p.over != "" {
parts = append(parts, p.over)
@@ -175,62 +171,16 @@ func buildWords(builds []BuildNote) string {
return "builds: " + strings.Join(parts, ", ")
}
// carriedWords say the commits a verdict was carried from, for the line of pull request p (ADR 0548 §8): its own trunk's
// commit as twelve characters, a sibling's with its name before it; only those that moved, sorted. Empty when none did.
func carriedWords(from map[string]string, v delivery.Verdict, cs *Case, p Pull) string {
var own string
var others []string
for r, old := range from {
now, ok := v.Trunk[r]
if cs != nil && !ok {
now, ok = cs.Trunk[r]
}
if !ok {
now = v.Beside[r]
}
if now == "" || now == old {
continue
}
if strings.EqualFold(r, p.Repository) {
own = short12(old)
continue
}
name := r
if i := strings.LastIndex(r, "/"); i >= 0 {
name = r[i+1:]
}
others = append(others, name+" "+short12(old))
}
slices.Sort(others)
if own != "" {
others = append([]string{own}, others...)
}
return strings.Join(others, ", ")
}
// parts makes the pieces of the line of a pull request of a case.
func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string, judged map[string]string) pieces {
var pc pieces
pc.judged = judged[pullKey(p)]
if cs != nil {
pc.warning = cs.Warning(pullKey(p))
}
commits := short12(p.Head) + " on " + short12(p.Trunk)
switch v.State {
case delivery.VerdictRunning:
pc.head = "checking " + commits
if cs != nil && cs.CarriedFrom != nil {
// Out of date and being carried to the new commit (ADR 0548 §6): pending, and saying so.
pc.head = "being carried to " + commits
if w := carriedWords(cs.CarriedFrom, v, cs, p); w != "" {
pc.carried = "from " + w
}
}
case delivery.VerdictPass:
pc.head = "pass " + commits
if w := carriedWords(v.CarriedFrom, v, cs, p); w != "" {
pc.carried = "carried from " + w // the counts that follow say what the carry ran and kept
}
default:
pc.head = "failed: " + firstFailed(v)
if n := failedCount(v); n > 1 {
@@ -1,34 +0,0 @@
package check
import (
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// **An ask the build seat gave up on ends its step** (novox/hq issue 734): mesh-sdk buildrun.LostAskWaiter hands the
// waiting asker a failed step on the node buildrun.LostNode when DEAD_LETTERS keeps the ask's copy (buildrun.HearDeadLetters,
// granted by buscompose DeadLetterReads), and, under buildrun.NotTakenBound, when no build node took the ask within the
// bound. What follows is the checker's: a try ends as error (Tryer.run), with the not-taken bound at the step bound
// (Tryer.NotTaken); a check is asked again under its claim, at most LostAskTries times, then fails as not heard and is
// judged again when the nodes change (Checker.try). A check's ask has no not-taken bound: it waits in the stream as long as
// the backlog is (ADR 0525 §9), and a check ended for a backlog would owe its pull request a failure.
// LostAskTries is how often a check asks again whose ask was given up, before it fails as not heard (UnheardWords) and is
// judged again when the nodes change.
const LostAskTries = 3
// IsLostAsk says whether a verdict failed because the build seat gave its ask up (buildrun.LostWords).
func IsLostAsk(v delivery.Verdict) bool {
return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, buildrun.LostWords)
}
// IsNotTaken says whether a verdict failed because no build node took its ask within the bound (buildrun.NotTakenWords).
func IsNotTaken(v delivery.Verdict) bool {
return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, buildrun.NotTakenWords)
}
// HearingBucket is the checker's state hearing: whether it hears DEAD_LETTERS (buildrun.Hearing under
// buildrun.HearingKey), which checker_status says (dead_letters).
const HearingBucket = delivery.CheckerModule + "_" + buildrun.HearingState
@@ -1,170 +0,0 @@
package check
import (
"context"
"encoding/json"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// losingAsks is the build seat over a bus that gives up the first `lose` asks put (DEAD_LETTERS keeps them, as on
// 2026-10-26 during a bus reload: try-1793017340003993788) and answers the rest; with never set it neither answers nor
// gives up (an ask no node takes).
type losingAsks struct {
*buildrun.FakeAsks
answer func(delivery.StepAsk) delivery.Step
dead *buildrun.DeadLetters
never bool
mu sync.Mutex
lose int
}
func (l *losingAsks) PublishAsk(ctx context.Context, subject, id string, data []byte) error {
if err := l.FakeAsks.PublishAsk(ctx, subject, id, data); err != nil {
return err
}
l.mu.Lock()
lost := l.lose > 0
if lost {
l.lose--
}
l.mu.Unlock()
switch {
case l.never:
case lost:
h := map[string][]string{buildrun.DeadHeaderMsgID: {id}, buildrun.DeadHeaderDeliveries: {"5"}, buildrun.DeadHeaderStream: {"SEAT_NODE_BUILD_GO"},
buildrun.DeadHeaderGaveUp: {"2026-10-26T12:26:40Z"}}
g, ok := buildrun.GivenUpOf(h)
if !ok {
return nil
}
go l.dead.Heard(g)
default:
a, err := delivery.DecodeStepAsk(data)
if err != nil {
return err
}
l.Write(l.answer(a))
}
return nil
}
func losing(b *bench, lose int) *losingAsks {
l := &losingAsks{FakeAsks: &buildrun.FakeAsks{}, answer: b.answer, dead: &buildrun.DeadLetters{}, lose: lose}
b.ex.Asks = buildrun.Asker{Publisher: l, Waiter: buildrun.LostAskWaiter{Inner: l.FakeAsks, Dead: l.dead}, Bound: 5 * time.Second}
return l
}
// endable is the tries state as the end tool sees it.
type endable struct{ kv *memKV }
func (e endable) Entry(key string) (json.RawMessage, uint64, error) {
v, rev, _, err := e.kv.Get(context.Background(), key)
return v, rev, err
}
func (e endable) Update(key string, value []byte, revision uint64) error {
_, err := e.kv.Update(context.Background(), key, value, revision)
return err
}
// Issue 734: a try whose ask the build seat gave up on ends as error saying so, and the next ask of its head is a new try;
// one no node takes ends at the step bound; one ended by hand frees its place and stops being waited for.
func TestAnAskGivenUpOrNeverTakenEndsItsTryAsErrorAndItsHeadIsAskedAnew(t *testing.T) {
ctx := context.Background()
for _, row := range []struct {
name string
lose int
never bool
want string
}{
{"given up by the build seat", 1, false, buildrun.LostWords + " after 5 deliveries at 2026-10-26T12:26:40Z"},
{"taken by no build node within the bound", 0, true, buildrun.NotTakenWords + " within 200ms"},
} {
t.Run(row.name, func(t *testing.T) {
kv := newMemKV()
at := time.Date(2026, 10, 26, 12, 22, 20, 0, time.UTC)
id, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, at)
if err != nil {
t.Fatal(err)
}
b := newBench(t)
l := losing(b, row.lose)
l.never = row.never
tr := &Tryer{Tries: kv, Judge: b.ex, NotTaken: 200 * time.Millisecond}
raw, _, _, _ := kv.Get(ctx, id)
tr.Put(ctx, id, raw)
tr.Wait()
got := readTry(t, kv, id)
if got.State != TryError || len(got.Failed) != 1 || !strings.Contains(got.Failed[0], row.want) || !strings.Contains(got.Said, row.want) {
t.Fatalf("try = %+v, want error saying %q", got, row.want)
}
again, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, at.Add(time.Minute))
if err != nil || again == id {
t.Errorf("the head asked again answered %q, %v, want a new try", again, err)
}
})
}
t.Run("ended by hand", func(t *testing.T) {
kv := newMemKV()
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now())
b := newBench(t)
losing(b, 0).never = true
tr := &Tryer{Tries: kv, Judge: b.ex} // no bound: only the hand ends it
raw, _, _, _ := kv.Get(ctx, id)
tr.Put(ctx, id, raw)
deadline := time.Now().Add(2 * time.Second)
for readTry(t, kv, id).State != TryRunning && time.Now().Before(deadline) {
time.Sleep(5 * time.Millisecond)
}
if _, err := EndTry(endable{kv}, id, "", time.Now()); err == nil {
t.Error("an end without a why was taken")
}
ended, err := EndTry(endable{kv}, id, "its ask was lost in the bus reload", time.Now())
if err != nil || ended.State != TryError || !strings.HasPrefix(ended.Said, "ended by hand ("+TryEndTool+"): its ask was lost") {
t.Fatalf("ended = %+v, %v", ended, err)
}
raw, _, _, _ = kv.Get(ctx, id)
tr.Put(ctx, id, raw) // the watch of the tries hands the end over
tr.Wait() // returns: the try's judging stopped
if got := readTry(t, kv, id); got.State != TryError || got.Said != ended.Said {
t.Errorf("try = %+v, want the hand's end kept", got)
}
if _, err := EndTry(endable{kv}, id, "again", time.Now()); err == nil || !strings.Contains(err.Error(), "ended already") {
t.Errorf("a second end answered %v", err)
}
})
}
// Issue 734: a pull request's check whose ask the build seat gave up on is asked again under its claim, its status
// pending, and judged; one given up every time fails as not heard and is judged again when the nodes change.
func TestACheckWhoseAskWasGivenUpIsAskedAgain(t *testing.T) {
ctx := context.Background()
for _, row := range []struct {
lose int
state string
asks int
}{
{1, delivery.VerdictPass, 2},
{LostAskTries, delivery.VerdictFail, LostAskTries},
} {
b := newBench(t)
l := losing(b, row.lose)
r := newRig(t, b.ex)
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
v := r.verdict(t, keyOf(t, "feat/x", cand))
if v.State != row.state || len(l.Asks()) != row.asks {
t.Errorf("lose %d: verdict %s (%s) after %d asks, want %s after %d", row.lose, v.State, v.Failed.Words, len(l.Asks()), row.state, row.asks)
}
if row.state == delivery.VerdictFail && (!IsLostAsk(v) || !IsNodesFailure(v)) {
t.Errorf("a check given up every time = %q, want lost and judged again when the nodes change", v.Failed.Words)
}
}
}
@@ -38,11 +38,11 @@ func TestAPackageFailureIsNeverWordedPass(t *testing.T) { // 1901/AC5
t.Errorf("1901/AC5: the words keep %q: %q", l, v.Failed.Words)
}
}
if !strings.HasPrefix(afterHid(v.Failed.Words), c.want) {
t.Errorf("1901/AC5: words after the loud line start %q, want %q", firstLine(afterHid(v.Failed.Words)), c.want)
if !strings.HasPrefix(v.Failed.Words, c.want) {
t.Errorf("1901/AC5: words start %q, want %q", firstLine(v.Failed.Words), c.want)
}
if got := firstFailed(*v); !strings.Contains(got, "the build module hid the failure") || strings.HasSuffix(got, ": PASS") {
t.Errorf("1901/AC5, AC8: status = %q", got)
if got := firstFailed(*v); !strings.Contains(got, c.want) || strings.HasSuffix(got, ": PASS") {
t.Errorf("1901/AC5: status = %q", got)
}
})
}
@@ -63,7 +63,7 @@ func TestALongCauseIsCutToALinesBound(t *testing.T) { // 1901/AC5
func TestAPackageStoppedWithOnlyItsFailLineIsStillWorded(t *testing.T) { // 1901/AC5
for _, out := range []string{"FAIL\texample.com/timeout\t2.017s", "FAIL\texample.com/exiting\t0.006s", "FAIL\texample.com/killed\t0.006s"} {
v := levelFailure(&target{}, "integration", Tested{Failed: []Failure{{Package: "example.com/x", Output: out}}})
if !strings.HasPrefix(afterHid(v.Failed.Words), "the package failed with no test to name and no panic, signal or build error in its output") {
if !strings.HasPrefix(v.Failed.Words, "the package failed with no test to name and no panic, signal or build error in its output") {
t.Errorf("1901/AC5: %q is worded %q", out, v.Failed.Words)
}
}
@@ -74,7 +74,7 @@ func TestAPackageStoppedWithOnlyItsFailLineIsStillWorded(t *testing.T) { // 1901
func TestOnlyGosExactFormsAreSummaryOrCause(t *testing.T) { // 1901/AC5
out := " x_test.go:3: ok so far\nok then: the seat is free\n panic: child retried and recovered\npanic: the real one\nFAIL\tp\t1.0s"
v := levelFailure(&target{}, "unit", Tested{Failed: []Failure{{Package: "p", Output: out}}})
if !strings.HasPrefix(afterHid(v.Failed.Words), "panic: the real one") || !strings.Contains(v.Failed.Words, "ok then: the seat is free") {
if !strings.HasPrefix(v.Failed.Words, "panic: the real one") || !strings.Contains(v.Failed.Words, "ok then: the seat is free") {
t.Errorf("1901/AC5: words = %q", v.Failed.Words)
}
}
@@ -90,13 +90,3 @@ func TestTheWordsKeepTheirBoundExactly(t *testing.T) { // 1901/AC5
}
}
}
// afterHid is a verdict's words after the checker's loud line, which a failure with no cause from an older build module
// now starts with (1901/AC8).
func afterHid(w string) string {
if strings.HasPrefix(w, "the build module hid the failure") {
_, rest, _ := strings.Cut(w, "\n")
return rest
}
return w
}
@@ -240,12 +240,7 @@ func TestTheSiblingsAreWhereTheTestsStateThem(t *testing.T) { // review D2, ADR
catalogue + " go.mod " + head2: "module git.novox.be/novox/mesh-catalog\n"}, onTrunk: map[string]bool{"cb54756c3865": true}}
b.ex.Forge = forge
// A group of the controller and the catalogue: the controller's tests read the catalogue's trunk, not the group's head.
out := b.judge(t, caseOf(t, pullOf(controller, 7, head1), pullOf(catalogue, 12, head2)))
// ADR 0548 §6, issue 568's own example: the verdict names the node-engine at the commit the mesh runs, and mesh-sdk at the
// trunk the catalogue's tests read it at; the catalogue, a member here, is not named.
if out.Beside[host] != hostRuns || out.Beside[catalogue] != "" || out.Beside[SDKRepository] != "9999999999999999999999999999999999999999" {
t.Errorf("the siblings named for the verdict = %v, want the node-engine at %s and mesh-sdk at its trunk", out.Beside, hostRuns[:12])
}
b.judge(t, caseOf(t, pullOf(controller, 7, head1), pullOf(catalogue, 12, head2)))
got := besideOf(t, b, controller)
if c := got[catalogue]; c.Commit != trunkS || c.Ref != "" {
t.Errorf("the catalogue beside the controller = %+v, want its trunk %s", c, trunkS)
@@ -329,11 +324,7 @@ func TestTheChangesOwnRepositoryIsBesideItAtItsTrunk(t *testing.T) {
func TestACatalogueChangeHasTheCatalogueBesideItAtItsTrunk(t *testing.T) {
b := newBench(t)
b.ex.Trunks = trunksOf(t, map[string]string{catalogueRepository: trunkA, SDKRepository: trunkS, controllerRepo: trunkA2})
out := b.judge(t, caseOf(t, pullOf(catalogueRepository, 12, head1)))
// ADR 0548 §6: the outcome names the siblings read at their trunk's head, never the repository it merges.
if out.Beside[SDKRepository] != trunkS || out.Beside[controllerRepo] != trunkA2 || out.Beside[catalogueRepository] != "" {
t.Errorf("the siblings named for the verdict = %v, want mesh-sdk and mesh-controller at their trunks and not the catalogue", out.Beside)
}
b.judge(t, caseOf(t, pullOf(catalogueRepository, 12, head1)))
got := besideOf(t, b, catalogueRepository)
if c := got[catalogueRepository]; c.Commit != trunkA || c.Ref != "" {
t.Fatalf("the catalogue beside its own change = %+v, want its trunk %s", c, trunkA)
@@ -75,40 +75,3 @@ func TestCheckerStatusNamesTheWaitingChecksByKind(t *testing.T) { // 1897/AC4
t.Errorf("waiting = %v, want %v", st.Waiting, want)
}
}
// 568 note 21: a verdict that read the node-engine at its running commit is not waiting for a trunk move while that commit
// is still the one the mesh runs, though the node-engine's trunk has moved on (ADR 0548 §6).
func TestCheckerStatusReadsTheNodeEngineAtItsRunningCommit(t *testing.T) { // 568/21
cand := candidateOf(t, group(1), pullOf(repoA, 1, head1))
ckey, _ := delivery.CandidatesKey(group(1))
vkey := keyOf(t, group(1), cand)
dc, _ := delivery.DecodeCandidate(cand)
judged, err := delivery.EncodeVerdict(delivery.Verdict{Group: group(1), Heads: dc.HeadsHash, Trunk: map[string]string{repoA: trunkA}, Beside: map[string]string{nodeEngineRepo: "run111"}, State: delivery.VerdictPass, Levels: passing().Levels})
if err != nil {
t.Fatal(err)
}
heads := func() (map[string]string, error) {
return map[string]string{repoA: trunkA, nodeEngineRepo: "trunk222"}, nil
}
for _, row := range []struct {
name string
runs runsAt
want int
}{
{"the running commit the verdict read", runsAt{nodeEngineModule: "run111"}, 0},
{"the running commit moved", runsAt{nodeEngineModule: "run999"}, 1},
{"no running commit known: the trunk's", nil, 1},
} {
rd := Reader{CandidateState: fakeState{ckey: cand}.read(), VerdictState: fakeState{vkey: judged}.read(), Trunks: heads}
if row.runs != nil {
rd.Runs = row.runs
}
st, err := rd.Status()
if err != nil {
t.Fatal(err)
}
if got := len(st.Waiting[KindTrunk]); got != row.want {
t.Errorf("%s: waiting for a trunk move = %v, want %d", row.name, st.Waiting[KindTrunk], row.want)
}
}
}
@@ -8,7 +8,6 @@ import (
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// The read tools (novox/hq ADR 0484 clauses 8 and 13, issue 1686): what the checker stored, as stored, with the line of the
@@ -40,14 +39,7 @@ type Reader struct {
StepBound time.Duration // how long the checker waits for one step; DefaultStepBound when 0
// Trunks are the trunk heads of every repository (the gate's source); nil names no re-judge after a trunk move.
Trunks func() (map[string]string, error)
// TurnState is the gate's merge turns (ADR 0548 §3); without it no check is named as held by a turn.
TurnState StateRead
// Runs says which commit the mesh runs of a module (the indexer's modules), as the checker's decision reads it: the
// node-engine a verdict names is compared at its running commit, not at the trunk's (ADR 0548 §6); nil knows none.
Runs interface{ CommitOf(module string) string }
// HearingState is the checker's state hearing: whether it hears DEAD_LETTERS (issue 734).
HearingState StateRead
Now func() time.Time
Now func() time.Time
}
// VerdictRead is a verdict as stored with the full report of the status of each pull request it judged (Report), when the
@@ -320,68 +312,7 @@ type Status struct {
// Waiting are the candidates whose check waits, by kind (ADR 0536 §4): new (no verdict for its heads) and nodes (failed
// for the nodes, asked again when they change) start before trunk (judged on trunk heads that moved since).
Waiting map[string][]string `json:"waiting_by_kind"`
// HeldByTurn are the candidates whose check an approved merge's turn holds off (ADR 0548 §3), by group: the repository and
// the group of the merge that waits. Their check is claimed when that turn goes.
HeldByTurn map[string]string `json:"held_by_turn"`
// DeadLetters says whether the checker hears the asks the build seats give up on (issue 734): deaf, a try or check
// whose ask is given up waits until its bound.
DeadLetters DeadLettersHeard `json:"dead_letters"`
Notes []string `json:"notes,omitempty"`
}
// DeadLettersHeard is the checker's hearing of DEAD_LETTERS as the status says it: healthy or not, and why.
type DeadLettersHeard struct {
Heard bool `json:"heard"`
Why string `json:"why"`
}
// deadLetters reads the checker's kept hearing (buildrun.HearingHealth).
func (r Reader) deadLetters() DeadLettersHeard {
if r.HearingState.Get == nil {
return DeadLettersHeard{Why: "the checker's state hearing is not readable here"}
}
raw, err := r.HearingState.Get(buildrun.HearingKey)
if err != nil {
return DeadLettersHeard{Why: "the checker's state hearing cannot be read: " + err.Error()}
}
a := buildrun.HearingHealth(raw)
return DeadLettersHeard{Heard: a.Healthy, Why: a.Why}
}
// turns reads the gate's merge turns, by repository; nil when they cannot be read, said in a note.
func (r Reader) turns(st *Status) *Turns {
if r.TurnState.Keys == nil || r.TurnState.Get == nil {
st.Notes = append(st.Notes, "the gate's merge turns are not readable here, so no check is named as held by one")
return nil
}
keys, err := r.TurnState.Keys()
if err != nil {
st.Notes = append(st.Notes, "the gate's merge turns cannot be listed, so no check is named as held by one: "+err.Error())
return nil
}
t := &Turns{}
for _, k := range keys {
if raw, err := r.TurnState.Get(k); err == nil && len(raw) > 0 {
t.Put(k, raw, false)
}
}
return t
}
// heldBy says the turn that holds a candidate's check off, or "".
func heldBy(t *Turns, c delivery.Candidate, heads map[string]string) string {
cs := CaseOf(c, heads)
if t == nil || cs == nil {
return ""
}
repos := make([]string, 0, len(cs.Trunk))
for r := range cs.Trunk {
repos = append(repos, r)
}
if repo, w, ok := t.Holding(repos, cs.Heads); ok {
return repo + ": the approved merge of " + w.Group + " waits on it"
}
return ""
Notes []string `json:"notes,omitempty"`
}
// Status reads the build modules' steps and the candidates against their verdicts.
@@ -452,8 +383,6 @@ func (r Reader) Status() (Status, error) {
}
st.Unjudged = []string{}
st.Waiting = map[string][]string{KindNew: {}, KindNodes: {}, KindTrunk: {}}
st.HeldByTurn = map[string]string{}
turns := r.turns(&st)
var heads map[string]string
if r.Trunks == nil {
st.Notes = append(st.Notes, "the trunk heads are not readable here, so no check is named as waiting for a trunk move")
@@ -506,9 +435,6 @@ func (r Reader) Status() (Status, error) {
if len(raw) == 0 {
st.Unjudged = append(st.Unjudged, c.Group)
st.Waiting[KindNew] = append(st.Waiting[KindNew], c.Group)
if why := heldBy(turns, c, heads); why != "" {
st.HeldByTurn[c.Group] = why
}
continue
}
v, err := delivery.DecodeVerdict(raw)
@@ -517,16 +443,12 @@ func (r Reader) Status() (Status, error) {
case IsNodesFailure(v):
st.Waiting[KindNodes] = append(st.Waiting[KindNodes], c.Group)
case heads != nil:
if cs := CaseOf(c, heads); cs != nil && (!sameTrunk(v.Trunk, cs.Trunk) || besideMoved(v.Beside, besideAt(heads, r.Runs))) {
if cs := CaseOf(c, heads); cs != nil && !sameTrunk(v.Trunk, cs.Trunk) {
st.Waiting[KindTrunk] = append(st.Waiting[KindTrunk], c.Group)
if why := heldBy(turns, c, heads); why != "" {
st.HeldByTurn[c.Group] = why
}
}
}
}
}
st.DeadLetters = r.deadLetters()
st.Notes = append(st.Notes, "nothing holds the checker and it holds no place: running_steps are the build modules' keys that can still be running (started within the step bound, their build module running on the node); a candidate without a verdict has its check ask in a build seat's stream, waiting for a node with a free slot, whose backlog the delivery view shows (ADR 0525 §4, §9)")
return st, nil
}
@@ -144,10 +144,7 @@ func TestTheStatusToolReadsRunningSteps(t *testing.T) {
CandidateState: fakeState{ckey: cand}.read(),
VerdictState: fakeState{}.read(),
HaveState: fakeState{"shanks.mesh-build-go": haveOf(t, delivery.PhaseRunning)}.read(),
HearingState: fakeState{"dead-letters": []byte(`{"heard":false,"streams":["SEAT_NODE_BUILD_GO"],` +
`"why":"DEAD_LETTERS is not heard, so an ask a build seat gives up on is waited for until its bound: SEAT_NODE_BUILD_GO: permissions violation",` +
`"at":"2026-10-26T13:00:00Z"}`)}.read(),
Now: func() time.Time { return time.Date(2026, 10, 22, 15, 10, 0, 0, time.UTC) },
Now: func() time.Time { return time.Date(2026, 10, 22, 15, 10, 0, 0, time.UTC) },
}
st, err := rd.Status()
if err != nil {
@@ -159,13 +156,9 @@ func TestTheStatusToolReadsRunningSteps(t *testing.T) {
if len(st.Unjudged) != 1 || st.Unjudged[0] != "feat/x" {
t.Errorf("unjudged = %v", st.Unjudged)
}
// Issue 734, review round 1: a checker that does not hear DEAD_LETTERS says so in its status, with why.
if st.DeadLetters.Heard || !strings.Contains(st.DeadLetters.Why, "permissions violation") {
t.Errorf("dead letters = %+v, want not heard, saying why", st.DeadLetters)
}
// What cannot be read is said, not guessed.
st, err = Reader{}.Status()
if err != nil || len(st.Notes) == 0 || st.DeadLetters.Heard {
if err != nil || len(st.Notes) == 0 {
t.Errorf("empty status = %+v, %v", st, err)
}
}
@@ -241,18 +234,6 @@ func TestTheStatusCountsAnHQOnlyGroupAsUnjudged(t *testing.T) {
if len(st.Unjudged) != 1 || st.Unjudged[0] != "pb-x" {
t.Errorf("unjudged = %v, want pb-x", st.Unjudged)
}
// ADR 0548 §3: a candidate whose check an approved merge's turn holds off is named with the turn.
held := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
hkey, _ := delivery.CandidatesKey("feat/x")
st, err = Reader{CandidateState: fakeState{hkey: held}.read(), VerdictState: fakeState{}.read(),
Trunks: func() (map[string]string, error) { return map[string]string{repoA: trunkA}, nil },
TurnState: fakeState{turnKey(t, repoA): turnOf(t, repoA, "fix/other", "0123456789ab")}.read()}.Status()
if err != nil {
t.Fatal(err)
}
if why := st.HeldByTurn["feat/x"]; !strings.Contains(why, repoA) || !strings.Contains(why, "fix/other") {
t.Errorf("held_by_turn = %v, want feat/x held by the merge of fix/other on %s", st.HeldByTurn, repoA)
}
}
// slowBucket wraps a state with a Get that takes delay and counts the values read, the lists and the most Gets in flight.
@@ -3,9 +3,6 @@ package check
import (
"encoding/json"
"strings"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The shapes of what the build modules' steps answer, as their code writes them (mesh-catalog modules/mesh-build-go
@@ -45,17 +42,6 @@ type Fetched struct {
Conflicts []string `json:"conflicts,omitempty"`
Changed []string `json:"changed,omitempty"`
Trees map[string]MergeTree `json:"trees,omitempty"`
// Programs are the changed files whose first bytes are an executable binary, read by the build module in its workspace
// (issue 1577); nil for a build module that does not report it, a pointer to an empty list for one that found none.
Programs *[]ProgramFile `json:"programs,omitempty"`
}
// ProgramFile is a changed file that is a compiled program, and its format (ELF, Mach-O or PE).
type ProgramFile struct {
Path string `json:"path"`
Kind string `json:"kind"`
// Commit is set for a program a commit of the branch added and a later one removed: it stays in history once merged.
Commit string `json:"commit,omitempty"`
}
// GoPackage is one package of a Go test result.
@@ -66,13 +52,16 @@ type GoPackage struct {
Elapsed float64 `json:"elapsed"`
Tests int `json:"tests"`
NoTests bool `json:"no_tests,omitempty"`
// Skipped is how many of the package's tests skipped; an older build module names none.
Skipped int `json:"skipped,omitempty"`
}
// Failure is a test that failed, or a package that failed with no test to name: the build modules' failure contract
// (buildstep.Failure, issue 1901), its cause never empty and never a passing line from a build module that keeps it.
type Failure = buildstep.Failure
// Failure is a test that failed, or a package that failed with no test to name.
type Failure struct {
Package string `json:"package"`
Test string `json:"test,omitempty"`
Output string `json:"output,omitempty"`
// Cause is the first line of what the test reported as its error, when the build module told it apart (issue 1901).
Cause string `json:"cause,omitempty"`
}
// TestTime is one test and the seconds it took.
type TestTime struct {
@@ -105,9 +94,6 @@ type Tested struct {
// JudgedBy is the program that ran the step (ADR 0535 §5): "installed", or "candidate <merge commit>" under runner
// workspace; absent from a build module that does not take the runner.
JudgedBy string `json:"judged_by,omitempty"`
// By is the build module that answered and the SDK it was built with (buildstep.By); absent from a build module from
// before the failure contract.
By string `json:"by,omitempty"`
}
// Digested is the answer of the Go build module's digest step: one digest per package that has tests at the level, by import
@@ -121,6 +107,4 @@ type Digested struct {
WithoutTests []string `json:"without_tests,omitempty"`
// JudgedBy is the program that ran the digest, as Tested.JudgedBy says.
JudgedBy string `json:"judged_by,omitempty"`
// Needs are the packages whose tests need something of the node at this level (issue 890).
Needs []delivery.PackageNeeds `json:"needs,omitempty"`
}
@@ -10,7 +10,7 @@ TestFetchWithABaseMergesAndAnswersThePreview and TestFetchWithAConflictEndsPasse
into a throwaway test that wrote them out, at mesh-catalog feat/r4c5-checker d5804927 (the git commands are the test's fake,
so the shas are its constants; the shape and the field names are the module's own marshalling). node-fetch-clean.json is the
FetchResult json.Marshal of mesh-build-node's fetch (fetch.go, preview.go) in its test TestFetchWithABaseMergesAndAnswersThePreview
(cmd/mesh-build-node/preview_test.go) at the same commit. Captured on 2026-10-21 for issue 1679. Both clean fetches carry `"programs":[]`, which a merging fetch of the build modules answers since issue 1577 (mesh-sdk buildstep.MergeInto's Programs), added by hand to the captured bytes: a fixture without it is an older build module's, and its line warns that no compiled program is refused.
(cmd/mesh-build-node/preview_test.go) at the same commit. Captured on 2026-10-21 for issue 1679.
forge-get-file-go-mod.json is the answer of the forge's gitea_get_file (mesh-catalog modules/gitea, tools/index.ts) for modules/mesh-delivery-builder/go.mod of
novox/mesh-delivery at main, and forge-compare-trunk.json the answer of gitea_compare for novox/mesh-sdk base main, head cb54756c3865 (a commit on the
@@ -1 +1 @@
{"workspace":"mvii1iwu5414e4","commit":"0123456789abcdef0123456789abcdef01234567","modules":["."],"base":"abcdef0123456789abcdef0123456789abcdef01","merge":"fedcba9876543210fedcba9876543210fedcba98","changed":["a.go","modules/m/main.go"],"programs":[],"trees":{"modules/m":{"tree":"1111111111111111111111111111111111111111","manifest":"ca3d163bab055381827226140568f3bef7eaac187cebd76878e0b63e9e442356","locks":{"go.sum":"c5fc83c01e92404452b986527d239140ccf9a48b88e0c268fbf38c2e1429e9c9"}}}}
{"workspace":"mvii1iwu5414e4","commit":"0123456789abcdef0123456789abcdef01234567","modules":["."],"base":"abcdef0123456789abcdef0123456789abcdef01","merge":"fedcba9876543210fedcba9876543210fedcba98","changed":["a.go","modules/m/main.go"],"trees":{"modules/m":{"tree":"1111111111111111111111111111111111111111","manifest":"ca3d163bab055381827226140568f3bef7eaac187cebd76878e0b63e9e442356","locks":{"go.sum":"c5fc83c01e92404452b986527d239140ccf9a48b88e0c268fbf38c2e1429e9c9"}}}}
@@ -1 +1 @@
{"workspace":"mvii1sxt979f67","commit":"0123456789abcdef0123456789abcdef01234567","modules":["modules/m"],"installed":[],"refused":[],"base":"abcdef0123456789abcdef0123456789abcdef01","merge":"0123456789abcdef0123456789abcdef01234567","changed":["modules/m/package.json"],"programs":[],"trees":{"modules/m":{"tree":"0123456789abcdef0123456789abcdef01234567","manifest":"0581b699d9d05517e69b21f5cedbeb2eee8aba2b205280b561be731594d7feef"}}}
{"workspace":"mvii1sxt979f67","commit":"0123456789abcdef0123456789abcdef01234567","modules":["modules/m"],"installed":[],"refused":[],"base":"abcdef0123456789abcdef0123456789abcdef01","merge":"0123456789abcdef0123456789abcdef01234567","changed":["modules/m/package.json"],"trees":{"modules/m":{"tree":"0123456789abcdef0123456789abcdef01234567","manifest":"0581b699d9d05517e69b21f5cedbeb2eee8aba2b205280b561be731594d7feef"}}}
@@ -6,7 +6,6 @@ import (
"errors"
"fmt"
"regexp"
"slices"
"sort"
"strconv"
"strings"
@@ -14,7 +13,6 @@ import (
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// A try is a pushed head judged by the checker as a pull request's head is, before its pull request is opened (ADR 0501
@@ -26,17 +24,11 @@ import (
const (
TryTool = "checker_try"
TryReadTool = "checker_try_read"
TryEndTool = "checker_try_end"
// TriesState is the checker's state of tries: the bucket mesh-delivery-checker_tries, keys try-<unix nanoseconds>.
TriesState = "tries"
TriesBucket = delivery.CheckerModule + "_" + TriesState
// MaxTries and MaxTriesWaiting are checked when a try is asked, mesh-wide: a new try is refused while MaxTries tries run
// on a build node or MaxTriesWaiting wait for one (queued, or their asks not yet taken by a node). It is no bound on
// what runs later: tries let in while waiting may later all run at once, and each node's build capacity bounds what
// runs on it. A try waiting behind the seat's other asks holds no node, so it no longer blocks the next try (issue
// 1890: at 19:53 on 2026-10-26 four tries sat queued behind 44 checks for up to 47 minutes and no fifth could be asked).
MaxTries = 4
MaxTriesWaiting = 8
// MaxTries is how many tries may wait or run at once, mesh-wide: each holds a build node for minutes.
MaxTries = 4
// TryScan is how many of the newest tries a new ask reads to find the ones that wait or run (issue 1827).
TryScan = 64
// TryKeep is how long an ended try's key is kept before the checker removes it.
@@ -83,104 +75,9 @@ type Try struct {
Failed []string `json:"failed,omitempty"`
Levels map[string]delivery.LevelResult `json:"levels,omitempty"`
Machines []string `json:"machines,omitempty"`
// Asks are the ids of the asks the try put on the build seats' streams, each ending on the step key <node>.<id> of its
// build module: the delivery view counts those steps as a try's, not a candidate's check (build_usage).
Asks []string `json:"asks,omitempty"`
// Steps say where each ask stands while the try runs and after: asked until a node takes it, then the node, its
// state, the phase its step says and when (issue 1890: a 31-minute try could not be told queued from running).
Steps []TryStep `json:"steps,omitempty"`
Asked time.Time `json:"asked"`
Started time.Time `json:"started,omitzero"`
Ended time.Time `json:"ended,omitzero"`
}
// TryAsked is a TryStep's state before a build node took its ask: it waits in the seat's stream.
const TryAsked = "asked"
// TryStep is one ask of a try as its step key last said it.
type TryStep struct {
Ask string `json:"ask"`
Node string `json:"node,omitempty"`
State string `json:"state"`
Phase string `json:"phase,omitempty"`
Started time.Time `json:"started,omitzero"`
PhaseAt time.Time `json:"phase-at,omitzero"`
Ended time.Time `json:"ended,omitzero"`
}
// tryWriter writes a running try's record again as its asks are put and their steps move: one writer per try, holding
// the revision it wrote last, so the try's end is written at that revision. A write refused (the try ended by hand)
// stops the writes; the end's own write then says it.
type tryWriter struct {
mu sync.Mutex
tr *Tryer
ctx context.Context
key string
t Try
rev uint64
stopped bool
}
// heard takes an ask put (step nil) or a value of its step key in, and writes the record when it changed.
func (w *tryWriter) heard(askID string, step *delivery.Step) {
w.mu.Lock()
defer w.mu.Unlock()
if w.stopped {
return
}
changed := false
if !slices.Contains(w.t.Asks, askID) {
w.t.Asks, changed = append(w.t.Asks, askID), true
}
i := slices.IndexFunc(w.t.Steps, func(s TryStep) bool { return s.Ask == askID })
if i < 0 {
w.t.Steps, changed = append(w.t.Steps, TryStep{Ask: askID, State: TryAsked}), true
i = len(w.t.Steps) - 1
}
if step != nil {
now := TryStep{Ask: askID, Node: step.Node, State: step.State, Phase: step.Phase, Started: step.Started,
PhaseAt: step.PhaseAt, Ended: step.Ended}
changed = changed || now != w.t.Steps[i]
w.t.Steps[i] = now
}
if changed {
w.write()
}
}
// write writes the record at the revision written last; a refusal stops the writes.
func (w *tryWriter) write() {
data, err := encodeTry(w.t)
if err != nil {
return
}
rev, err := w.tr.Tries.Update(w.ctx, w.key, data, w.rev)
if err != nil {
w.stopped = true
w.tr.logf("%s: its asks' progress is no longer written: %v", w.key, err)
return
}
w.rev = rev
}
// end stops the writes and answers the record and its revision, for the try's end.
func (w *tryWriter) end() (Try, uint64) {
w.mu.Lock()
defer w.mu.Unlock()
w.stopped = true
return w.t, w.rev
}
// HoldsANode says whether the try holds a build node now: it runs, and a step of its asks runs on a node, or it names
// no step (a checker that writes its asks only at the end, or one about to put its first).
func (t Try) HoldsANode() bool {
if t.State != TryRunning {
return false
}
if len(t.Steps) == 0 {
return true
}
return slices.ContainsFunc(t.Steps, func(s TryStep) bool { return s.State == delivery.StepRunning })
Asked time.Time `json:"asked"`
Started time.Time `json:"started,omitzero"`
Ended time.Time `json:"ended,omitzero"`
}
// Over says whether the try came to its end.
@@ -222,8 +119,8 @@ func NotChecked(repository string) error {
}
// AskTry writes a new try, queued, and answers its id: a try of the same head that waits or runs is the answer to a second
// ask, and a new try is refused while MaxTries tries run on a build node or MaxTriesWaiting wait for one. trunks are the
// gate's trunk heads of every repository (source.main): a repository with none is no repository the checker checks. A trunk asked by name is its trunk head now.
// ask, and at most MaxTries wait or run at once. trunks are the gate's trunk heads of every repository (source.main): a
// repository with none is no repository the checker checks. A trunk asked by name is its trunk head now.
func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (string, error) {
repo := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(a.Repository), ".git"))
head := strings.ToLower(strings.TrimSpace(a.Head))
@@ -286,7 +183,7 @@ func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (
}); err != nil {
return "", err
}
var holding, waiting []string
var open []string
for i, cur := range tries {
if !live[i] {
continue
@@ -294,18 +191,10 @@ func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (
if cur.Repository == t.Repository && cur.Head == t.Head && cur.Trunk == t.Trunk && cur.Tests == t.Tests && cur.Level == t.Level {
return cur.ID, nil
}
said := fmt.Sprintf("%s (%s@%.12s, %s)", cur.ID, cur.Repository, cur.Head, cur.State)
if cur.HoldsANode() {
holding = append(holding, said)
} else {
waiting = append(waiting, said)
}
open = append(open, fmt.Sprintf("%s (%s@%.12s, %s)", cur.ID, cur.Repository, cur.Head, cur.State))
}
// Checked at this ask only: a try queued behind the seat's checks holds no node; tries let in while waiting may later
// all run at once, and each node's build capacity bounds what runs on it.
if len(holding) >= MaxTries || len(waiting) >= MaxTriesWaiting {
return "", fmt.Errorf("a new try is refused while %d tries run on a build node or %d wait for one; now %d run and %d wait: running %s; waiting %s; ask again when one ends",
MaxTries, MaxTriesWaiting, len(holding), len(waiting), orNone(holding), orNone(waiting))
if len(open) >= MaxTries {
return "", fmt.Errorf("%d tries wait or run, the most at once: %s; ask again when one ends", len(open), strings.Join(open, ", "))
}
t.ID = "try-" + strconv.FormatInt(now.UnixNano(), 10)
data, err := encodeTry(t)
@@ -333,52 +222,6 @@ func ReadTry(store TryStore, id string) (json.RawMessage, error) {
return raw, nil
}
// TryEnder is the tries state as the end tool sees it: one key with its revision, and a write at that revision.
type TryEnder interface {
Entry(key string) (json.RawMessage, uint64, error)
Update(key string, value []byte, revision uint64) error
}
// EndTry ends a try that waits or runs as error, saying why and that it was ended by hand (issue 734): its place is
// free at once, and the next checker_try of its head asks anew. A try that ended already is refused, naming its end.
func EndTry(store TryEnder, id, why string, now time.Time) (Try, error) {
if !tryKeyOK.MatchString(id) {
return Try{}, fmt.Errorf("%q is not a try's id, try-<digits>", id)
}
why = strings.TrimSpace(why)
if why == "" {
return Try{}, errors.New("say why the try is ended: the why is kept in what it says")
}
raw, rev, err := store.Entry(id)
if err != nil {
return Try{}, err
}
if len(raw) == 0 {
return Try{}, fmt.Errorf("no try %s: it was never asked, or it ended more than %s ago and was removed", id, TryKeep)
}
t, err := DecodeTry(raw)
if err != nil {
return Try{}, err
}
if t.Over() {
return Try{}, fmt.Errorf("%s ended already, %s at %s", id, t.State, t.Ended.Format(time.RFC3339))
}
t.State, t.Ended = TryError, now.UTC()
if !t.Started.IsZero() {
t.Took = t.Ended.Sub(t.Started).Round(time.Second).String()
}
t.Said = strings.TrimSpace("ended by hand (" + TryEndTool + "): " + why + " | " + t.Said)
t.Said = strings.TrimSuffix(t.Said, " |")
data, err := encodeTry(t)
if err != nil {
return Try{}, err
}
if err := store.Update(id, data, rev); err != nil {
return Try{}, fmt.Errorf("%s changed while it was ended, read it and end it again: %w", id, err)
}
return t, nil
}
// TryOptions narrow what a try runs; a pull request's check has none.
type TryOptions struct {
Level string
@@ -391,14 +234,12 @@ type Tryer struct {
Tries KV
Judge Judge
Bound time.Duration
// NotTaken ends a try whose ask no build node took within it as error (issue 734); none when 0.
NotTaken time.Duration
Now func() time.Time
Logf func(format string, args ...any)
Now func() time.Time
Logf func(format string, args ...any)
mu sync.Mutex
wg sync.WaitGroup
inflight map[string]context.CancelFunc
inflight map[string]bool
seen map[string]bool // looked at since this process started: a try left running is taken over the first time
}
@@ -426,31 +267,25 @@ func (tr *Tryer) Put(ctx context.Context, key string, value []byte) {
tr.logf("tries.%s holds what is not a try, left: %v", key, err)
return
}
if t.Over() {
return
}
tr.mu.Lock()
if tr.inflight == nil {
tr.inflight, tr.seen = map[string]context.CancelFunc{}, map[string]bool{}
tr.inflight, tr.seen = map[string]bool{}, map[string]bool{}
}
if t.Over() {
// Ended while it ran (checker_try_end): its judging stops, and its asks are no longer waited for.
if stop := tr.inflight[key]; stop != nil {
stop()
}
if tr.inflight[key] || (t.State == TryRunning && tr.seen[key]) {
tr.mu.Unlock()
return
}
if tr.inflight[key] != nil || (t.State == TryRunning && tr.seen[key]) {
tr.mu.Unlock()
return
}
run, stop := context.WithCancel(ctx)
tr.seen[key] = true
tr.inflight[key] = stop
tr.inflight[key] = true
tr.mu.Unlock()
tr.wg.Add(1)
go func() {
defer tr.wg.Done()
defer func() { tr.mu.Lock(); delete(tr.inflight, key); tr.mu.Unlock(); stop() }()
tr.run(run, key)
defer func() { tr.mu.Lock(); delete(tr.inflight, key); tr.mu.Unlock() }()
tr.run(ctx, key)
}()
}
@@ -501,37 +336,27 @@ func (tr *Tryer) run(ctx context.Context, key string) {
}
return
}
progress := &tryWriter{tr: tr, ctx: ctx, key: key, t: t, rev: rev}
cs := &Case{Key: key, Group: key, Watch: progress.heard,
cs := &Case{Key: key, Group: key,
Pulls: []Pull{{Repository: t.Repository, Head: t.Head, Trunk: t.Trunk}},
Trunk: map[string]string{t.Repository: t.Trunk},
Queued: t.Asked,
Progress: &Progress{},
Try: &TryOptions{Level: t.Level, Run: t.Tests}}
judging := ctx
if tr.NotTaken > 0 {
judging = buildrun.NotTakenBound(ctx, tr.NotTaken)
}
out, judged := judgeWithin(judging, tr.Judge, cs, tr.Bound)
out, judged := judgeWithin(ctx, tr.Judge, cs, tr.Bound)
if ctx.Err() != nil {
return // left running: the checker takes it over when it starts; or ended by hand, which wrote its end
return // left running: the checker takes it over when it starts
}
t, rev = progress.end()
v := out.Verdict
t.Ended = tr.now().UTC()
t.Took = t.Ended.Sub(t.Started).Round(time.Second).String()
t.Levels, t.Machines, t.Asks = v.Levels, v.Machines, cs.Asks()
t.Levels, t.Machines = v.Levels, v.Machines
t.Said = Report(v, cs, cs.Pulls[0], out.Builds, out.Slowest, out.Judged)
for _, n := range cs.Notes() {
t.Said += " | " + n
}
switch {
case !judged || IsNodesFailure(v) || IsLostAsk(v) || IsNotTaken(v):
// Not the change's: its ask was given up or never taken (issue 734), or a part of the mesh was not heard.
case !judged || IsNodesFailure(v):
t.State = TryError
if IsLostAsk(v) || IsNotTaken(v) {
t.Failed = []string{v.Failed.Words}
}
case v.State == delivery.VerdictPass:
t.State = TryPass
default:
@@ -578,11 +403,3 @@ func judgeWithin(ctx context.Context, j Judge, cs *Case, bound time.Duration) (O
}
return out, true
}
// orNone joins the tries said, or says none.
func orNone(said []string) string {
if len(said) == 0 {
return "none"
}
return strings.Join(said, ", ")
}
@@ -3,7 +3,6 @@ package check
import (
"context"
"encoding/json"
"fmt"
"strings"
"testing"
"time"
@@ -24,19 +23,6 @@ func (m memTries) Create(key string, value []byte) error {
return err
}
// writtenTries is a memKV that keeps every running try it was given to write, in order.
type writtenTries struct {
*memKV
running []Try
}
func (w *writtenTries) Update(ctx context.Context, key string, value []byte, rev uint64) (uint64, error) {
if t, err := DecodeTry(value); err == nil && !t.Over() {
w.running = append(w.running, t)
}
return w.memKV.Update(ctx, key, value, rev)
}
var tryTrunks = map[string]string{repoA: trunkA, repoSDK: trunkS, hqRepo: trunkA}
func readTry(t testing.TB, kv *memKV, id string) Try {
@@ -68,39 +54,15 @@ func TestATryIsAskedQueuedOncePerHeadAndAtMostFourAtOnce(t *testing.T) { // ADR
if err != nil || again != id {
t.Errorf("a second ask of the same head answered %q, %v, want %q", again, err, id)
}
// Issue 1890: tries on a build node (a step of theirs running, or none named yet) and tries waiting for one (queued, or
// their asks not taken) are bounded apart, MaxTries and MaxTriesWaiting; the refusal names both counts.
onNode := func(i int) Try {
return Try{ID: fmt.Sprintf("try-%d", i), Repository: repoA, Head: fmt.Sprintf("%040x", i), Trunk: trunkA, State: TryRunning,
Steps: []TryStep{{Ask: "a", State: delivery.StepRunning}}}
}
waitingFor := func(i int) Try {
return Try{ID: fmt.Sprintf("try-%d", i), Repository: repoA, Head: fmt.Sprintf("%040x", i), Trunk: trunkA, State: TryRunning,
Steps: []TryStep{{Ask: "a", State: TryAsked}}}
}
for _, c := range []struct {
what string
holding, wait int
refused string
}{
{"four queued behind the checks, none on a node: a fifth is taken", 0, 4, ""},
{"four on a node", MaxTries, 0, "now 4 run and 0 wait"},
{"three on a node, eight waiting", 3, MaxTriesWaiting, "a new try is refused while 4 tries run on a build node or 8 wait for one; now 3 run and 8 wait"},
} {
kv := newMemKV()
for i := range c.holding + c.wait {
tr := waitingFor(i + 1)
if i < c.holding {
tr = onNode(i + 1)
}
data, _ := encodeTry(tr)
_, _ = kv.Create(context.Background(), tr.ID, data)
}
_, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoSDK, Head: head1}, at.Add(time.Minute))
if (c.refused == "") != (err == nil) || (err != nil && !strings.Contains(err.Error(), c.refused)) {
t.Errorf("%s: answered %v, want refused %q", c.what, err, c.refused)
heads := []string{head2, trunkA2, trunkS2}
for i, h := range heads {
if _, err := AskTry(store, tryTrunks, TryAsk{Repository: repoA, Head: h, Trunk: trunkA}, at.Add(time.Duration(i+2)*time.Second)); err != nil {
t.Fatalf("try %d: %v", i+2, err)
}
}
if _, err := AskTry(store, tryTrunks, TryAsk{Repository: repoSDK, Head: head1}, at.Add(time.Minute)); err == nil || !strings.Contains(err.Error(), "4 tries wait or run") {
t.Errorf("a fifth try answered %v, want refused naming the four", err)
}
}
func TestATryOfARepositoryWithNoCheckedLevelIsRefused(t *testing.T) { // the agreed error of the try
@@ -127,8 +89,7 @@ func TestATryIsJudgedAsAPullRequestsHeadAndOwesTheForgeNothing(t *testing.T) { /
t.Fatal(err)
}
b := newBench(t)
written := &writtenTries{memKV: kv}
tr := &Tryer{Tries: written, Judge: b.ex}
tr := &Tryer{Tries: kv, Judge: b.ex}
raw, _, _, _ := kv.Get(ctx, id)
tr.Put(ctx, id, raw)
tr.Wait()
@@ -136,21 +97,6 @@ func TestATryIsJudgedAsAPullRequestsHeadAndOwesTheForgeNothing(t *testing.T) { /
if got.State != TryPass || got.Took == "" || got.Head != head1 || got.Trunk != trunkA || got.Said == "" || got.Levels[delivery.LevelUnit].Run != 2 {
t.Fatalf("try = %+v", got)
}
// Issue 1890: while it runs, the try names its ask as soon as it is put, and then where its step stands; the end
// keeps each ask's node and state.
if len(written.running) < 3 || len(written.running[1].Asks) != 1 || written.running[1].Steps[0].State != TryAsked ||
written.running[2].Steps[0].Node != "shanks" {
t.Errorf("the running try's writes = %+v, want its ask named while it ran, then its node", written.running)
}
if len(got.Steps) != 1 || got.Steps[0].Node != "shanks" || got.Steps[0].State != delivery.StepPassed {
t.Errorf("the ended try's steps = %+v", got.Steps)
}
// The try names the asks it put, so the delivery view counts their steps as a try's (build_usage).
if len(got.Asks) != 1 {
t.Errorf("asks = %v, want the one check ask the try put", got.Asks)
} else if _, err := delivery.AskStepKey("shanks", got.Asks[0]); err != nil {
t.Errorf("the try names %q, which is no ask id: %v", got.Asks[0], err)
}
fetch := b.steps.tool("fetch")
if len(fetch) != 1 || fetch[0].args["commit"] != head1 || fetch[0].args["base"] != trunkA || fetch[0].args["ref"] != "" {
t.Errorf("fetch = %+v, want the head merged onto the trunk, with no pull request's ref", fetch)
@@ -1,78 +0,0 @@
package check
import (
"sort"
"strings"
"sync"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// Turns holds the gate's merge turns (novox/hq ADR 0548 §3), by repository, as a watch of the gate's turns bucket puts them
// in: the checker claims no check of a repository a turn names, except the check of a waiting delivery's own heads, so an
// approved merge goes before a check not yet started. A bucket not there is read as empty.
type Turns struct {
mu sync.Mutex
byRepo map[string]delivery.Turn
}
// Put takes a value of the turns bucket in, or its delete, and answers whether what the checker holds changed. A value that
// is not a turn is dropped as a delete would be: a turn the checker cannot read holds nothing.
func (t *Turns) Put(key string, value []byte, deleted bool) bool {
repository, err := delivery.ParseTurnsKey(key)
if err != nil {
return false
}
t.mu.Lock()
defer t.mu.Unlock()
if t.byRepo == nil {
t.byRepo = map[string]delivery.Turn{}
}
cur, had := t.byRepo[repository]
if !deleted {
if turn, err := delivery.DecodeTurn(value); err == nil {
t.byRepo[repository] = turn
return !had || !delivery.SameTurn(cur, turn)
}
}
delete(t.byRepo, repository)
return had
}
// Holding names the first of the repositories (by name) a turn holds off the check of the heads, and the delivery that
// waits first in it; false when none does.
func (t *Turns) Holding(repositories []string, heads string) (string, delivery.TurnWait, bool) {
if t == nil {
return "", delivery.TurnWait{}, false
}
sorted := make([]string, 0, len(repositories))
for _, r := range repositories {
sorted = append(sorted, strings.ToLower(r))
}
sort.Strings(sorted)
t.mu.Lock()
defer t.mu.Unlock()
for _, r := range sorted {
turn, ok := t.byRepo[r]
if !ok || !delivery.TurnHolds(turn, heads) {
continue
}
for _, w := range turn.Waiting {
if w.Heads != heads {
return r, w, true
}
}
}
return "", delivery.TurnWait{}, false
}
// All is every turn held now, by repository (a copy).
func (t *Turns) All() map[string]delivery.Turn {
t.mu.Lock()
defer t.mu.Unlock()
out := make(map[string]delivery.Turn, len(t.byRepo))
for r, turn := range t.byRepo {
out[r] = turn
}
return out
}
@@ -93,11 +93,11 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
t.Errorf("module = %q", m.Module)
}
// delivery.CheckerKeptHistory, CheckerOwedHistory, PreviewsHistory, CheckerVerdictsHistory; the names follow the SDK's buckets.
if want := []state{{Name: "hearing", History: 1}, {Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "tries", History: 1}, {Name: "verdicts", History: 5}}; !reflect.DeepEqual(m.State, want) {
if want := []state{{Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "tries", History: 1}, {Name: "verdicts", History: 5}}; !reflect.DeepEqual(m.State, want) {
t.Errorf("state = %+v, want %+v", m.State, want)
}
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-controller.assignments",
"mesh-delivery-assigner.compositions", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-gate.turns", "mesh-delivery-indexer.candidates",
"mesh-delivery-assigner.compositions", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates",
"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) {
t.Errorf("reads = %v, want %v", m.Reads, want)
}
@@ -124,8 +124,8 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
t.Errorf("streams = %+v, want %+v", m.Streams, want)
}
// The read tools (issue 1686) are served by the tools bundle, not by the checker's process.
// The try's tools (ADR 0501 §5; the end by hand, issue 734) beside them: mesh-controller's try calls them.
if want := []string{"checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
// The try's two tools (ADR 0501 §5) beside them: mesh-controller's try calls them.
if want := []string{"checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_try", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} {
@@ -10,7 +10,6 @@
{ "name": "checks", "events": ["preview.**", "owed.**"] }
],
"state": [
{ "name": "hearing", "history": 1 },
{ "name": "kept", "history": 1 },
{ "name": "owed", "history": 5 },
{ "name": "previews", "history": 5 },
@@ -26,7 +25,6 @@
"mesh-delivery-assigner.compositions",
"mesh-delivery-builder.builds",
"mesh-delivery-gate.source",
"mesh-delivery-gate.turns",
"mesh-delivery-indexer.candidates",
"mesh-delivery-indexer.manifests",
"mesh-delivery-indexer.modules",
@@ -43,7 +41,6 @@
"checker_pull",
"checker_status",
"checker_try",
"checker_try_end",
"checker_try_read",
"checker_verdicts"
],
@@ -91,28 +91,6 @@ func treesVia(ctx context.Context, call caller) bridge.Trees {
}
}
// listedVia asks the forge for the files directly under one directory of a commit through call: gitea_commit_trees with
// listed (issue 1126), answer {listed: {directory: {name: blob id}}}.
func listedVia(ctx context.Context, call caller) bridge.Listed {
return func(owner, repo, sha, dir string) (map[string]string, error) {
raw, err := call(ctx, commitTreesTool, map[string]any{"owner": owner, "repo": repo, "sha": sha, "listed": []string{dir}})
if err != nil {
return nil, err
}
var a struct {
Listed map[string]map[string]string `json:"listed"`
}
if err := json.Unmarshal(raw, &a); err != nil {
return nil, fmt.Errorf("the answer of %s is not {listed}: %w", commitTreesTool, err)
}
if a.Listed == nil {
// A forge module from before listed answers none: placement cannot be read, never read as empty.
return nil, fmt.Errorf("the answer of %s lists no directory; the forge module does not list yet", commitTreesTool)
}
return a.Listed[dir], nil
}
}
// getFileTool is the forge's tool for one file at a ref (mesh-catalog modules/gitea tools/index.ts gitea_get_file):
// arguments owner, repo, path and ref, answer {file: {path, ref, sha, size, content}}, the content decoded to text
// (client.ts getFile). A path the forge does not have fails with `Gitea API <path>: 404 <body>` (client.ts request).
@@ -21,7 +21,6 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
)
// busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream,
@@ -272,7 +271,7 @@ func startBed(t *testing.T, openReply string, prepare func(ctx context.Context,
}
broker := filepath.Join(t.TempDir(), "broker")
cred, _ := json.Marshal(process.Account{URL: srv.ClientURL(), Node: "n1", Module: "mesh-delivery-forge-bridge", User: "n1.mesh-delivery-forge-bridge", Password: "pw"})
cred, _ := json.Marshal(credential{URL: srv.ClientURL(), Node: "n1", Module: "mesh-delivery-forge-bridge", User: "n1.mesh-delivery-forge-bridge", Password: "pw"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
@@ -6,12 +6,18 @@ package main
import (
"context"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"os"
"os/signal"
"strings"
"sync"
"syscall"
"time"
@@ -33,7 +39,7 @@ const eventsStream = "EVENTS"
// The bridge's own stream FORGE takes `mesh.stream.<module>.event.<event>` (mesh-control internal/broker
// modstreams.go ModuleStreamSubject).
func forgeSubject(event string) string {
return process.ModuleStreamSubject(delivery.ForgeBridgeModule, event)
return "mesh.stream." + delivery.ForgeBridgeModule + ".event." + event
}
func main() {
@@ -45,6 +51,50 @@ func main() {
}
}
// credential is the broker file the controller issues to the module: JSON of the bus address, the certificate
// fingerprint to pin, the principal and its password (as the watchman reads it; issue 1242 moves this into the SDK).
type credential struct {
URL string `json:"url"`
Fingerprint string `json:"fingerprint"`
Node string `json:"node"`
Module string `json:"module"`
User string `json:"user"`
Password string `json:"password"`
}
func readCredential(path string) (credential, error) {
raw, err := os.ReadFile(path)
if err != nil {
return credential{}, err
}
var c credential
if err := json.Unmarshal(raw, &c); err != nil {
return credential{}, fmt.Errorf("the broker credential is not one: %w", err)
}
if c.URL == "" || c.User == "" {
return credential{}, errors.New("the broker credential names no bus address or user")
}
return c, nil
}
// pinned accepts exactly the certificate with this SHA-256 and no other.
func pinned(want string) *tls.Config {
return &tls.Config{
InsecureSkipVerify: true, //nolint:gosec // replaced by the pin below, which is stricter
MinVersion: tls.VersionTLS12,
VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
if len(rawCerts) == 0 {
return errors.New("the bus presented no certificate")
}
sum := sha256.Sum256(rawCerts[0])
if got := "sha256:" + hex.EncodeToString(sum[:]); got != want {
return fmt.Errorf("the bus presented a certificate this mesh does not know (%.23s…)", got)
}
return nil
},
}
}
// delivered is the part of a JetStream message the bridge uses.
type delivered interface {
Subject() string
@@ -67,8 +117,8 @@ type publisher interface {
// handle turns one event into its message on FORGE and acknowledges it. An event it cannot read is terminated, never
// redelivered, and the log names its subject and why. A message it could not write stays unacknowledged: the server
// delivers it again (the server's ack wait is the only clock).
func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, list bridge.Listed, logger *log.Logger) {
msg, err := bridge.ConvertWith(m.Subject(), m.Data(), trees, files, list)
func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, logger *log.Logger) {
msg, err := bridge.Convert(m.Subject(), m.Data(), trees, files)
switch {
case errors.Is(err, bridge.ErrRetry):
logger.Printf("%v; heard again in %s", err, retryDelay)
@@ -102,23 +152,40 @@ func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees,
// run is the program: it returns when ctx ends or the bus cannot be used.
func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
logger := log.New(logOut, "mesh-delivery-forge-bridge: ", log.LstdFlags)
cred, err := process.ReadBrokerAccount(getenv)
path := strings.TrimSpace(getenv("MESH_BROKER_FILE"))
if path == "" {
return errors.New("MESH_BROKER_FILE is not set, so there is no credential to hear the bus with")
}
cred, err := readCredential(path)
if err != nil {
return err
}
again := make(chan struct{}, 1)
conn, js, err := cred.Dial(
opts := []nats.Option{
nats.Name(cred.Node + "." + cred.Module),
nats.UserInfo(cred.User, cred.Password),
nats.CustomInboxPrefix("_INBOX." + cred.User),
nats.MaxReconnects(-1),
nats.RetryOnFailedConnect(true),
nats.ReconnectHandler(func(*nats.Conn) {
select {
case again <- struct{}{}:
default:
}
}),
)
}
if strings.TrimSpace(cred.Fingerprint) != "" {
opts = append(opts, nats.Secure(pinned(cred.Fingerprint)))
}
conn, err := nats.Connect(cred.URL, opts...)
if err != nil {
return fmt.Errorf("connecting to the bus: %w", err)
}
defer conn.Close()
js, err := jetstream.New(conn)
if err != nil {
return err
}
defer conn.Close()
// What the forge holds open, said before any event is heard, and again at each reconnect (ADR 0429 §13).
call := callerOn(conn, cred.User)
// No forge event is heard while the open pull requests are read and said (ADR 0443 §5): events and syncs take turns.
@@ -160,19 +227,20 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return err
}
go onReconnect(ctx, again, said, logger)
cons, err := cred.EventConsumer(ctx, js, eventsStream)
cons, err := js.Consumer(ctx, eventsStream, cred.Node+"_"+cred.Module)
if err != nil {
// Essential: the forge's events are what the bridge carries.
return fmt.Errorf("the consumer %s_%s on %s cannot be read (%v): the bridge hears no forge event without it, so it stops; where the bus refused it or did not answer, this module's consumes (made by the controller with its grant) have not reached the bus yet", cred.Node, cred.Module, eventsStream, err)
}
cc, err := process.HearEvents(cons, func(m jetstream.Msg) {
cc, err := cons.Consume(func(m jetstream.Msg) {
hear(&turn, func() {
// Bounded: the turn is held, and no sync or other event goes on without it.
hctx, cancel := context.WithTimeout(ctx, syncDeadline)
defer cancel()
handle(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), listedVia(hctx, call), logger)
handle(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), logger)
})
}, logger.Printf)
},
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
if err != nil {
return err
}
@@ -81,7 +81,7 @@ func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit
})
var logs bytes.Buffer
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{}
handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0))
handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0))
if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull" {
t.Fatalf("published on %v", pub.subjects)
}
@@ -97,7 +97,7 @@ func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit
func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5
var logs bytes.Buffer
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(map[string]any{"head_sha": "short"})}, &fakePub{}
handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0))
handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0))
if len(pub.subjects) != 0 {
t.Errorf("a message was written: %v", pub.subjects)
}
@@ -112,7 +112,7 @@ func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5
func TestUnwrittenMessageIsNotAcked(t *testing.T) { // 1235 (a failed write is redelivered by the server, not retried here)
var logs bytes.Buffer
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{err: errors.New("no stream")}
handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0))
handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0))
if m.acked != 0 || m.termd != 0 {
t.Errorf("acked %d terminated %d, want neither", m.acked, m.termd)
}
@@ -134,7 +134,7 @@ func TestTreesRefusedLeavesTheEventForTenSecondsAndLogs(t *testing.T) { // 1236/
call := func(context.Context, string, any) (json.RawMessage, error) {
return nil, errors.New("the tree listing of novox/mesh-delivery was cut")
}
handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), nil, log.New(&logs, "", 0))
handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), log.New(&logs, "", 0))
if len(pub.subjects) != 0 {
t.Errorf("a message was written: %v", pub.subjects)
}
@@ -163,7 +163,7 @@ func TestMergeCallsTheToolOnceAndWritesTheMergedMessage(t *testing.T) { // 1236/
}
return json.RawMessage(`{"root":"r0","trees":{"":"t0","modules/a":"t1"}}`), nil
}
handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), nil, log.New(&logs, "", 0))
handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), log.New(&logs, "", 0))
if len(keys) == 0 || keys[0] != "gitea.gitea_commit_trees" || slices.Index(keys[1:], "gitea.gitea_commit_trees") >= 0 {
t.Fatalf("calls %v, want the trees once and first", keys)
}
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+4 -2
View File
@@ -1,5 +1,7 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952 h1:TmveT3Bija643RGWkilvkzP/h8UWlVvHFZZtQfJK5oU=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -7,7 +7,6 @@ import (
"encoding/json"
"errors"
"fmt"
"maps"
"path"
"regexp"
"slices"
@@ -78,24 +77,6 @@ type pull struct {
ModuleDirsSaid bool `json:"module_dirs_said"`
Base string `json:"base"`
MergeSHA string `json:"merge_commit_sha"`
// Paths are the files the pull request changes (a removed or renamed one included), PathsTruncated says the forge
// cut the list; nil when the forge did not say them.
Paths *[]string `json:"paths"`
PathsTruncated bool `json:"paths_truncated"`
}
// touchesPlacement says whether a pull request may change a placement file: a changed path under
// delivery.PlacementDir, or paths the forge did not say or cut (the safe direction: the files are read).
func (p pull) touchesPlacement() bool {
if p.Paths == nil || p.PathsTruncated {
return true
}
for _, path := range *p.Paths {
if strings.HasPrefix(path, delivery.PlacementDir+"/") {
return true
}
}
return false
}
func unreadable(subject, why string, args ...any) error {
@@ -107,19 +88,6 @@ func unreadable(subject, why string, args ...any) error {
// the trunk asks trees once and becomes a merged message; a merge into any other branch is a closed pull message, as the
// pull request left its group. A forge that cannot say is ErrRetry; a merge with no module.json is ErrNoModules.
func Convert(subject string, data []byte, trees Trees, files Files) (Message, error) {
return ConvertWith(subject, data, trees, files, nil)
}
// Listed asks the forge for the files directly under one directory of a commit, by name to blob id (gitea_commit_trees
// with listed); a directory the commit lacks lists none.
type Listed func(owner, repo, sha, dir string) (map[string]string, error)
// ConvertWith is Convert; with list, a merge of the catalogue repository (delivery.CatalogueRepository) also carries every
// placement file of its merge commit (novox/hq ADR 0549 §2, issue 1126), read whole or not at all: a forge that cannot
// say is ErrRetry, as for the manifests, so the indexer never takes a part of placement for all of it. So does an
// updated pull request of the catalogue that touches placement/ (touchesPlacement), with every placement file at its
// head, which the indexer writes as placement candidates (ADR 0549 §4: the nodes it places answer it before it merges).
func ConvertWith(subject string, data []byte, trees Trees, files Files, list Listed) (Message, error) {
var action string
switch subject {
case UpdatedSubject:
@@ -146,7 +114,7 @@ func ConvertWith(subject string, data []byte, trees Trees, files Files, list Lis
return Message{}, unreadable(subject, "no branch")
}
if subject == MergedSubject && p.Base == Trunk {
return merged(subject, p, trees, files, list)
return merged(subject, p, trees, files)
}
f := delivery.Forge{Kind: delivery.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number,
Head: p.HeadSHA, Action: action}
@@ -158,13 +126,6 @@ func ConvertWith(subject string, data []byte, trees Trees, files Files, list Lis
return Message{}, fmt.Errorf("%w: %s: the manifests of %s/%s at the head %s: %v", ErrRetry, subject, p.Owner, p.Repo, p.HeadSHA, err)
}
}
if action == delivery.PullUpdated && list != nil && delivery.SameRepository(f.Repository, delivery.CatalogueRepository) && p.touchesPlacement() {
var err error
if f.Placements, err = placementsOf(p.Owner, p.Repo, p.HeadSHA, list, files); err != nil {
return Message{}, fmt.Errorf("%w: %s: the placement files of %s at the head %s: %v", ErrRetry, subject, f.Repository, p.HeadSHA, err)
}
f.PlacementsRead = true
}
event, err := delivery.ForgeEvent(f.Repository, delivery.ForgePull)
if err != nil {
return Message{}, unreadable(subject, "%v", err)
@@ -266,7 +227,7 @@ func inputsOf(owner, repo, ref string, dirs []string, files Files) (manifests ma
}
// merged is the merged message of a merge into the trunk: the trees of the merge commit, asked once.
func merged(subject string, p pull, trees Trees, files Files, list Listed) (Message, error) {
func merged(subject string, p pull, trees Trees, files Files) (Message, error) {
if !sha40.MatchString(p.MergeSHA) {
return Message{}, unreadable(subject, "the merge commit %q is not 40 hex digits", p.MergeSHA)
}
@@ -291,15 +252,8 @@ func merged(subject string, p pull, trees Trees, files Files, list Listed) (Mess
if err != nil {
return Message{}, unreadable(subject, "%v", err)
}
f := delivery.Forge{Kind: delivery.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number,
Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs, Manifests: manifests, Locks: locks}
if list != nil && delivery.SameRepository(repository, delivery.CatalogueRepository) {
if f.Placements, err = placementsOf(p.Owner, p.Repo, p.MergeSHA, list, files); err != nil {
return Message{}, fmt.Errorf("%w: %s: the placement files of %s at the merge commit %s: %v", ErrRetry, subject, repository, p.MergeSHA, err)
}
f.PlacementsRead = true
}
body, err := delivery.EncodeForge(f)
body, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number,
Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs, Manifests: manifests, Locks: locks})
if err != nil {
return Message{}, unreadable(subject, "%v", err)
}
@@ -396,30 +350,3 @@ func OpenMessage(answer []byte) (Message, error) {
}
return Message{Event: delivery.OpenEvent(), Data: body}, nil
}
// placementsOf is every placement file of the catalogue at a commit, by node, its bytes exact: each `<node>.json` directly
// under delivery.PlacementDir. A file whose name is no node is left out and the rest read; the catalogue's check refuses
// such a file before it merges (ADR 0549 §1). Nil when there is none.
func placementsOf(owner, repo, sha string, list Listed, files Files) (map[string]string, error) {
names, err := list(owner, repo, sha, delivery.PlacementDir)
if err != nil {
return nil, err
}
var out map[string]string
for _, name := range slices.Sorted(maps.Keys(names)) {
file := delivery.PlacementDir + "/" + name
node, ok := delivery.PlacementFileNode(file)
if !ok {
continue
}
raw, err := files(owner, repo, file, sha)
if err != nil {
return nil, fmt.Errorf("%s: %w", file, err)
}
if out == nil {
out = map[string]string{}
}
out[node] = string(raw)
}
return out, nil
}
@@ -79,47 +79,6 @@ func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1
if want := map[string]map[string]string{"modules/a": {"go.sum": hexSum(sum)}}; !reflect.DeepEqual(f.Locks, want) {
t.Errorf("locks = %v, want %v", f.Locks, want)
}
if f.Placements != nil || f.PlacementsRead {
t.Errorf("a merge of another repository than the catalogue carries placement: %v", f.Placements)
}
// Issue 1126 (ADR 0549 §2): a merge of the catalogue carries every placement file of its merge commit, by node; a file
// whose name is no node is left out; a listing or a file the forge cannot give is a retry, never a part.
catalogue := mergedBody(map[string]any{"repo": "mesh-catalog"})
forge.have["placement/ace.json"] = []byte(`{"v":1,"node":"ace","modules":[]}`)
forge.have["placement/g14.json"] = []byte(`{"v":1,"node":"g14","modules":[{"module":"zsh"}]}`)
listing := map[string]string{"ace.json": "b1", "g14.json": "b2", "README.md": "b3", "a.b.json": "b4"}
var listedAt []string
list := func(owner, repo, sha, dir string) (map[string]string, error) {
listedAt = append(listedAt, owner+"/"+repo+"@"+sha+":"+dir)
return listing, nil
}
msg, err = ConvertWith(mergedSubject, catalogue, treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get, list)
if err != nil {
t.Fatal(err)
}
if f, err = delivery.DecodeForge(msg.Data); err != nil {
t.Fatal(err)
}
want := map[string]string{"ace": `{"v":1,"node":"ace","modules":[]}`, "g14": `{"v":1,"node":"g14","modules":[{"module":"zsh"}]}`}
if !f.PlacementsRead || !reflect.DeepEqual(f.Placements, want) || !reflect.DeepEqual(listedAt, []string{"novox/mesh-catalog@" + merge + ":placement"}) {
t.Errorf("placements = %v (read %v), listed at %v", f.Placements, f.PlacementsRead, listedAt)
}
listing = map[string]string{}
if msg, err = ConvertWith(mergedSubject, catalogue, treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get, list); err != nil {
t.Fatal(err)
}
if f, _ = delivery.DecodeForge(msg.Data); !f.PlacementsRead || f.Placements != nil {
t.Errorf("a catalogue with no placement file says it read none: %v, %v", f.Placements, f.PlacementsRead)
}
listing = map[string]string{"gone.json": "b5"}
if _, err := ConvertWith(mergedSubject, catalogue, treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get, list); !errors.Is(err, ErrRetry) || !strings.Contains(err.Error(), "placement/gone.json") {
t.Errorf("a placement file the forge cannot give: %v", err)
}
failing := func(_, _, _, _ string) (map[string]string, error) { return nil, errors.New("forge down") }
if _, err := ConvertWith(mergedSubject, catalogue, treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get, failing); !errors.Is(err, ErrRetry) {
t.Errorf("a listing the forge cannot give: %v", err)
}
}
func TestPullWithKnownDirsCarriesTheSameAtTheHead(t *testing.T) { // 1448/AC2
@@ -146,55 +105,6 @@ func TestPullWithKnownDirsCarriesTheSameAtTheHead(t *testing.T) { // 1448/AC2
if f.Manifests != nil || f.Locks != nil || len(forge.calls) != 0 {
t.Errorf("manifests %v locks %v calls %v", f.Manifests, f.Locks, forge.calls)
}
// Issue 1126 (ADR 0549 §4): an updated pull request of the catalogue that touches placement/ carries every placement
// file at its head, read whole or not at all; one that touches none, or of another repository, carries none and lists
// nothing; paths the forge did not say or cut are read as touching; a closed one carries none.
forge = &forgeFiles{have: map[string][]byte{"modules/a/module.json": manifest,
"placement/ace.json": []byte(`{"v":1,"node":"ace","modules":[]}`), "placement/g14.json": []byte(`{"v":1,"node":"g14","modules":[]}`)}}
listing := map[string]string{"ace.json": "b1", "g14.json": "b2", "README.md": "b3"}
var listedAt []string
list := func(owner, repo, sha, dir string) (map[string]string, error) {
listedAt = append(listedAt, owner+"/"+repo+"@"+sha+":"+dir)
return listing, nil
}
all := map[string]string{"ace": `{"v":1,"node":"ace","modules":[]}`, "g14": `{"v":1,"node":"g14","modules":[]}`}
catalogue := map[string]any{"repo": "mesh-catalog", "paths": []string{"placement/g14.json"}}
for _, row := range []struct {
name string
subject string
body map[string]any
want map[string]string
read bool
}{
{"a catalogue pull touching placement", updatedSubject, catalogue, all, true},
{"a catalogue pull touching no placement", updatedSubject, map[string]any{"repo": "mesh-catalog"}, nil, false},
{"a catalogue pull whose paths are cut", updatedSubject, map[string]any{"repo": "mesh-catalog", "paths_truncated": true}, all, true},
{"a catalogue pull whose paths are not said", updatedSubject, map[string]any{"repo": "mesh-catalog", "paths": nil}, all, true},
{"another repository's pull touching placement/", updatedSubject, map[string]any{"paths": []string{"placement/g14.json"}}, nil, false},
{"a closed catalogue pull", closedSubject, catalogue, nil, false},
} {
listedAt = nil
body := updated(row.body)
msg, err := ConvertWith(row.subject, body, nil, forge.get, list)
if err != nil {
t.Fatalf("%s: %v", row.name, err)
}
f, err := delivery.DecodeForge(msg.Data)
if err != nil {
t.Fatalf("%s: %v", row.name, err)
}
if f.PlacementsRead != row.read || !reflect.DeepEqual(f.Placements, row.want) || (len(listedAt) > 0) != row.read {
t.Errorf("%s: placements %v (read %v), listed %v", row.name, f.Placements, f.PlacementsRead, listedAt)
}
if row.read && listedAt[0] != "novox/mesh-catalog@"+head+":placement" {
t.Errorf("%s: listed at %v, want the head", row.name, listedAt)
}
}
listing = map[string]string{"gone.json": "b5"}
if _, err := ConvertWith(updatedSubject, updated(catalogue), nil, forge.get, list); !errors.Is(err, ErrRetry) || !strings.Contains(err.Error(), "placement/gone.json") {
t.Errorf("a placement file the forge cannot give at the head: %v", err)
}
}
func TestAForgeErrorOrAMissingManifestIsRetryAndALockNotFoundIsLeftOut(t *testing.T) { // 1448/AC3
@@ -1,130 +0,0 @@
// Command mesh-delivery-indexer-tools serves the indexer's doing tool (novox/hq issue 1945): the runtime launches it for a
// call. indexer_take_in writes one value of the indexer's takeins state and waits for the indexer's answer there. The
// indexer is the other program: it serves no request subject (ADR 0437 §4). The read tools of the indexer's states are
// mesh-delivery-view's (indexer_module, indexer_modules, indexer_candidate, indexer_candidates).
package main
import (
"context"
"encoding/json"
"fmt"
"os"
"strings"
"sync"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"mesh-delivery-indexer/internal/index"
)
// asker writes a takeins value and waits for the indexer's answer to it.
type asker func(key string, value []byte) (index.TakeInAsk, error)
// Tools are the indexer's tools.
func Tools(ask asker, now func() time.Time) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
return []stdio.Tool{
{Name: index.TakeInTool,
Description: "Take a registered module in now: the indexer reads its manifest from the repository and commit the controller's " +
"build records register it from (at the head the gate's source names, when it names one), whatever it tried before, and " +
"writes it to its modules state as a merge would. For a module registered from outside the indexed repositories whose " +
"repository the forge could not read (a 404 before the forge's account could read it). Answers stored, from which " +
"repository, commit and path, or why not (held already, no registration, the forge's error, no manifest there). " +
"The indexer reads such a module again by itself on the next source revision and merged message; this asks now. " +
"indexer_module (mesh-delivery-view) reads the result.",
Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "the module's name, as the catalogue registers it"},
"why": map[string]any{"type": "string", "description": "why it is taken in now, in plain words"}}},
Run: func(args map[string]any) (any, error) {
key, value, err := index.NewTakeIn(str(args, "module"), str(args, "why"), stdio.CallerOf(args), now())
if err != nil {
return nil, err
}
a, err := ask(key, value)
if err != nil {
return nil, err
}
return a.Result, nil
}},
{Name: index.PlacementRereadTool,
Description: "Read the catalogue's placement files again now (novox/hq ADR 0559 §8): the indexer asks the forge for the trunk's head and " +
"every open catalogue pull request's, and writes its states placement (placement.<node>) and placement-candidates (<group>.<node>) " +
"as a merge and a pull request's head would, only where a key differs; a candidate key whose file no longer differs, or whose pull " +
"request is closed, is deleted. The recovery for a forge message the indexer missed. Answers the heads read, the keys written or " +
"deleted in each state, or why nothing was (the forge could not say). indexer_placement (mesh-delivery-view) reads the result.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"why": map[string]any{"type": "string", "description": "why it is read again now, in plain words"}}},
Run: func(args map[string]any) (any, error) {
key, value, err := index.NewPlacementReread(str(args, "why"), stdio.CallerOf(args), now())
if err != nil {
return nil, err
}
a, err := ask(key, value)
if err != nil {
return nil, err
}
return a.Reread, nil
}},
}
}
// askTakeIn writes the takeins value at key and waits, on a watch of the key, at most bound for the indexer's answer.
func askTakeIn(s stdio.KeptState, bound time.Duration) asker {
return func(key string, value []byte) (index.TakeInAsk, error) {
answered := make(chan index.TakeInAsk, 1)
var rev uint64
var mu sync.Mutex
take := func(revision uint64, v []byte) bool {
mu.Lock()
mine := rev
mu.Unlock()
if mine == 0 || revision <= mine {
return false
}
a, err := index.DecodeTakeIn(v)
if err != nil || !a.Done() {
return false
}
select {
case answered <- a:
default:
}
return true
}
if err := s.Watch(key, func(c stdio.StateChange) error {
if c.Op == "put" {
take(c.Revision, c.Value)
}
return nil
}); err != nil {
return index.TakeInAsk{}, fmt.Errorf("watching %s.%s: %w", index.TakeInsState, key, err)
}
r, err := s.Put(key, json.RawMessage(value))
if err != nil {
return index.TakeInAsk{}, fmt.Errorf("the ask could not be written to %s: %w", index.TakeInsBucket, err)
}
mu.Lock()
rev = r
mu.Unlock()
// The watch may have heard the answer before the revision was known: read it once too.
if e, err := s.Get(key); err == nil && e != nil {
take(e.Revision, e.Value)
}
wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8)
defer cancel()
select {
case a := <-answered:
return a, nil
case <-wait.Done():
return index.TakeInAsk{}, fmt.Errorf("the indexer did not answer within %s: the take-in is still running (a forge read can take 30 s); its answer will be in %s.%s", bound, index.TakeInsBucket, key)
}
}
}
func main() {
if err := stdio.Serve("", Tools(askTakeIn(stdio.State(index.TakeInsState), 25*time.Second), time.Now)); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err)
os.Exit(1)
}
}
@@ -20,7 +20,6 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
)
// busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the
@@ -172,7 +171,7 @@ func startIndexer(t *testing.T, put func(context.Context, jetstream.KeyValue), b
}
broker := filepath.Join(t.TempDir(), "broker")
cred, _ := json.Marshal(process.Account{URL: srv.ClientURL(), Node: "n1", Module: "mesh-delivery-indexer", User: "n1.mesh-delivery-indexer", Password: "pw"})
cred, _ := json.Marshal(credential{URL: srv.ClientURL(), Node: "n1", Module: "mesh-delivery-indexer", User: "n1.mesh-delivery-indexer", Password: "pw"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
@@ -1,8 +1,7 @@
// Command mesh-delivery-indexer is delivery's indexer (novox/hq ADR 0437 §4, §5, issue 1238): it hears the forge
// bridge's messages on the stream FORGE and keeps the candidate groups (candidates.<group>): the pull requests that share
// a branch, their heads and the modules each touches. It reads the controller's builds to know which module a changed
// directory is. It serves no request subject and holds no timer: every reaction is a message or a watch (ADR 0437 §8). Its
// tool indexer_take_in (the program mesh-delivery-indexer-tools) asks through its state takeins, which it watches.
// directory is. It serves no request subject and holds no timer: every reaction is a message or a watch (ADR 0437 §8).
//
// Its build of 2026-10-24 00:37 named a program whose push never reached the artifact store, and a retry sends that
// same build again; this paragraph was added only to give the indexer a new build (issue 1933).
@@ -10,6 +9,10 @@ package main
import (
"context"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
@@ -17,6 +20,7 @@ import (
"log"
"os"
"os/signal"
"strings"
"syscall"
"github.com/nats-io/nats.go"
@@ -42,56 +46,48 @@ func main() {
}
}
// awaitedStore is a state bound when it appears (process.AwaitBucket) as the indexer's store: until then every call is
// process.ErrNotYet.
type awaitedStore struct{ b *process.AwaitedBucket }
func (s awaitedStore) store() (kvStore, error) {
kv := s.b.KV()
if kv == nil {
return kvStore{}, process.ErrNotYet
}
return kvStore{kv}, nil
// credential is the broker file the controller issues to the module: JSON of the bus address, the certificate
// fingerprint to pin, the principal and its password (as the watchman reads it; issue 1242 moves this into the SDK).
type credential struct {
URL string `json:"url"`
Fingerprint string `json:"fingerprint"`
Node string `json:"node"`
Module string `json:"module"`
User string `json:"user"`
Password string `json:"password"`
}
func (s awaitedStore) Get(ctx context.Context, key string) ([]byte, uint64, bool, error) {
st, err := s.store()
func readCredential(path string) (credential, error) {
raw, err := os.ReadFile(path)
if err != nil {
return nil, 0, false, err
return credential{}, err
}
return st.Get(ctx, key)
var c credential
if err := json.Unmarshal(raw, &c); err != nil {
return credential{}, fmt.Errorf("the broker credential is not one: %w", err)
}
if c.URL == "" || c.User == "" {
return credential{}, errors.New("the broker credential names no bus address or user")
}
return c, nil
}
func (s awaitedStore) Create(ctx context.Context, key string, value []byte) error {
st, err := s.store()
if err != nil {
return err
// pinned accepts exactly the certificate with this SHA-256 and no other.
func pinned(want string) *tls.Config {
return &tls.Config{
InsecureSkipVerify: true, //nolint:gosec // replaced by the pin below, which is stricter
MinVersion: tls.VersionTLS12,
VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
if len(rawCerts) == 0 {
return errors.New("the bus presented no certificate")
}
sum := sha256.Sum256(rawCerts[0])
if got := "sha256:" + hex.EncodeToString(sum[:]); got != want {
return fmt.Errorf("the bus presented a certificate this mesh does not know (%.23s…)", got)
}
return nil
},
}
return st.Create(ctx, key, value)
}
func (s awaitedStore) Update(ctx context.Context, key string, value []byte, revision uint64) error {
st, err := s.store()
if err != nil {
return err
}
return st.Update(ctx, key, value, revision)
}
func (s awaitedStore) Delete(ctx context.Context, key string, revision uint64) error {
st, err := s.store()
if err != nil {
return err
}
return st.Delete(ctx, key, revision)
}
func (s awaitedStore) Keys(ctx context.Context) ([]string, error) {
st, err := s.store()
if err != nil {
return nil, err
}
return st.Keys(ctx)
}
// kvStore is the candidates bucket as the indexer's store: compare-and-set through Create, Update and Delete at a revision.
@@ -143,7 +139,6 @@ type delivered interface {
Subject() string
Data() []byte
Ack() error
DoubleAck(ctx context.Context) error
Term() error
}
@@ -164,12 +159,7 @@ func handle(ctx context.Context, m delivered, ix *index.Indexer, logger *log.Log
case err != nil:
logger.Printf("%s: %s#%d left for redelivery: %v", m.Subject(), f.Repository, f.Number, err)
default:
// Waits for the server, so the consumer's info no longer counts the message as unacknowledged when the fill that
// follows asks whether the stream is applied (issue 1280); a failed ack is redelivered after the ack wait, which
// kicks the fill again.
if err := m.DoubleAck(ctx); err != nil {
logger.Printf("%s: %s#%d applied, its acknowledgement was not confirmed: %v", m.Subject(), f.Repository, f.Number, err)
}
_ = m.Ack()
}
}
@@ -223,15 +213,33 @@ func follow(ctx context.Context, w jetstream.KeyWatcher, each func(jetstream.Key
// run is the program: it returns when ctx ends or the bus cannot be used.
func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
logger := log.New(logOut, "mesh-delivery-indexer: ", log.LstdFlags)
cred, err := process.ReadBrokerAccount(getenv)
path := strings.TrimSpace(getenv("MESH_BROKER_FILE"))
if path == "" {
return errors.New("MESH_BROKER_FILE is not set, so there is no credential to hear the bus with")
}
cred, err := readCredential(path)
if err != nil {
return err
}
conn, js, err := cred.Dial()
opts := []nats.Option{
nats.Name(cred.Node + "." + cred.Module),
nats.UserInfo(cred.User, cred.Password),
nats.CustomInboxPrefix("_INBOX." + cred.User),
nats.MaxReconnects(-1),
nats.RetryOnFailedConnect(true),
}
if strings.TrimSpace(cred.Fingerprint) != "" {
opts = append(opts, nats.Secure(pinned(cred.Fingerprint)))
}
conn, err := nats.Connect(cred.URL, opts...)
if err != nil {
return err
return fmt.Errorf("connecting to the bus: %w", err)
}
defer conn.Close()
js, err := jetstream.New(conn)
if err != nil {
return err
}
candidates, err := process.EssentialBucket(ctx, js, "indexer", delivery.CandidatesBucket, "write a candidate's fingerprints")
if err != nil {
return err
@@ -244,15 +252,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil {
return err
}
// Its taken state (ADR 0556 clause 1) is bound when it appears, as every state a delivery adds (issue 1790): until then
// a merge's taken key is said and not written, and the rest of the indexer runs.
taken := process.AwaitBucket(ctx, js, conn, delivery.TakenBucket, "no repository's taken key is written", logger.Printf)
// Its placement state (ADR 0549 §2) is bound when it appears, as every state a delivery adds (issue 1790): until then a
// catalogue merge's placement is said and not written, and the rest of the indexer runs.
placement := process.AwaitBucket(ctx, js, conn, delivery.PlacementBucket, "no catalogue merge's placement is written", logger.Printf)
// Its placement candidates (issue 1126, ADR 0549 §4) are bound the same way: until the state appears a catalogue pull
// request's placement is said and not written, so no node answers it and its check names every placed node as silent.
placementCandidates := process.AwaitBucket(ctx, js, conn, delivery.PlacementCandidatesBucket, "no catalogue pull request's placement candidate is written", logger.Printf)
// The controller's build records only name the directory of a module whose key names none (fillPaths): without them
// such a module waits for its path, every other is indexed (issue 1945).
builds := process.OptionalBucket(ctx, js, delivery.BuildsBucket, "a module whose key names no directory is not given one from the controller's build records until the indexer starts again", logger.Printf)
@@ -267,24 +266,26 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
}
errs := make(chan error, 1)
var buildsWatch jetstream.KeyWatcher
if builds != nil {
if w, err := settle(ctx, builds, each); err != nil {
process.NotWatched(logger.Printf, delivery.BuildsBucket, err, "a module whose key names no directory is not given one until the indexer starts again")
} else {
buildsWatch = w
go func() { errs <- follow(ctx, w, each) }()
}
}
// The indexer asks the forge for the objects of a manifest's named paths as its own account (ADR 0499 clause 3).
calls, err := process.Connect(cred, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
account, err := process.ReadAccount(path)
if err != nil {
return err
}
calls, err := process.Connect(account, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
if err != nil {
return err
}
defer calls.Close()
ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known, Logf: logger.Printf,
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call),
Placement: awaitedStore{placement}, PlacementCandidates: awaitedStore{placementCandidates}, ReadPlacement: placementVia(calls.Call), Say: func(event string, body any) error { return calls.Emit(event, body) }}
ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Builds: known, Logf: logger.Printf,
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call)}
// The gate's source bucket (mesh-delivery-gate_source) comes with the gate (release 3e, novox/hq ADR 0450 §1): its key
// main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does
// not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate.
@@ -335,40 +336,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err := ix.RecomputeAtStart(ctx); err != nil {
logger.Printf("computing the new-form fingerprints at start: %v", err)
}
// A module registered in the catalogue from its own repository gets its manifest indexed when the controller's records
// name it and the bucket holds none (registered.go); the records that follow are heard below.
if err := ix.IndexRegisteredAll(ctx); err != nil {
logger.Printf("indexing the registered modules at start: %v", err)
}
if buildsWatch != nil {
go func() {
errs <- follow(ctx, buildsWatch, func(e jetstream.KeyValueEntry) {
each(e)
if !opOf(e) {
return
}
var r delivery.BuildRecord
if json.Unmarshal(e.Value(), &r) != nil || r.Module == "" {
return
}
if err := ix.IndexRegisteredModule(ctx, r.Module); err != nil {
logger.Printf("the registered module %s is not indexed: %v", r.Module, err)
}
})
}()
}
// A person takes a registered module in now with indexer_take_in, which writes the takeins state (issue 1945): watched,
// as every other input. A module the forge could not read is also read again by itself on the next source revision and
// merged message (index.RetryUnread).
// A state not on the bus yet at this start is watched once it appears (issue 1790).
process.AwaitBucket(ctx, js, conn, index.TakeInsBucket, "indexer_take_in is not answered; a module the forge could not read is still read again by itself", logger.Printf).Ready(func(kv jetstream.KeyValue) {
eachTakeIn := func(e jetstream.KeyValueEntry) { takenIn(ctx, ix, kv, e, logger.Printf) }
if tw, err := settle(ctx, kv, eachTakeIn); err != nil {
process.NotWatched(logger.Printf, index.TakeInsBucket, err, "indexer_take_in is not answered until the indexer starts again")
} else {
go func() { errs <- follow(ctx, tw, eachTakeIn) }()
}
})
// The modules the trunk no longer holds go from the bucket (issue 1900); a merge does the same from its own message.
if err := ix.Reconcile(ctx); err != nil {
logger.Printf("dropping the modules the trunk does not hold at start: %v", err)
@@ -378,43 +345,36 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// with the build module, as with the gate's source.
for _, module := range sortedModules() {
stream := index.ToolchainStreams[module]
tcons, err := cred.EventConsumer(ctx, js, stream)
tcons, err := js.Consumer(ctx, stream, cred.Node+"_"+cred.Module)
if err != nil {
logger.Printf("the consumer %s_%s on %s does not exist yet, so %s's reports are not heard until the indexer starts again: %v", cred.Node, cred.Module, stream, module, err)
continue
}
tc, err := process.HearEvents(tcons, func(m jetstream.Msg) { tools.Reported(m) }, logger.Printf)
tc, err := tcons.Consume(func(m jetstream.Msg) { tools.Reported(m) },
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
if err != nil {
return err
}
defer tc.Stop()
}
switch pcons, err := cred.EventConsumer(ctx, js, PreviewsStream); {
switch pcons, err := js.Consumer(ctx, PreviewsStream, cred.Node+"_"+cred.Module); {
case err != nil:
logger.Printf("the consumer %s_%s on %s does not exist yet, so the checker's previews are not heard until the indexer starts again: %v", cred.Node, cred.Module, PreviewsStream, err)
default:
pc, err := process.HearEvents(pcons, func(m jetstream.Msg) { previews.Written(m) }, logger.Printf)
pc, err := pcons.Consume(func(m jetstream.Msg) { previews.Written(m) },
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
if err != nil {
return err
}
defer pc.Stop()
}
cons, err := cred.EventConsumer(ctx, js, forgeStream)
cons, err := js.Consumer(ctx, forgeStream, cred.Node+"_"+cred.Module)
if err != nil {
// Essential: the forge's merges are what the indexer indexes.
return fmt.Errorf("the consumer %s_%s on %s cannot be read (%v): the indexer hears no merge without it, so it stops; where the bus refused it or did not answer, this module's consumes (made by the controller with its grant) have not reached the bus yet", cred.Node, cred.Module, forgeStream, err)
}
// A merge applied while the taken state was missing, and the gate's heads once the forge stream is applied whole, get
// their taken key (issue 1280): when the state appears (Ready runs once, now or then), and after each applied message.
ix.Drained = func(ctx context.Context) (bool, error) {
info, err := cons.Info(ctx)
if err != nil {
return false, err
}
return info.NumPending == 0 && info.NumAckPending == 0, nil
}
taken.Ready(func(jetstream.KeyValue) { ix.KickFill(ctx) })
cc, err := process.HearEvents(cons, func(m jetstream.Msg) { handle(ctx, m, ix, logger); ix.KickFill(ctx) }, logger.Printf)
cc, err := cons.Consume(func(m jetstream.Msg) { handle(ctx, m, ix, logger) },
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
if err != nil {
return err
}
@@ -16,18 +16,12 @@ type fakeMsg struct {
subject string
data []byte
acked, termd int
doubled int // acked by DoubleAck, which waits for the server
}
func (f *fakeMsg) Subject() string { return f.subject }
func (f *fakeMsg) Data() []byte { return f.data }
func (f *fakeMsg) Ack() error { f.acked++; return nil }
func (f *fakeMsg) DoubleAck(context.Context) error {
f.acked++
f.doubled++
return nil
}
func (f *fakeMsg) Term() error { f.termd++; return nil }
func (f *fakeMsg) Term() error { f.termd++; return nil }
type memStore struct{ vals map[string][]byte }
@@ -59,7 +53,7 @@ func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part)
m := &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull", data: data}
handle(context.Background(), m, ix, log.New(&bytes.Buffer{}, "", 0))
c, err := delivery.DecodeCandidate(st.vals["feat/x"])
if err != nil || len(c.Pulls) != 1 || c.Pulls[0].Modules[0] != "a" || m.acked != 1 || m.doubled != 1 || m.termd != 0 {
if err != nil || len(c.Pulls) != 1 || c.Pulls[0].Modules[0] != "a" || m.acked != 1 || m.termd != 0 {
t.Errorf("candidate %+v err %v acked %d terminated %d", c, err, m.acked, m.termd)
}
}
@@ -2,16 +2,11 @@ package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"path"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-indexer/internal/index"
)
// commitTreesTool is the forge's tool for a commit's trees (mesh-catalog modules/gitea): with `paths`, by module directory
@@ -98,182 +93,3 @@ func containsVia(call toolCaller) func(ctx context.Context, repository, commit,
return *a.Contains, nil
}
}
// getFileTool is the forge's tool for one file at a ref (mesh-catalog modules/gitea gitea_get_file): arguments owner,
// repo, path and ref, answer {file: {content}}; a path it does not have fails with a 404 in its words.
const getFileTool = "gitea.gitea_get_file"
// lockFiles are read beside a manifest, as the forge bridge reads them (bridge.LockFiles).
var lockFiles = []string{"go.sum", "package-lock.json"}
// readVia reads a repository at a commit as the forge bridge does for a merged message: the trees by gitea_commit_trees,
// then the module.json and lock files of the one directory dir by gitea_get_file (a repository can hold hundreds). A manifest the
// forge does not have is an error; a directory with no module.json in the trees is answered with no manifest; a lock file it does not have is left out.
func readVia(call toolCaller) func(ctx context.Context, repository, sha, dir string) (index.RepoRead, error) {
return func(_ context.Context, repository, sha, dir string) (index.RepoRead, error) {
owner, repo, ok := strings.Cut(delivery.RepositoryName(repository), "/")
if !ok {
return index.RepoRead{}, fmt.Errorf("%q is not <owner>/<name>", repository)
}
raw, err := call(commitTreesTool, map[string]any{"owner": owner, "repo": repo, "sha": sha})
if err != nil {
return index.RepoRead{}, err
}
var a struct {
Root string `json:"root"`
Trees map[string]string `json:"trees"`
}
if err := json.Unmarshal(raw, &a); err != nil {
return index.RepoRead{}, fmt.Errorf("the answer of %s is not {root, trees}: %w", commitTreesTool, err)
}
out := index.RepoRead{Root: a.Root, Trees: a.Trees, Manifests: map[string]string{}, Locks: map[string]map[string]string{}}
file := func(dir, name string) (string, error) {
p := name
if dir != "" {
p = dir + "/" + name
}
raw, err := call(getFileTool, map[string]string{"owner": owner, "repo": repo, "path": p, "ref": sha})
if err != nil {
return "", err
}
var f struct {
File *struct {
Content *string `json:"content"`
} `json:"file"`
}
if err := json.Unmarshal(raw, &f); err != nil || f.File == nil || f.File.Content == nil {
return "", fmt.Errorf("the answer of %s is not {file: {content}}: %v", getFileTool, err)
}
return *f.File.Content, nil
}
if _, held := a.Trees[dir]; held {
m, err := file(dir, "module.json")
if err != nil {
return index.RepoRead{}, fmt.Errorf("%s: %w", path.Join(dir, "module.json"), err)
}
out.Manifests[dir] = m
for _, name := range lockFiles {
c, err := file(dir, name)
if err != nil {
if strings.Contains(err.Error(), ": 404 ") {
continue
}
return index.RepoRead{}, fmt.Errorf("%s: %w", path.Join(dir, name), err)
}
sum := sha256.Sum256([]byte(c))
if out.Locks[dir] == nil {
out.Locks[dir] = map[string]string{}
}
out.Locks[dir][name] = hex.EncodeToString(sum[:])
}
}
return out, nil
}
}
// The forge's tools the placement reread asks (indexer_placement_reread, novox/hq ADR 0559 §8): a repository's branches
// with their commits, and every open pull request with its branch and head.
const (
listBranchesTool = "gitea.gitea_list_branches"
openPullsTool = "gitea.gitea_open_pulls"
placementTrunk = "main"
)
// placementVia reads the catalogue's placement now through call: the trunk's head (gitea_list_branches), every placement
// file at it, and each open pull request of the catalogue into the trunk (gitea_open_pulls) with every placement file at
// its head, each read whole or not at all (gitea_commit_trees with listed, gitea_get_file).
func placementVia(call toolCaller) func(ctx context.Context) (index.PlacementRead, error) {
owner, repo, _ := strings.Cut(delivery.CatalogueRepository, "/")
filesAt := func(sha string) (map[string]string, error) {
raw, err := call(commitTreesTool, map[string]any{"owner": owner, "repo": repo, "sha": sha, "listed": []string{delivery.PlacementDir}})
if err != nil {
return nil, err
}
var a struct {
Listed map[string]map[string]string `json:"listed"`
}
if err := json.Unmarshal(raw, &a); err != nil || a.Listed == nil {
return nil, fmt.Errorf("the answer of %s lists no directory: %v", commitTreesTool, err)
}
var out map[string]string
for name := range a.Listed[delivery.PlacementDir] {
file := delivery.PlacementDir + "/" + name
node, ok := delivery.PlacementFileNode(file)
if !ok {
continue
}
raw, err := call(getFileTool, map[string]string{"owner": owner, "repo": repo, "path": file, "ref": sha})
if err != nil {
return nil, fmt.Errorf("%s at %s: %w", file, sha, err)
}
var f struct {
File *struct {
Content *string `json:"content"`
} `json:"file"`
}
if err := json.Unmarshal(raw, &f); err != nil || f.File == nil || f.File.Content == nil {
return nil, fmt.Errorf("the answer of %s for %s is not {file: {content}}: %v", getFileTool, file, err)
}
if out == nil {
out = map[string]string{}
}
out[node] = *f.File.Content
}
return out, nil
}
return func(context.Context) (index.PlacementRead, error) {
raw, err := call(listBranchesTool, map[string]string{"owner": owner, "repo": repo})
if err != nil {
return index.PlacementRead{}, err
}
var b struct {
Branches []struct {
Name string `json:"name"`
Commit string `json:"commit"`
} `json:"branches"`
}
if err := json.Unmarshal(raw, &b); err != nil {
return index.PlacementRead{}, fmt.Errorf("the answer of %s is not {branches}: %w", listBranchesTool, err)
}
var out index.PlacementRead
for _, br := range b.Branches {
if br.Name == placementTrunk {
out.Trunk = br.Commit
}
}
if out.Trunk == "" {
return index.PlacementRead{}, fmt.Errorf("%s names no branch %s of %s", listBranchesTool, placementTrunk, delivery.CatalogueRepository)
}
if out.Files, err = filesAt(out.Trunk); err != nil {
return index.PlacementRead{}, err
}
raw, err = call(openPullsTool, map[string]any{})
if err != nil {
return index.PlacementRead{}, err
}
var o struct {
Pulls *[]struct {
Owner string `json:"owner"`
Repo string `json:"repo"`
Number int `json:"number"`
Base string `json:"base"`
Head string `json:"head"`
HeadSHA string `json:"head_sha"`
} `json:"pulls"`
}
if err := json.Unmarshal(raw, &o); err != nil || o.Pulls == nil {
return index.PlacementRead{}, fmt.Errorf("the answer of %s is not {pulls}: %v", openPullsTool, err)
}
for _, p := range *o.Pulls {
if !delivery.SameRepository(p.Owner+"/"+p.Repo, delivery.CatalogueRepository) || p.Base != placementTrunk || p.Number <= 0 || p.Head == "" || p.HeadSHA == "" {
continue
}
files, err := filesAt(p.HeadSHA)
if err != nil {
return index.PlacementRead{}, fmt.Errorf("%s#%d: %w", delivery.CatalogueRepository, p.Number, err)
}
out.Pulls = append(out.Pulls, index.PullPlacement{Branch: p.Head, Number: p.Number, Head: p.HeadSHA, Files: files})
}
return out, nil
}
}
@@ -3,7 +3,6 @@ package main
import (
"context"
"encoding/json"
"errors"
"slices"
"strings"
"testing"
@@ -51,33 +50,3 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3
t.Error("the trunk asked for paths")
}
}
func TestReadViaReadsManifestsAndLocks(t *testing.T) {
var files []string
call := func(key string, body any) (json.RawMessage, error) {
switch key {
case commitTreesTool:
return json.RawMessage(`{"root":"r0","trees":{"":"t1","modules/other":"t2"}}`), nil
case getFileTool:
files = append(files, body.(map[string]string)["path"])
switch body.(map[string]string)["path"] {
case "module.json":
return json.RawMessage(`{"file":{"content":"{\"module\":\"snake\"}"}}`), nil
case "go.sum":
return json.RawMessage(`{"file":{"content":"x"}}`), nil
}
return nil, errors.New("Gitea API /x: 404 not found")
}
return nil, errors.New("unexpected " + key)
}
got, err := readVia(call)(context.Background(), "http://novox.internal:20000/jschoubben/snake.git", "abc", "")
if err != nil {
t.Fatal(err)
}
if len(got.Manifests) != 1 || len(files) != 3 { // the one directory asked: its manifest and two lock files, never the others
t.Fatalf("manifests = %v, files asked = %v", got.Manifests, files)
}
if got.Root != "r0" || got.Manifests[""] != `{"module":"snake"}` || len(got.Locks[""]) != 1 || got.Locks[""]["go.sum"] == "" {
t.Fatalf("read = %+v", got)
}
}
@@ -3,12 +3,11 @@ package main
import (
"context"
"errors"
"git.novox.be/novox/mesh-sdk/go/process"
"log"
"strings"
"sync"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
@@ -1,51 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"time"
"github.com/nats-io/nats.go/jetstream"
"mesh-delivery-indexer/internal/index"
)
// takerIn is the part of the indexer a takeins value asks through.
type takerIn interface {
TakeIn(ctx context.Context, module string) index.TakenIn
RereadPlacement(ctx context.Context) index.PlacementReread
}
// takenIn answers one value of the takeins state (the tool indexer_take_in, issue 1945): a value not yet answered takes the
// module in now and is written back, by compare-and-set at its revision, with what the indexer stored or why not. A value
// answered already (this indexer's own write) or deleted is left.
func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) {
if e.Operation() != jetstream.KeyValuePut {
return
}
a, err := index.DecodeTakeIn(e.Value())
if err != nil {
logf("%s.%s holds what is not a take-in ask, left: %v", index.TakeInsState, e.Key(), err)
return
}
if !a.Answered.IsZero() {
return
}
if a.Placement {
r := ix.RereadPlacement(ctx)
a.Reread, a.Answered = &r, time.Now().UTC()
logf("a placement reread asked by %s (%s): %s", a.By, a.Why, r.Answer)
} else {
r := ix.TakeIn(ctx, a.Module)
a.Result, a.Answered = &r, time.Now().UTC()
logf("%s: a take-in asked by %s (%s): %s", a.Module, a.By, a.Why, r.Answer)
}
data, err := json.Marshal(a)
if err != nil {
return
}
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
logf("%s.%s: the answer could not be written: %v", index.TakeInsState, e.Key(), err)
}
}
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+4 -2
View File
@@ -1,5 +1,7 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952 h1:TmveT3Bija643RGWkilvkzP/h8UWlVvHFZZtQfJK5oU=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -2,8 +2,6 @@ package index
import (
"encoding/json"
"maps"
"slices"
"strings"
"sync"
@@ -90,102 +88,3 @@ func (b *Builds) PathOf(module, repository string) (string, bool) {
}
return pathOf(strings.Trim(best.Path, "/")), true
}
// Registration is a module registered in the catalogue from its own repository, as the controller's newest outcome record
// of it names it: the repository, the directory of its manifest there ("" or "." for the root) and the commit it was
// built at.
type Registration struct {
Module, Repository, Path, Commit, Ref string
}
// registered is whether the record is an outcome that did not fail: what registers a module. Unlike IsTrunkBuild it
// asks for no recorded source inputs, which no build records since ADR 0530 clause 1, and for no particular branch: a
// repository's trunk is named main, master, or what its owner chose (the registered photos, de-spiegel and link2pay were
// built from nox-mesh, and every record of them names that ref).
func registered(r delivery.BuildRecord) bool {
return r.State == "" && r.Failed == ""
}
// RegistrationsOf are the places a module was registered from, by its outcome records: the newest record of each
// repository and directory (a module's name was once another repository's, as photos was a directory of the catalogue's).
// Records of a trunk's ref (main, master, trunk, none) come before those of another branch, then the newer first, so the
// first that holds the module's manifest is the one indexed.
func (b *Builds) RegistrationsOf(module string) []Registration {
type place struct{ repository, path string }
byPlace := map[place][]delivery.BuildRecord{}
b.mu.Lock()
for _, r := range b.records {
if r.Module == module && registered(r) {
k := place{delivery.RepositoryName(r.Repository), pathOf(strings.Trim(r.Path, "/"))}
byPlace[k] = append(byPlace[k], r)
}
}
b.mu.Unlock()
// Within one place the record is that of the ref a registration comes from, never a newer pull request's or try's: a
// trunk's ref (main, master, trunk, none), else the ref most of the place's records name (the registered photos, de-spiegel
// and link2pay were built from nox-mesh), the newer of two equal; then its newest record.
best := map[place]delivery.BuildRecord{}
for k, recs := range byPlace {
count, latest := map[string]int{}, map[string]delivery.BuildRecord{}
for _, r := range recs {
count[r.Ref]++
if cur, ok := latest[r.Ref]; !ok || newer(r, cur) {
latest[r.Ref] = r
}
}
var pick string
first := true
for ref, rec := range latest {
switch {
case first:
case trunkLike[ref] != trunkLike[pick]:
if !trunkLike[ref] {
continue
}
case count[ref] != count[pick]:
if count[ref] < count[pick] {
continue
}
case !newer(rec, latest[pick]):
continue
}
pick, first = ref, false
}
best[k] = latest[pick]
}
recs := slices.Collect(maps.Values(best))
slices.SortFunc(recs, func(x, y delivery.BuildRecord) int {
switch {
case trunkLike[x.Ref] != trunkLike[y.Ref]:
if trunkLike[x.Ref] {
return -1
}
return 1
case newer(x, y):
return -1
case newer(y, x):
return 1
}
return strings.Compare(x.Repository+x.Path, y.Repository+y.Path)
})
out := make([]Registration, 0, len(recs))
for _, r := range recs {
out = append(out, Registration{Module: r.Module, Repository: r.Repository, Path: r.Path, Commit: r.Commit, Ref: r.Ref})
}
return out
}
// Registered is the registrations of every module a record names, by module name.
func (b *Builds) Registered() []Registration {
b.mu.Lock()
names := map[string]bool{}
for _, r := range b.records {
names[r.Module] = true
}
b.mu.Unlock()
var out []Registration
for _, name := range slices.Sorted(maps.Keys(names)) {
out = append(out, b.RegistrationsOf(name)...)
}
return out
}
@@ -8,8 +8,6 @@ import (
"slices"
"strings"
"sync"
"sync/atomic"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -48,50 +46,22 @@ type Indexer struct {
Modules Store
// Manifests is the manifests bucket; nil writes no manifests (issue 1449).
Manifests Store
// Taken is the taken bucket: per repository the last merge taken in (taken.go, ADR 0556); nil writes none.
Taken Store
// Now is the clock of a taken value; nil is time.Now.
Now func() time.Time
Builds *Builds
Builds *Builds
// Named asks the forge for the git object of each named path of the module directories given, at a commit
// (gitea.gitea_commit_trees with paths; novox/hq ADR 0499 clauses 2 and 3): the indexer is the one reader of a
// manifest's paths (ADR 0495 clause 1, the lead's decision L8 of R4e). Nil asks nothing, and a module naming paths
// has none.
Named func(ctx context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error)
Trunk func(ctx context.Context, repository, sha string) ([]string, error)
// Read asks the forge for a repository's trees at a commit and the manifest and lock-file hashes of the one module directory dir (the
// root spelled ""), as the forge bridge reads them for a merged message (registered.go). Nil indexes no registered module.
Read func(ctx context.Context, repository, sha, dir string) (RepoRead, error)
// Contains says whether a commit is the second or an ancestor of it (gitea.gitea_contains).
Contains func(ctx context.Context, repository, commit, in string) (bool, error)
Logf func(format string, args ...any)
// Placement is the placement bucket (delivery.PlacementBucket, ADR 0549 §2); nil writes no placement.
Placement Store
// PlacementCandidates is the placement candidates bucket (delivery.PlacementCandidatesBucket, issue 1126); nil writes
// no candidate (placement_candidates.go).
PlacementCandidates Store
// ReadPlacement asks the forge for the catalogue's placement now, at the trunk's head and each open pull request's
// (indexer_placement_reread, ADR 0559 §8); nil rereads none.
ReadPlacement func(ctx context.Context) (PlacementRead, error)
// Say says one of the indexer's events (placement-changed); nil says none.
Say func(event string, body any) error
// Drained says whether the indexer has applied every message of the forge stream (the consumer has none pending and none
// unacknowledged); FillTaken writes the source's heads only then. Nil fills none from the source (taken.go).
Drained func(ctx context.Context) (bool, error)
mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach
source sourceNow
previews map[string]delivery.Preview // by previews key (previews.go)
toolchains map[string]map[string]delivery.Toolchain // language, then node (toolchains.go)
tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go)
unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread
retrying atomic.Bool // a RetryUnread runs in the background (kickRetry)
retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one
retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on
recompute sync.Mutex // one Recompute at a time
untaken map[string]string // repository to the latest merge applied while the taken state was missing (taken.go)
filling atomic.Bool // a FillTaken runs in the background (KickFill)
fillAgain atomic.Bool // a kick arrived while it ran
}
func (ix *Indexer) logf(format string, args ...any) {
@@ -115,16 +85,6 @@ func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error {
if err := ix.applyMerged(ctx, f); err != nil {
return err
}
if err := ix.writeTaken(ctx, f); err != nil {
return err
}
ix.kickRetry(ctx) // a registered module the forge could not read is read again on a merge, in the background (issue 1945)
}
if err := ix.applyPlacement(ctx, f); err != nil {
return err
}
if err := ix.applyPlacementCandidates(ctx, f); err != nil {
return err
}
var digests map[string]string
var held map[string]delivery.Module
@@ -180,14 +140,6 @@ func (ix *Indexer) applyOpen(ctx context.Context, f delivery.Forge) error {
for _, p := range f.Open {
named[p] = true
}
if ix.PlacementCandidates != nil {
// A pull request the forge no longer has open keeps no placement candidate (issue 1126).
if err := ix.dropPlacementCandidates(ctx, func(_ string, c delivery.PlacementCandidate) bool {
return !named[delivery.OpenPull{Repository: c.Pull.Repository, Number: c.Pull.Number}]
}, "its pull request is not open"); err != nil {
return err
}
}
keys, err := ix.Store.Keys(ctx)
if err != nil {
return fmt.Errorf("listing the candidates: %w", err)
@@ -17,9 +17,8 @@ const (
// memStore is the candidates bucket's compare-and-set behaviour in memory: a revision per write, a conflict when the
// revision read is no longer the key's. Writes counts the writes that went through; beforeWrite lets a test change the
// key under the indexer; unbound, when set, is every read's error (a state not yet on the bus).
// key under the indexer.
type memStore struct {
unbound error
vals map[string][]byte
revs map[string]uint64
next uint64
@@ -30,9 +29,6 @@ type memStore struct {
func newMem() *memStore { return &memStore{vals: map[string][]byte{}, revs: map[string]uint64{}} }
func (m *memStore) Get(_ context.Context, key string) ([]byte, uint64, bool, error) {
if m.unbound != nil {
return nil, 0, false, m.unbound
}
v, ok := m.vals[key]
return v, m.revs[key], ok, nil
}
@@ -43,8 +43,6 @@ func withManifests(f delivery.Forge, manifests map[string]string, locks map[stri
}
func TestAMergedManifestIsKeptByItsDigestAndNamedByTheModule(t *testing.T) { // 1449/AC1
// A merge's placement files are written as its manifests are (issue 1126): folded here to hold the count.
placementIsWrittenAtAMergeOnlyWhereItChanged(t)
avahi := catalogueManifest(t, "avahi")
ix, mods, manifests := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, nil)
@@ -25,13 +25,6 @@ import (
// A value the module already holds but for its commit is not written again: its commit stays the one its fingerprint first
// appeared at (writeOverlay), so a merge writes only the modules it changes.
func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
return ix.applyManifests(ctx, f, true)
}
// applyManifests is applyMerged; drop says whether the modules of the repository the message holds no manifest for are
// deleted. A message that names only some of a repository's modules (a registered module read on its own, registered.go)
// drops none.
func (ix *Indexer) applyManifests(ctx context.Context, f delivery.Forge, drop bool) error {
digests, err := ix.keepManifests(ctx, f)
if err != nil {
return err
@@ -91,7 +84,7 @@ func (ix *Indexer) applyManifests(ctx context.Context, f delivery.Forge, drop bo
}
// A module of this repository the merge commit holds no module.json for is gone from the trunk, and so is its key
// (issue 1900). A message that carries no manifests (an older writer) or one that cannot be read whole says nothing.
if drop && len(f.Manifests) > 0 && every {
if len(f.Manifests) > 0 && every {
if err := ix.dropRemoved(ctx, f.Repository, held, func(key string, _ delivery.Module) bool { return named[key] }); err != nil {
return err
}
@@ -8,7 +8,6 @@ import (
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
)
const merge = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
@@ -95,92 +94,22 @@ func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value
// writes is the values written by then; commit is each module's commit after.
writes int
commit map[string]string
// taken is the merge the repository's taken key names after then (issue 1280, ADR 0556 clause 1).
taken string
}{
{name: "the same merge again", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: merge,
{name: "the same merge again", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge},
then: func(ix *Indexer) error { return ix.Apply(context.Background(), mergedMsg(repo, three)) }},
{name: "a merge touching one of three modules", writes: 1, commit: map[string]string{"a": next, "b": merge, "c": merge}, taken: next,
{name: "a merge touching one of three modules", writes: 1, commit: map[string]string{"a": next, "b": merge, "c": merge},
then: func(ix *Indexer) error {
m := mergedMsg(repo, map[string]string{"modules/a": "treeA2", "modules/b": "treeB", "modules/c": "treeC"})
m.Merge = next
return ix.Apply(context.Background(), m)
}},
{name: "a merge touching none of them is still taken in", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: next,
then: func(ix *Indexer) error {
m := mergedMsg(repo, three)
m.Merge = next
return ix.Apply(context.Background(), m)
}},
{name: "a merge while the taken state is not on the bus yet is still taken in", writes: 1, commit: map[string]string{"a": next, "b": merge, "c": merge}, taken: merge,
then: func(ix *Indexer) error { // issue 1790: the state is bound when it appears, so the indexer runs before it does
ix.Taken = &memStore{unbound: process.ErrNotYet}
m := mergedMsg(repo, map[string]string{"modules/a": "treeA2", "modules/b": "treeB", "modules/c": "treeC"})
m.Merge = next
return ix.Apply(context.Background(), m)
}},
{name: "a merge heard while the taken state was missing is filled when the state appears", writes: 2, commit: map[string]string{"a": next, "b": merge, "c": merge}, taken: next,
then: func(ix *Indexer) error { // issue 1280: the key was missing, the merge was taken in without it
on := ix.Taken.(*memStore)
key, _ := delivery.RepositoryKey(repo)
delete(on.vals, key)
ix.Taken = &memStore{unbound: process.ErrNotYet}
m := mergedMsg(repo, map[string]string{"modules/a": "treeA2", "modules/b": "treeB", "modules/c": "treeC"})
m.Merge = next
if err := ix.Apply(context.Background(), m); err != nil {
return err
}
if err := ix.ApplySource(context.Background(), 8, sourceOf(repo, next)); err != nil {
return err
}
ix.Taken = on
return ix.FillTaken(context.Background()) // no Drained: only the merge applied is written, not the source's word
}},
{name: "a source head whose merge was not applied is not a taken key", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: "",
then: func(ix *Indexer) error { // the forge stream still holds the merge: the consumer is not drained
delete(ix.Taken.(*memStore).vals, takenKey(repo))
ix.Drained = func(context.Context) (bool, error) { return false, nil }
if err := ix.ApplySource(context.Background(), 8, sourceOf(repo, next)); err != nil {
return err
}
return ix.FillTaken(context.Background())
}},
{name: "a source head of a drained forge stream fills a missing key", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: next,
then: func(ix *Indexer) error {
delete(ix.Taken.(*memStore).vals, takenKey(repo))
ix.Drained = func(context.Context) (bool, error) { return true, nil }
if err := ix.ApplySource(context.Background(), 8, sourceOf(repo, next)); err != nil {
return err
}
return ix.FillTaken(context.Background())
}},
{name: "filling from an older source head leaves a later key", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: next,
then: func(ix *Indexer) error {
ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { return commit == merge && in == next, nil }
m := mergedMsg(repo, three)
m.Merge = next
if err := ix.Apply(context.Background(), m); err != nil {
return err
}
ix.Drained = func(context.Context) (bool, error) { return true, nil }
return ix.FillTaken(context.Background()) // the source still names merge
}},
{name: "an earlier merge heard again after a later one leaves taken", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: merge,
then: func(ix *Indexer) error {
ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { return commit == next && in == merge, nil }
m := mergedMsg(repo, three)
m.Merge = next
return ix.Apply(context.Background(), m)
}},
{name: "a later source revision of another repository's merge", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: merge,
{name: "a later source revision of another repository's merge", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge},
then: func(ix *Indexer) error {
return ix.ApplySource(context.Background(), 9, delivery.Source{Group: "g", Heads: map[string]string{repo: merge, "novox/mesh-catalog": otherMerge}})
}},
} {
t.Run(tc.name, func(t *testing.T) {
ix, mods := modulesIndexer(t)
taken := newMem()
ix.Taken = taken
ctx := context.Background()
if err := ix.Apply(ctx, mergedMsg(repo, three)); err != nil {
t.Fatal(err)
@@ -200,13 +129,6 @@ func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value
t.Errorf("%s = %+v (%v), want commit %.8s", module, m, ok, want)
}
}
key, _ := delivery.RepositoryKey(repo)
if _, held := taken.vals[key]; !held && tc.taken == "" {
return
}
if got, err := delivery.DecodeTaken(taken.vals[key]); err != nil || got.Merge != tc.taken || got.Repository != repo {
t.Errorf("taken.%s = %+v (%v), want merge %.8s", key, got, err, tc.taken)
}
})
}
}
@@ -330,8 +252,3 @@ func TestTwoDirectoriesNamingOneModuleWriteOneAndSayTheOther(t *testing.T) { //
t.Errorf("module = %+v, want the held directory kept", m)
}
}
func takenKey(repo string) string {
key, _ := delivery.RepositoryKey(repo)
return key
}
@@ -1,123 +0,0 @@
package index
import (
"context"
"errors"
"fmt"
"maps"
"slices"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
)
// Placement of modules on nodes (novox/hq ADR 0549 §2, issue 1126): at each merge of the catalogue the forge bridge
// carries every placement file of the merge commit, and the indexer writes each node's key `placement.<node>` in its
// state `placement` only when it differs (the revision alone is no change), deletes the key of a node whose file is
// gone, and says `placement-changed` for each node it wrote or deleted, naming the modules added, removed or changed.
// A merge that changes no placement writes nothing and says nothing.
//
// A file that does not read as a placement, or names another node than its file, is said in the log and its key is left
// as it was: the catalogue's own check refuses such a file before it merges (ADR 0549 §1), so it is a fault of that
// check, and a node keeps composing from the placement it had rather than from none.
// applyPlacement takes the placement files of a merged message in; a message that read none (another repository, an
// older bridge) changes nothing.
func (ix *Indexer) applyPlacement(ctx context.Context, f delivery.Forge) error {
if ix.Placement == nil || f.Kind != delivery.ForgeMerged || !f.PlacementsRead {
return nil
}
keys, err := ix.Placement.Keys(ctx)
if errors.Is(err, process.ErrNotYet) {
// The state is not on the bus yet: said, and the merge's candidates are still taken out (issue 1790's binding
// writes the next catalogue merge's placement once it appears).
ix.logf("%s at %s: its placement files are not written: %v", f.Repository, f.Merge, err)
return nil
}
if err != nil {
return fmt.Errorf("reading the placement keys: %w", err)
}
gone := map[string]bool{}
for _, key := range keys {
if node, err := delivery.ParsePlacementKey(key); err == nil {
gone[node] = true
}
}
var refused []string
for _, node := range slices.Sorted(maps.Keys(f.Placements)) {
delete(gone, node)
next, err := delivery.DecodePlacement([]byte(f.Placements[node]))
if err == nil && next.Node != node {
err = fmt.Errorf("the file of %s names the node %q", node, next.Node)
}
if err != nil {
refused = append(refused, node)
ix.logf("%s at %s: the placement file of %s is refused and placement.%s is left as it was (the catalogue's check should have refused it): %v",
f.Repository, f.Merge, node, node, err)
continue
}
next.Revision = f.Merge
if err := ix.writePlacement(ctx, node, &next); err != nil {
return err
}
}
for _, node := range slices.Sorted(maps.Keys(gone)) {
if err := ix.writePlacement(ctx, node, nil); err != nil {
return err
}
}
if len(refused) > 0 {
// Said, and the merge's candidates are still taken out: a refused file holds nobody's delivery.
ix.logf("%s at %s: the placement files of %v do not read; their keys are left as they were", f.Repository, f.Merge, refused)
}
return nil
}
// writePlacement writes one node's key, or deletes it for next nil, only when it differs, and says what changed.
func (ix *Indexer) writePlacement(ctx context.Context, node string, next *delivery.Placement) error {
key, err := delivery.PlacementKey(node)
if err != nil {
return err
}
var change *delivery.PlacementChange
err = write(ctx, ix.Placement, key, func(value []byte, found bool) ([]byte, bool, error) {
change = nil
var before delivery.Placement
if found {
cur, err := delivery.DecodePlacement(value)
if err != nil {
ix.logf("placement.%s holds what is not a placement, written again: %v", node, err)
} else {
before = cur
}
}
if next == nil {
if !found {
return nil, false, nil
}
c := delivery.ChangeOf(before, delivery.Placement{Node: node})
change = &c
return nil, true, nil
}
if found && delivery.SamePlacement(before, *next) {
return nil, false, nil
}
data, err := delivery.EncodePlacement(*next)
if err != nil {
return nil, false, err
}
c := delivery.ChangeOf(before, *next)
change = &c
return data, false, nil
})
if err != nil || change == nil {
return err
}
if ix.Say != nil {
if err := ix.Say(delivery.PlacementChangedEvent, *change); err != nil {
// The key is written: a reader that watches it hears the change; the event is said loudly as not said.
ix.logf("placement.%s is written, and %s could not be said: %v", node, delivery.PlacementChangedEvent, err)
}
}
return nil
}
@@ -1,302 +0,0 @@
package index
import (
"context"
"errors"
"fmt"
"maps"
"slices"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
)
// Placement candidates (issue 1126; novox/hq ADR 0549 §4: a placement change passes the composition check before any
// node sees it). An updated pull request of the catalogue that touches placement/ carries every placement file at its
// head (the forge bridge, PlacementsRead). For each node whose file at the head differs from the trunk's key
// `placement.<node>` (SamePlacement, the revision left out), or whose file the head removes, the indexer writes the key
// `<group>.<node>` of its state `placement-candidates`; a node whose file is the trunk's has none, and a key the head no
// longer needs is deleted. The key is written only when its value changes, so its write is the node's event to compose
// in shadow and answer. When the pull request closes or merges, or an open message no longer names it, its keys are
// deleted, and so are they when a head the forge says the changes of (DirsKnown) carries no placement: it touches none.
// A pull message that says nothing of its head (a resend of the open pull requests) leaves them as they are.
//
// A file at the head that does not read as a placement, or names another node, is said in the log and that node gets no
// key (one it had is deleted): the catalogue's own check refuses the pull request (ADR 0549 §1), and a node answers no
// candidate that is not one.
// applyPlacementCandidates takes one forge message into the placement candidates.
func (ix *Indexer) applyPlacementCandidates(ctx context.Context, f delivery.Forge) error {
if ix.PlacementCandidates == nil || !delivery.SameRepository(f.Repository, delivery.CatalogueRepository) || trunkLike[f.Branch] {
return nil
}
if f.Kind == delivery.ForgeMerged || f.Action == delivery.PullClosed {
return ix.dropPlacementCandidates(ctx, func(_ string, c delivery.PlacementCandidate) bool {
return c.Group == f.Branch && delivery.SameRepository(c.Pull.Repository, f.Repository) && c.Pull.Number == f.Number
}, fmt.Sprintf("%s#%d %s", f.Repository, f.Number, map[bool]string{true: "merged", false: "closed"}[f.Kind == delivery.ForgeMerged]))
}
if f.Kind != delivery.ForgePull {
return nil
}
if !f.PlacementsRead {
if f.DirsKnown {
// The forge said what this head changes (its directories and paths) and the bridge read no placement: the head
// changes no placement file, so a key an earlier head left goes (ADR 0559 §3).
return ix.dropPlacementCandidates(ctx, func(_ string, c delivery.PlacementCandidate) bool {
return c.Group == f.Branch && delivery.SameRepository(c.Pull.Repository, f.Repository) && c.Pull.Number == f.Number
}, fmt.Sprintf("%s#%d at %s changes no placement file", f.Repository, f.Number, f.Head))
}
return nil
}
trunk, err := ix.trunkPlacements(ctx)
if err != nil {
ix.logf("%s#%d at %s: its placement candidates are not written: %v", f.Repository, f.Number, f.Head, err)
return nil
}
nodes := map[string]bool{}
for node := range f.Placements {
nodes[node] = true
}
for node := range trunk {
nodes[node] = true
}
pull := delivery.Head{Repository: f.Repository, Number: f.Number, Head: f.Head}
for _, node := range slices.Sorted(maps.Keys(nodes)) {
key, err := delivery.PlacementCandidateKey(f.Branch, node)
if err != nil {
return fmt.Errorf("%w: %s#%d: %v", ErrUnreadable, f.Repository, f.Number, err)
}
var next *delivery.PlacementCandidate // nil: no key for this node
raw, atHead := f.Placements[node]
before, inTrunk := trunk[node]
switch {
case atHead:
p, err := delivery.DecodePlacement([]byte(raw))
if err == nil && p.Node != node {
err = fmt.Errorf("the file of %s names the node %q", node, p.Node)
}
if err != nil {
ix.logf("%s#%d at %s: the placement file of %s is refused, so %s has no placement candidate (the catalogue's check refuses the pull request): %v",
f.Repository, f.Number, f.Head, node, node, err)
break
}
if inTrunk && delivery.SamePlacement(before, p) {
break
}
next = &delivery.PlacementCandidate{Group: f.Branch, Pull: pull, Placement: &p}
case inTrunk:
next = &delivery.PlacementCandidate{Group: f.Branch, Pull: pull} // the head removes the node's file
}
if err := ix.writePlacementCandidate(ctx, key, next); err != nil {
return err
}
}
// A node the head and the trunk both no longer name keeps no key of this group.
return ix.dropPlacementCandidates(ctx, func(node string, c delivery.PlacementCandidate) bool {
return c.Group == f.Branch && delivery.SameRepository(c.Pull.Repository, f.Repository) && c.Pull.Number == f.Number && !nodes[node]
}, fmt.Sprintf("%s#%d at %s", f.Repository, f.Number, f.Head))
}
// trunkPlacements is every node's placement in the trunk's state placement, by node.
func (ix *Indexer) trunkPlacements(ctx context.Context) (map[string]delivery.Placement, error) {
out := map[string]delivery.Placement{}
if ix.Placement == nil {
return nil, errors.New("the indexer reads no placement state")
}
keys, err := ix.Placement.Keys(ctx)
if err != nil {
return nil, fmt.Errorf("reading the placement keys: %w", err)
}
for _, key := range keys {
node, err := delivery.ParsePlacementKey(key)
if err != nil {
continue
}
value, _, found, err := ix.Placement.Get(ctx, key)
if err != nil {
return nil, fmt.Errorf("reading %s: %w", key, err)
}
if !found {
continue
}
p, err := delivery.DecodePlacement(value)
if err != nil {
return nil, fmt.Errorf("%s holds what is not a placement: %w", key, err)
}
out[node] = p
}
return out, nil
}
// writePlacementCandidate writes one candidate key, or deletes it for next nil, only when it changes.
func (ix *Indexer) writePlacementCandidate(ctx context.Context, key string, next *delivery.PlacementCandidate) error {
var data []byte
if next != nil {
var err error
if data, err = delivery.EncodePlacementCandidate(*next); err != nil {
return fmt.Errorf("%w: %s.%s: %v", ErrUnreadable, delivery.PlacementCandidatesState, key, err)
}
}
err := write(ctx, ix.PlacementCandidates, key, func(value []byte, found bool) ([]byte, bool, error) {
switch {
case next == nil:
return nil, found, nil
case found && string(value) == string(data):
return nil, false, nil
}
return data, false, nil
})
if errors.Is(err, process.ErrNotYet) {
ix.logf("%s.%s is not written: %v", delivery.PlacementCandidatesState, key, err)
return nil
}
return err
}
// dropPlacementCandidates deletes every candidate key whose node and value gone says, and says each with why.
func (ix *Indexer) dropPlacementCandidates(ctx context.Context, gone func(node string, c delivery.PlacementCandidate) bool, why string) error {
keys, err := ix.PlacementCandidates.Keys(ctx)
if errors.Is(err, process.ErrNotYet) {
return nil
}
if err != nil {
return fmt.Errorf("reading the placement candidate keys: %w", err)
}
for _, key := range keys {
_, node, err := delivery.ParsePlacementCandidateKey(key)
if err != nil {
continue
}
dropped := false
err = write(ctx, ix.PlacementCandidates, key, func(value []byte, found bool) ([]byte, bool, error) {
if !found {
return nil, false, nil
}
c, err := delivery.DecodePlacementCandidate(value)
if err != nil {
ix.logf("%s.%s holds what is not a placement candidate, deleted: %v", delivery.PlacementCandidatesState, key, err)
return nil, true, nil
}
dropped = gone(node, c)
return nil, dropped, nil
})
if err != nil {
return err
}
if dropped {
ix.logf("%s.%s is deleted: %s", delivery.PlacementCandidatesState, key, why)
}
}
return nil
}
// PlacementRead is the catalogue's placement as the forge has it now (indexer_placement_reread, ADR 0559 §8): the trunk's
// head and every placement file at it, by node, and each open pull request into the trunk with its own.
type PlacementRead struct {
Trunk string
Files map[string]string
Pulls []PullPlacement
}
// PullPlacement is one open catalogue pull request and every placement file at its head, by node.
type PullPlacement struct {
Branch string
Number int
Head string
Files map[string]string
}
// PlacementReread is the answer of indexer_placement_reread: the heads read, the keys written or deleted in each state,
// and in words what happened.
type PlacementReread struct {
Trunk string `json:"trunk,omitempty"`
Pulls []string `json:"pulls,omitempty"`
Placement []string `json:"placement"`
Candidates []string `json:"candidates"`
Answer string `json:"answer"`
}
// RereadPlacement reads the catalogue's placement now, at the trunk's head and at the head of each open catalogue pull
// request, and writes placement and placement-candidates as a merged and a pull message would (ADR 0549 §2, ADR 0559
// §3), only where a key differs; a candidate key of a pull request that is no longer open, or whose file no longer
// differs at its head, is deleted. It is the recovery for a forge message the indexer missed (ADR 0559 §8).
func (ix *Indexer) RereadPlacement(ctx context.Context) PlacementReread {
out := PlacementReread{Placement: []string{}, Candidates: []string{}}
if ix.ReadPlacement == nil || ix.Placement == nil || ix.PlacementCandidates == nil {
out.Answer = "the indexer reads no placement here: it has no forge reader or no placement state"
return out
}
read, err := ix.ReadPlacement(ctx)
if err != nil {
out.Answer = "the forge could not say the catalogue's placement, nothing is written: " + err.Error()
return out
}
out.Trunk = read.Trunk
before := revisions(ctx, ix.Placement)
beforeC := revisions(ctx, ix.PlacementCandidates)
if err := ix.applyPlacement(ctx, delivery.Forge{Kind: delivery.ForgeMerged, Repository: delivery.CatalogueRepository, Merge: read.Trunk,
Placements: read.Files, PlacementsRead: true}); err != nil {
out.Answer = "the trunk's placement could not be written: " + err.Error()
return out
}
open := map[int]bool{}
var failed []string
for _, p := range read.Pulls {
open[p.Number] = true
out.Pulls = append(out.Pulls, fmt.Sprintf("%s#%d at %s", delivery.CatalogueRepository, p.Number, p.Head))
f := delivery.Forge{Kind: delivery.ForgePull, Repository: delivery.CatalogueRepository, Branch: p.Branch, Number: p.Number, Head: p.Head,
Action: delivery.PullUpdated, Placements: p.Files, PlacementsRead: true}
if err := ix.applyPlacementCandidates(ctx, f); err != nil {
failed = append(failed, fmt.Sprintf("#%d: %v", p.Number, err))
}
}
if err := ix.dropPlacementCandidates(ctx, func(_ string, c delivery.PlacementCandidate) bool {
return delivery.SameRepository(c.Pull.Repository, delivery.CatalogueRepository) && !open[c.Pull.Number]
}, "its pull request is not open (indexer_placement_reread)"); err != nil {
failed = append(failed, err.Error())
}
out.Placement = changed(before, revisions(ctx, ix.Placement))
out.Candidates = changed(beforeC, revisions(ctx, ix.PlacementCandidates))
switch {
case len(failed) > 0:
out.Answer = "read at the trunk's head " + read.Trunk + "; some candidates are not written: " + fmt.Sprint(failed)
case len(out.Placement)+len(out.Candidates) == 0:
out.Answer = "read at the trunk's head " + read.Trunk + " and every open catalogue pull request's: every key was as the forge has it, nothing written"
default:
out.Answer = fmt.Sprintf("read at the trunk's head %s and every open catalogue pull request's: %d placement keys and %d candidate keys written or deleted",
read.Trunk, len(out.Placement), len(out.Candidates))
}
return out
}
// revisions is every key of a store with its revision; a state not on the bus yet has none.
func revisions(ctx context.Context, st Store) map[string]uint64 {
out := map[string]uint64{}
keys, err := st.Keys(ctx)
if err != nil {
return out
}
for _, key := range keys {
if _, rev, found, err := st.Get(ctx, key); err == nil && found {
out[key] = rev
}
}
return out
}
// changed is every key written, created or deleted between two revisions maps, sorted.
func changed(before, after map[string]uint64) []string {
out := []string{}
for key, rev := range after {
if before[key] != rev {
out = append(out, key)
}
}
for key := range before {
if _, ok := after[key]; !ok {
out = append(out, key)
}
}
slices.Sort(out)
return out
}
@@ -1,201 +0,0 @@
package index
import (
"context"
"reflect"
"slices"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// ADR 0549 §2 (issue 1126): a merge changing one node's file writes that key alone and says one event; a merge with no
// placement change writes nothing and says nothing; a node whose file is gone loses its key and says its modules
// removed; a file that does not read leaves its key; a message that read no placement changes none.
func placementIsWrittenAtAMergeOnlyWhereItChanged(t *testing.T) {
ctx := context.Background()
ix, _, logs := newIndexer(t)
place := newMem()
ix.Placement = place
var said []delivery.PlacementChange
ix.Say = func(event string, body any) error {
if event != delivery.PlacementChangedEvent {
t.Errorf("said %s", event)
}
said = append(said, body.(delivery.PlacementChange))
return nil
}
merged := func(merge string, files map[string]string) delivery.Forge {
return delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: delivery.CatalogueRepository, Branch: "main",
Number: 7, Head: "h", Merge: merge, Root: "r", Trees: map[string]string{"": "t"}, Placements: files, PlacementsRead: true}
}
ace := `{"v":1,"node":"ace","modules":[{"module":"plex"}]}`
g14 := `{"v":1,"node":"g14","modules":[{"module":"zsh"}]}`
written := func(node string) (delivery.Placement, uint64) {
t.Helper()
v, ok := place.vals["placement."+node]
if !ok {
return delivery.Placement{}, 0
}
p, err := delivery.DecodePlacement(v)
if err != nil {
t.Fatal(err)
}
return p, place.revs["placement."+node]
}
if err := ix.Apply(ctx, merged("m1", map[string]string{"ace": ace, "g14": g14})); err != nil {
t.Fatal(err)
}
if p, _ := written("ace"); p.Revision != "m1" || !reflect.DeepEqual(p.Assigned(), []string{"plex"}) || len(said) != 2 {
t.Fatalf("first merge: %+v, said %v", p, said)
}
_, aceRev := written("ace")
said = nil
if err := ix.Apply(ctx, merged("m2", map[string]string{"ace": ace, "g14": g14})); err != nil {
t.Fatal(err)
}
if p, rev := written("ace"); rev != aceRev || p.Revision != "m1" || len(said) != 0 {
t.Errorf("a merge with no placement change wrote %+v (revision %d, was %d) and said %v", p, rev, aceRev, said)
}
said = nil
if err := ix.Apply(ctx, merged("m3", map[string]string{"ace": ace, "g14": `{"v":1,"node":"g14","modules":[{"module":"zsh"},{"module":"xterm"}]}`})); err != nil {
t.Fatal(err)
}
if _, rev := written("ace"); rev != aceRev || len(said) != 1 || said[0].Node != "g14" || said[0].Revision != "m3" ||
!reflect.DeepEqual(said[0].Added, []string{"xterm"}) {
t.Errorf("one file changed: ace's revision %d (was %d), said %+v", rev, aceRev, said)
}
said = nil
if err := ix.Apply(ctx, merged("m4", map[string]string{"ace": ace, "g14": `{"v":1,"node":"ace","modules":[]}`})); err != nil {
t.Fatal(err)
}
if p, _ := written("g14"); len(p.Modules) != 2 || len(said) != 0 || !strings.Contains(logs.String(), "the placement file of %s is refused") {
t.Errorf("a file naming another node replaced the key: %+v, said %v, logs %s", p, said, logs)
}
said = nil
if err := ix.Apply(ctx, merged("m5", map[string]string{"ace": ace})); err != nil {
t.Fatal(err)
}
if _, rev := written("g14"); rev != 0 || len(said) != 1 || !reflect.DeepEqual(said[0].Removed, []string{"xterm", "zsh"}) {
t.Errorf("a file removed: g14's key at revision %d, said %+v", rev, said)
}
said = nil
unread := merged("m6", nil)
unread.PlacementsRead = false
if err := ix.Apply(ctx, unread); err != nil {
t.Fatal(err)
}
if p, _ := written("ace"); p.Revision != "m1" || len(said) != 0 {
t.Errorf("a message that read no placement changed it: %+v, %v", p, said)
}
// Issue 1126 (ADR 0549 §4), placement candidates: the trunk now places ace alone. A catalogue pull request writes
// `<group>.<node>` for each node whose file at its head differs from the trunk's, or that the head removes, and only
// when the value changes; a pull message that read no placement, or a pull request changing none, writes nothing; a
// file that does not read gets no key; a merge, a close or an open message not naming it deletes its keys.
cands := newMem()
ix.PlacementCandidates = cands
said = nil
pullOf := func(branch string, number int, head string, files map[string]string) delivery.Forge {
return delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: delivery.CatalogueRepository, Branch: branch,
Number: number, Head: head, Action: delivery.PullUpdated, Placements: files, PlacementsRead: files != nil}
}
candidate := func(key string) (delivery.PlacementCandidate, uint64, bool) {
t.Helper()
v, ok := cands.vals[key]
if !ok {
return delivery.PlacementCandidate{}, 0, false
}
c, err := delivery.DecodePlacementCandidate(v)
if err != nil {
t.Fatal(err)
}
return c, cands.revs[key], true
}
keys := func() string { k, _ := cands.Keys(ctx); slices.Sort(k); return strings.Join(k, ",") }
g14New := `{"v":1,"node":"g14","modules":[{"module":"zsh"}]}`
for _, row := range []struct {
name string
f delivery.Forge
keys string
}{
{"a pull request adding g14", pullOf("feat/p", 9, h1, map[string]string{"ace": ace, "g14": g14New}), "feat/p.g14"},
{"the same head again", pullOf("feat/p", 9, h1, map[string]string{"ace": ace, "g14": g14New}), "feat/p.g14"},
{"a pull message that read no placement", pullOf("feat/p", 9, h2, nil), "feat/p.g14"},
{"a pull request changing no placement", pullOf("feat/q", 10, h1, map[string]string{"ace": ace}), "feat/p.g14"},
{"a new head removing ace", pullOf("feat/p", 9, h2, map[string]string{"g14": g14New}), "feat/p.ace,feat/p.g14"},
{"a head whose g14 file does not read", pullOf("feat/p", 9, h3, map[string]string{"g14": `{"v":1,"node":"ace","modules":[]}`}), "feat/p.ace"},
{"the head adding g14 again", pullOf("feat/p", 9, h2, map[string]string{"g14": g14New}), "feat/p.ace,feat/p.g14"},
{"a head the forge says changes no placement file", func() delivery.Forge { f := pullOf("feat/p", 9, h3, nil); f.DirsKnown = true; return f }(), ""},
{"the head adding g14 and removing ace again", pullOf("feat/p", 9, h2, map[string]string{"g14": g14New}), "feat/p.ace,feat/p.g14"},
{"an open message not naming it", delivery.Forge{V: delivery.Version, Kind: delivery.ForgeOpen, Open: []delivery.OpenPull{{Repository: delivery.CatalogueRepository, Number: 10}}}, ""},
{"the pull request heard again", pullOf("feat/p", 9, h2, map[string]string{"g14": g14New}), "feat/p.ace,feat/p.g14"},
{"its merge", func() delivery.Forge {
m := merged("m7", map[string]string{"g14": g14New})
m.Branch, m.Number = "feat/p", 9
return m
}(), ""},
} {
_, revBefore, _ := candidate("feat/p.g14")
if err := ix.Apply(ctx, row.f); err != nil {
t.Fatalf("%s: %v", row.name, err)
}
if got := keys(); got != row.keys {
t.Errorf("%s: keys %q, want %q", row.name, got, row.keys)
}
c, rev, ok := candidate("feat/p.g14")
switch row.name {
case "a pull request adding g14":
if !ok || c.Group != "feat/p" || c.Pull != (delivery.Head{Repository: delivery.CatalogueRepository, Number: 9, Head: h1}) || c.Placement == nil ||
!reflect.DeepEqual(c.Placement.Assigned(), []string{"zsh"}) {
t.Errorf("%s: %+v", row.name, c)
}
case "the same head again", "a pull message that read no placement", "a pull request changing no placement":
if rev != revBefore {
t.Errorf("%s: feat/p.g14 written again (revision %d, was %d)", row.name, rev, revBefore)
}
case "a new head removing ace":
if a, _, _ := candidate("feat/p.ace"); a.Placement != nil || a.Pull.Head != h2 || c.Pull.Head != h2 {
t.Errorf("%s: ace %+v, g14 at %s", row.name, a, c.Pull.Head)
}
}
}
if p, _ := written("g14"); len(p.Modules) != 1 || p.Revision != "m7" {
t.Errorf("the merge's trunk placement of g14: %+v", p)
}
// ADR 0559 §8, indexer_placement_reread: a reread with nothing changed writes nothing; one after a missed merge and a
// missed head writes those keys; one after a missed revert deletes the candidate; a forge that cannot say writes nothing.
g14Other := `{"v":1,"node":"g14","modules":[{"module":"zsh"},{"module":"htop"}]}`
for _, row := range []struct {
name string
read PlacementRead
err error
placement, cands []string
}{
{"nothing changed", PlacementRead{Trunk: "m7", Files: map[string]string{"g14": g14New}}, nil, []string{}, []string{}},
{"a missed merge and a missed head", PlacementRead{Trunk: "m8", Files: map[string]string{"g14": g14New, "ace": ace},
Pulls: []PullPlacement{{Branch: "feat/r", Number: 11, Head: h1, Files: map[string]string{"g14": g14Other, "ace": ace}}}}, nil,
[]string{"placement.ace"}, []string{"feat/r.g14"}},
{"a missed revert", PlacementRead{Trunk: "m8", Files: map[string]string{"g14": g14New, "ace": ace},
Pulls: []PullPlacement{{Branch: "feat/r", Number: 11, Head: h2, Files: map[string]string{"g14": g14New, "ace": ace}}}}, nil,
[]string{}, []string{"feat/r.g14"}},
{"a forge that cannot say", PlacementRead{}, context.DeadlineExceeded, []string{}, []string{}},
} {
ix.ReadPlacement = func(context.Context) (PlacementRead, error) { return row.read, row.err }
r := ix.RereadPlacement(ctx)
if !reflect.DeepEqual(r.Placement, row.placement) || !reflect.DeepEqual(r.Candidates, row.cands) || r.Answer == "" {
t.Errorf("%s: %+v", row.name, r)
}
}
if keys() != "" {
t.Errorf("after the reread of a revert, candidates %q", keys())
}
}
@@ -164,7 +164,6 @@ func TestAPreviewHeardBeforeItsPullRequestGivesTheFingerprintWhenTheCandidateIsW
}
func TestAPreviewNamingNoManifestOfTheModulesDirectoryGivesAReason(t *testing.T) {
t.Run("the same preview again", testTheSamePreviewAppliedAgainWritesNothing)
ix, _, cands := previewBed(t, true)
if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil {
t.Fatal(err)
@@ -177,7 +176,7 @@ func TestAPreviewNamingNoManifestOfTheModulesDirectoryGivesAReason(t *testing.T)
}
}
func testTheSamePreviewAppliedAgainWritesNothing(t *testing.T) {
func TestTheSamePreviewAppliedAgainWritesNothing(t *testing.T) {
ix, _, cands := previewBed(t, true)
if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil {
t.Fatal(err)

Some files were not shown because too many files have changed in this diff Show More