Compare commits

..
130 changed files with 652 additions and 13507 deletions
@@ -182,42 +182,6 @@ func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
t.Errorf("without %s the assigner started", missing) t.Errorf("without %s the assigner started", missing)
} }
} }
// ADR 0546: without the puts' bucket it starts all the same, and once the bucket is made (by the controller at its start,
// ADR 0201) a pass opens it, so a put is taken further without a restart.
was := PutsReopenEvery
PutsReopenEvery = 0
defer func() { PutsReopenEvery = was }()
b := startBed(t, delivery.AssignerPutsBucket)
done := b.start(t)
select {
case err := <-done:
t.Fatalf("without %s the assigner stopped: %v", delivery.AssignerPutsBucket, err)
case <-time.After(2 * time.Second):
}
puts, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: delivery.AssignerPutsBucket, History: 5})
if err != nil {
t.Fatal(err)
}
pkey, _ := delivery.PutKey("n1", "web")
asked := delivery.Put{FP: delivery.SourcePrefix2 + strings.Repeat("b", 64), Why: "a try", At: time.Now()}.Said(time.Now(), delivery.PutAsked, "asked")
raw, _ := delivery.EncodePut(asked)
if _, err := puts.Put(b.ctx, pkey, raw); err != nil {
t.Fatal(err)
}
for deadline := time.Now().Add(15 * time.Second); ; time.Sleep(200 * time.Millisecond) {
// A change of a bucket it watches starts a pass, which opens the puts' bucket.
if _, err := b.buckets[delivery.PresenceBucket].Put(b.ctx, "n1", []byte(time.Now().String())); err != nil {
t.Fatal(err)
}
if e, err := puts.Get(b.ctx, pkey); err == nil {
if got, err := delivery.DecodePut(e.Value()); err == nil && got.Step == delivery.PutRefused && strings.Contains(got.Words, "not assigned") {
break
}
}
if time.Now().After(deadline) {
t.Fatal("the put was not taken further once its bucket was made")
}
}
} }
// putJSON writes a value into one of the bed's buckets. // putJSON writes a value into one of the bed's buckets.
@@ -16,7 +16,6 @@ import (
"os/signal" "os/signal"
"strings" "strings"
"syscall" "syscall"
"time"
"github.com/nats-io/nats.go" "github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream" "github.com/nats-io/nats.go/jetstream"
@@ -86,11 +85,11 @@ var keptStates = []announce.State{
// buckets is every bucket the assigner opened, by name. // buckets is every bucket the assigner opened, by name.
type buckets map[string]jetstream.KeyValue type buckets map[string]jetstream.KeyValue
// open opens every bucket the assigner reads and writes; a bucket it cannot open stops the start, named with its grant (issue 1945), // open opens every bucket the assigner reads and writes; a bucket it cannot open stops the start, named, since a pass
// since a pass without it would write wants from half the mesh. The puts' bucket is the exception: whatever keeps it from opening (it // without it would write wants from half the mesh. The puts' bucket is the exception: whatever keeps it from opening (it
// does not exist yet, the credential is not granted it yet, the bus does not answer in time), the assigner writes wants // does not exist yet, the credential is not granted it yet, the bus does not answer in time), the assigner writes wants
// without it and takes no put further until it starts again (ADR 0546). // without it and takes no put further until it starts again (ADR 0546).
func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any)) (buckets, error) { func open(ctx context.Context, js jetstream.JetStream, logf func(string, ...any)) (buckets, error) {
out := buckets{} out := buckets{}
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) { for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
kv, err := js.KeyValue(ctx, name) kv, err := js.KeyValue(ctx, name)
@@ -99,7 +98,7 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
continue continue
} }
if err != nil { if err != nil {
return nil, process.EssentialErr("assigner", name, "write a want from the whole mesh", err) return nil, fmt.Errorf("the bucket %s cannot be opened: %w", name, err)
} }
out[name] = kv out[name] = kv
} }
@@ -214,9 +213,7 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
return return
} }
if e == nil { if e == nil {
if !optional { // the puts never gate the first pass, so never the wants (ADR 0546) settled <- struct{}{}
settled <- struct{}{}
}
continue continue
} }
var value []byte var value []byte
@@ -230,39 +227,15 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
}() }()
return nil return nil
} }
// The puts' watch is not waited for before the first pass: a watch of them that ends before it handed what stands only followed := watched
// stops the puts, never the wants.
if kvs[delivery.AssignerPutsBucket] != nil { if kvs[delivery.AssignerPutsBucket] != nil {
if err := watch(delivery.AssignerPutsBucket); err != nil { if err := watch(delivery.AssignerPutsBucket); err != nil {
logger.Printf("%v: no put is taken further until it is opened again", err) logger.Printf("%v: no put is taken further until the assigner starts again", err)
delete(kvs, delivery.AssignerPutsBucket) delete(kvs, delivery.AssignerPutsBucket)
} else {
followed = append(append([]string{}, watched...), delivery.AssignerPutsBucket)
} }
} }
// The puts' bucket is made by the controller from the manifest's states at its start (ADR 0201), and the assigner's
// credential reads it once the bus's user list carries the grant: neither waits for the assigner. So while it is not open,
// a pass started by a change opens it again, at most once a minute, and from then on the puts are watched and taken
// further, without a restart.
var triedPuts time.Time
reopenPuts := func(p *passer) {
if kvs[delivery.AssignerPutsBucket] != nil || time.Since(triedPuts) < PutsReopenEvery {
return
}
triedPuts = time.Now()
kv, err := js.KeyValue(ctx, delivery.AssignerPutsBucket)
if err != nil {
return
}
kvs[delivery.AssignerPutsBucket] = kv
if err := watch(delivery.AssignerPutsBucket); err != nil {
logger.Printf("%v: no put is taken further until it is opened again", err)
delete(kvs, delivery.AssignerPutsBucket)
return
}
if bus != nil {
p.acts = newPutActs(bus, wake, logger.Printf)
}
logger.Printf("the bucket %s is open: puts are taken further", delivery.AssignerPutsBucket)
}
for _, name := range watched { for _, name := range watched {
var opts []jetstream.WatchOpt var opts []jetstream.WatchOpt
if name == delivery.HaveBucket { if name == delivery.HaveBucket {
@@ -275,7 +248,7 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
if err := watch(delivery.ManifestsBucket, jetstream.MetaOnly()); err != nil { if err := watch(delivery.ManifestsBucket, jetstream.MetaOnly()); err != nil {
return err return err
} }
for range len(watched) + 1 { for range len(followed) + 1 {
select { select {
case <-ctx.Done(): case <-ctx.Done():
return nil return nil
@@ -298,15 +271,11 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
return err return err
case <-settled: case <-settled:
case <-dirty: case <-dirty:
reopenPuts(p)
p.pass(ctx) p.pass(ctx)
} }
} }
} }
// PutsReopenEvery is how often, at most, a pass opens the puts' bucket again while it is not open.
var PutsReopenEvery = time.Minute
// sayVia publishes a condition on the operator-channel seat's one subject for conditions, under the assigner's own name // sayVia publishes a condition on the operator-channel seat's one subject for conditions, under the assigner's own name
// (ADR 0437 clause 9). // (ADR 0437 clause 9).
func sayVia(ctx context.Context, js jetstream.JetStream) func(asks.Condition) error { func sayVia(ctx context.Context, js jetstream.JetStream) func(asks.Condition) error {
@@ -5,18 +5,15 @@ import (
"errors" "errors"
"fmt" "fmt"
"io" "io"
"log"
"os" "os"
"path/filepath" "path/filepath"
"reflect" "reflect"
"slices" "slices"
"strings" "strings"
"testing" "testing"
"time"
"github.com/nats-io/nats.go/jetstream" "github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery"
) )
@@ -73,36 +70,10 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
switch { switch {
case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, "no put is taken further")): case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, "no put is taken further")):
t.Errorf("%s refused with %v: the start answered %v, %d buckets, logged %q", c.missing, c.err, err, len(kvs), logged) t.Errorf("%s refused with %v: the start answered %v, %d buckets, logged %q", c.missing, c.err, err, len(kvs), logged)
case !c.starts && (err == nil || !strings.Contains(err.Error(), "grant of "+c.missing)): // 1945: named with its grant case !c.starts && (err == nil || !strings.Contains(err.Error(), c.missing)):
t.Errorf("%s refused with %v: the start answered %v", c.missing, c.err, err) t.Errorf("%s refused with %v: the start answered %v", c.missing, c.err, err)
} }
} }
// And a watch of the puts that ends before it handed what stands (its subscription closed, its grant withdrawn) does not
// keep the first pass from being made: the wants are written all the same.
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
wrote := make(chan string, 8)
kvs := buckets{}
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
kvs[name] = &watchKV{memKV: &memKV{values: map[string][]byte{}}}
}
ckey, _ := delivery.ComposedNodeKey("ace")
cd, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("d", 64), Judgement: delivery.JudgedUnchanged})
if err != nil {
t.Fatal(err)
}
kvs[delivery.ControllerComposedBucket].(*watchKV).entries = []kvEntry{{key: ckey, value: cd}}
kvs[delivery.AssignerPutsBucket].(*watchKV).closed = true
kvs[delivery.AssignerWantsBucket].(*watchKV).wrote = wrote
go follow(ctx, nil, kvs, func(asks.Condition) error { return nil }, nil, log.New(io.Discard, "", 0))
select {
case key := <-wrote:
if key != "want.ace.node" {
t.Errorf("the first pass wrote %s", key)
}
case <-ctx.Done():
t.Error("a puts watch that ended before its first marker kept the first pass from being made: no want was written")
}
} }
// 1297/B2a: without a credential the program does not start, and says why. // 1297/B2a: without a credential the program does not start, and says why.
@@ -126,47 +97,3 @@ func (r refusingJS) KeyValue(_ context.Context, name string) (jetstream.KeyValue
} }
return &memKV{values: map[string][]byte{}}, nil return &memKV{values: map[string][]byte{}}, nil
} }
// watchKV is a memKV whose watch hands its entries and then the marker that all stands was handed, or ends at once when
// closed; a put on it is said on wrote.
type watchKV struct {
*memKV
entries []kvEntry
closed bool
wrote chan string
}
func (w *watchKV) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
ch := make(chan jetstream.KeyValueEntry, len(w.entries)+1)
if w.closed {
close(ch)
return keyWatcher{ch}, nil
}
for _, e := range w.entries {
ch <- e
}
ch <- nil
return keyWatcher{ch}, nil
}
func (w *watchKV) Put(ctx context.Context, key string, value []byte) (uint64, error) {
if w.wrote != nil {
w.wrote <- key
}
return 1, nil
}
type keyWatcher struct{ ch chan jetstream.KeyValueEntry }
func (k keyWatcher) Updates() <-chan jetstream.KeyValueEntry { return k.ch }
func (k keyWatcher) Stop() error { return nil }
type kvEntry struct {
jetstream.KeyValueEntry
key string
value []byte
}
func (e kvEntry) Key() string { return e.key }
func (e kvEntry) Value() []byte { return e.value }
func (e kvEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut }
@@ -3,7 +3,6 @@ package main
import ( import (
"context" "context"
"errors" "errors"
"slices"
"strings" "strings"
"time" "time"
@@ -36,9 +35,6 @@ type passer struct {
neither map[string]neitherOne neither map[string]neitherOne
// acts are the puts' acts (ADR 0546); nil: the puts are not taken further (no bus to call, or no puts bucket). // acts are the puts' acts (ADR 0546); nil: the puts are not taken further (no bus to call, or no puts bucket).
acts *putActs acts *putActs
// noWant is, by node and module, the reason last said why the node gets no want for the module (issue 1945): a reason is
// said in the journal when it appears or changes, and its clearing when the node has a want again; one line per change.
noWant map[string]string
} }
func (p *passer) clock() time.Time { func (p *passer) clock() time.Time {
@@ -98,7 +94,6 @@ func (p *passer) wants(ctx context.Context) {
} }
wanted := p.w.wantsOf() wanted := p.w.wantsOf()
puts := p.putsByModule() puts := p.putsByModule()
p.staleAsks(ctx, trunk)
names := map[string]bool{} names := map[string]bool{}
for name := range trunk { for name := range trunk {
names[name] = true names[name] = true
@@ -113,8 +108,7 @@ func (p *passer) wants(ctx context.Context) {
// quiet at the router. // quiet at the router.
first := p.neither == nil first := p.neither == nil
stale := map[string]neitherOne{} stale := map[string]neitherOne{}
left := map[string]string{} defer func() { p.sayNeither(neither, stale) }()
defer func() { p.sayNeither(neither, stale, left) }()
for name := range names { for name := range names {
fp := trunk[name].FP2 fp := trunk[name].FP2
name := name name := name
@@ -154,8 +148,6 @@ func (p *passer) wants(ctx context.Context) {
} }
out := assign.Wants(m, nodes, wanted[name], last) out := assign.Wants(m, nodes, wanted[name], last)
gatherNeither(neither, name, out) gatherNeither(neither, name, out)
leftNeither(left, m, wanted[name], out)
p.sayNoWant(name, m.Nodes, wanted[name], out)
if first { if first {
startClears(stale, m, out) startClears(stale, m, out)
if fp != "" && !m.Held && out.Held == nil && !out.Lifted { if fp != "" && !m.Held && out.Held == nil && !out.Lifted {
@@ -194,42 +186,6 @@ func (p *passer) wants(ctx context.Context) {
} }
} }
// sayNoWant says in the journal why a node of module gets no want after this pass, from the rule the outcome names for it
// (issue 1945: a new assignment whose trunk build was not done was held back with nothing said), once per reason, and says
// when the node has a want again. A node with a live want, written before and kept or written in this pass, has one.
func (p *passer) sayNoWant(module string, nodes []string, before map[string]delivery.Want, out assign.Outcome) {
if p.noWant == nil {
p.noWant = map[string]string{}
}
assigned := map[string]bool{}
for _, node := range nodes {
assigned[node] = true
key := node + "." + module
w, written := out.Wants[node]
if !written {
w, written = before[node]
}
reason := ""
if h, held := out.Rules[node]; held && (!written || w.Removed) {
reason = h.Reason()
}
last, said := p.noWant[key]
switch {
case reason != "" && reason != last:
p.logf("%s on %s gets no want: %s", module, node, reason)
p.noWant[key] = reason
case reason == "" && said && written && !w.Removed:
p.logf("%s on %s has a want", module, node)
delete(p.noWant, key)
}
}
for key := range p.noWant {
if node, m, _ := strings.Cut(key, "."); m == module && !assigned[node] {
delete(p.noWant, key) // no longer assigned there: its removed want says so
}
}
}
// neitherOne is a module assigned to a node that holds no want for it after a pass, and the rule holding its first want back. // neitherOne is a module assigned to a node that holds no want for it after a pass, and the rule holding its first want back.
type neitherOne struct { type neitherOne struct {
node, module string node, module string
@@ -243,48 +199,6 @@ func gatherNeither(into map[string]neitherOne, module string, out assign.Outcome
} }
} }
// leftNeither says, by node and module, for every node of m not on neither list after the pass, why: "" when the node has a live
// want for it (written before and kept, or written in this pass), else why it has none. sayNeither clears a neither condition in
// these words, so a clear never says a want the node does not hold (issue 1945: "nats on novox has a want again" while the bus bar
// left it with none).
func leftNeither(into map[string]string, m assign.Module, before map[string]delivery.Want, out assign.Outcome) {
for _, node := range m.Nodes {
if _, on := out.Neither[node]; on {
continue
}
w, written := out.Wants[node]
if !written {
w, written = before[node]
}
why := ""
h, ruled := out.Rules[node]
switch {
case written && !w.Removed:
case (m.Held && !out.Lifted) || out.Held != nil:
why = "it is held, and its held condition says so"
case slices.ContainsFunc(out.Record, func(r assign.RecordNeed) bool { return r.Node == node }):
why = "its move waits for the operator's yes on its record ask"
case putOn(m, node):
why = "its put waits on its own steps and the operator's answer"
case m.FP == "":
why = "the trunk has no build of it"
case ruled:
why = h.Reason()
default:
why = "the node does not run it"
}
into[node+"."+m.Name] = why
}
}
// putOn says whether an operator's put on node owns m's want there: one still on its steps (ADR 0546), so the rules write none
// meanwhile. An ended put (done, refused, released) stays in the bucket and owns nothing.
func putOn(m assign.Module, node string) bool {
p, put := m.Puts[node]
_, owns := assign.PutOwns(p)
return put && owns
}
// startClears adds, for the first pass after a start, every node of m that is not on neither list after the pass: a neither // startClears adds, for the first pass after a start, every node of m that is not on neither list after the pass: a neither
// condition a previous process raised for it is cleared. That covers a node with a want and one with none that is not on neither // condition a previous process raised for it is cleared. That covers a node with a want and one with none that is not on neither
// list, such as a held module (issue 1945: the hold is said on its own, so a neither raised for it before the restart would stay up // list, such as a held module (issue 1945: the hold is said on its own, so a neither raised for it before the restart would stay up
@@ -301,9 +215,7 @@ func startClears(into map[string]neitherOne, m assign.Module, out assign.Outcome
// once, the condition of each one said before that has a want again (issue 1945). A say that fails is made again at the next pass. // once, the condition of each one said before that has a want again (issue 1945). A say that fails is made again at the next pass.
// stale is, on the first pass after a start, the modules whose condition a previous process may have raised and that have a want: // stale is, on the first pass after a start, the modules whose condition a previous process may have raised and that have a want:
// each is cleared once. // each is cleared once.
// left is, by node and module, why each assigned node not on neither list after this pass has no want ("" when it has one); a key func (p *passer) sayNeither(now, stale map[string]neitherOne) {
// it does not hold is no longer assigned.
func (p *passer) sayNeither(now, stale map[string]neitherOne, left map[string]string) {
if p.neither == nil { if p.neither == nil {
p.neither = map[string]neitherOne{} p.neither = map[string]neitherOne{}
for key, one := range stale { for key, one := range stale {
@@ -323,29 +235,21 @@ func (p *passer) sayNeither(now, stale map[string]neitherOne, left map[string]st
continue continue
} }
} }
p.logf("%s is assigned to %s and on neither list: no want, and not in the node's own composition; what holds its first want back: %s", p.logf("%s is assigned to %s and on neither list: no want, and not in the node's own composition; rule %d holds its first want back: %s",
one.module, one.node, one.holding.Reason()) one.module, one.node, one.holding.Rule, one.holding.Words)
p.neither[key] = one p.neither[key] = one
} }
for key, one := range p.neither { for key, one := range p.neither {
if _, still := now[key]; still { if _, still := now[key]; still {
continue continue
} }
why, assigned := left[key]
if !assigned {
why = "it is no longer assigned there"
}
if p.say != nil { if p.say != nil {
if err := p.say(assign.NeitherLeftCondition(one.node, one.module, why)); err != nil { if err := p.say(assign.NeitherCondition(one.node, one.module, assign.Holding{}, false)); err != nil {
p.logf("the condition of %s on neither list on %s could not be cleared, and is cleared at the next pass: %v", one.module, one.node, err) p.logf("the condition of %s on neither list on %s could not be cleared, and is cleared at the next pass: %v", one.module, one.node, err)
continue continue
} }
} }
if why == "" { p.logf("%s on %s has a want again", one.module, one.node)
p.logf("%s on %s has a want again", one.module, one.node)
} else {
p.logf("%s on %s is no longer on neither list and has no want: %s", one.module, one.node, why)
}
delete(p.neither, key) delete(p.neither, key)
} }
} }
@@ -459,98 +363,6 @@ func (p *passer) lift(ctx context.Context, module, fp string) {
} }
p.remove(ctx, delivery.AssignerWantsBucket, held) p.remove(ctx, delivery.AssignerWantsBucket, held)
p.remove(ctx, delivery.AssignerAsksBucket, ask) p.remove(ctx, delivery.AssignerAsksBucket, ask)
// The restore asks of that fingerprint, on every node, go with the hold: every node runs it healthy, so no node waits to be
// set back from it (issue 1906, note 28).
for key := range p.w.read(delivery.AssignerAsksBucket) {
if k, err := delivery.ParseAnyAskKey(key); err == nil && k.Kind == delivery.AskKindRestore && k.Module == module && k.FP == fp {
p.remove(ctx, delivery.AssignerAsksBucket, key)
}
}
}
// staleAsk says whether the hold, restore or record ask at key (read as the SDK reads it) is about something the trunk has left
// behind: a module no longer on the trunk, a fingerprint that is no longer the module's trunk fingerprint, or a composition the
// controller no longer composes for the node at that fingerprint. A backup ask and an ask of another kind are never stale here.
// A module of the trunk whose fingerprint is not read yet, and a composition not read yet, are not stale: nothing known is
// nothing taken away.
func (p *passer) staleAsk(key string, trunk map[string]delivery.Module) bool {
k, err := delivery.ParseAnyAskKey(key)
if err != nil {
return false
}
switch k.Kind {
case delivery.AskKindHold, delivery.AskKindRestore:
m, on := trunk[k.Module]
return !on || (m.FP2 != "" && m.FP2 != k.FP)
case delivery.AskKindRecord:
m, on := trunk[k.Module]
if !on {
return true
}
// The assigner asks about a composition at more than the trunk fingerprint: a node held back by the rules is asked about the
// revised composition of the fingerprint it runs (ADR 0545), and a node's first want at the older fingerprint it runs. So a
// record ask is stale only when its digest is the composition of none of the fingerprints the node can still want (the
// trunk's, its want's, its have's), and only when every one of them is read.
for _, fp := range p.fingerprintsOf(k.Node, k.Module, m.FP2) {
d, ok := p.w.composed(k.Node, k.Module, fp)
if !ok || d.Digest == k.Composed {
return false
}
}
return len(p.fingerprintsOf(k.Node, k.Module, m.FP2)) > 0
}
return false
}
// fingerprintsOf are the fingerprints a node may still want of module: the trunk's, the one its want names and the one its have
// names, each once.
func (p *passer) fingerprintsOf(node, module, trunkFP string) []string {
var out []string
add := func(fp string) {
if fp != "" && !slices.Contains(out, fp) {
out = append(out, fp)
}
}
add(trunkFP)
if key, err := delivery.WantKey(node, module); err == nil {
if w, err := delivery.DecodeWant(p.w.value(delivery.AssignerWantsBucket, key)); err == nil {
add(w.FP)
}
}
if key, err := delivery.HaveKey(node, module); err == nil {
if h, err := delivery.DecodeHave(p.w.value(delivery.HaveBucket, key)); err == nil {
add(h.FP)
}
}
return out
}
// staleAsks takes away, at every pass, each hold, restore and record ask that staleAsk names (issue 1906, note 28): an ask
// outlived its fingerprint, and the teller takes its router ask back from the operator when the key goes. With the trunk not
// read at all (no module), nothing is taken away.
func (p *passer) staleAsks(ctx context.Context, trunk map[string]delivery.Module) {
if len(trunk) == 0 {
return
}
for key := range p.w.read(delivery.AssignerAsksBucket) {
if p.staleAsk(key, trunk) {
// The condition is cleared first and the ask goes only once that was said: the ask is what brings this key back at the
// next pass, so a clear that failed once is made again, with no state of its own (issue 1906, note 29).
k, kerr := delivery.ParseAnyAskKey(key)
held := kerr == nil && (k.Kind == delivery.AskKindHold || k.Kind == delivery.AskKindRestore)
if held && !p.clearHeld(k.Module, k.FP) {
continue
}
p.logf("the ask %s is about a fingerprint or composition the trunk has left behind and is taken away", key)
p.remove(ctx, delivery.AssignerAsksBucket, key)
// The held record stays: it is the memory that this fingerprint failed (ADR 0408 §4), and a revert returns the trunk to
// exactly it. Only the condition, which would else ask for an answer that can no longer be given, is cleared, and it is
// raised again if the trunk returns to the fingerprint.
if held {
delete(p.said, assign.HeldCondition(k.Module, k.FP, "", "", false, false).Key+".raised")
}
}
}
} }
// remove deletes one key of the assigner's own bucket when it holds it; a delete that fails is said and made again at the next // remove deletes one key of the assigner's own bucket when it holds it; a delete that fails is said and made again at the next
@@ -625,18 +437,16 @@ func (p *passer) answeredWith(key, option string) bool {
} }
// clearHeld clears, once, the condition of a hold the operator released. // clearHeld clears, once, the condition of a hold the operator released.
// It says whether nothing is left to say: the clear was said now or before, or there is no router to say it to. func (p *passer) clearHeld(module, fp string) {
func (p *passer) clearHeld(module, fp string) bool {
c := assign.HeldCondition(module, fp, "", "", false, false) c := assign.HeldCondition(module, fp, "", "", false, false)
if p.say == nil || p.said[c.Key+".cleared"] { if p.say == nil || p.said[c.Key+".cleared"] {
return true return
} }
if err := p.say(c); err != nil { if err := p.say(c); err != nil {
p.logf("the condition of the released %s could not be cleared, and is cleared at the next pass: %v", module, err) p.logf("the condition of the released %s could not be cleared, and is cleared at the next pass: %v", module, err)
return false return
} }
p.said[c.Key+".cleared"] = true p.said[c.Key+".cleared"] = true
return true
} }
// recordAsk writes, once, the ask about a move of a module of policy record that waits for the operator's yes. // recordAsk writes, once, the ask about a move of a module of policy record that waits for the operator's yes.
@@ -5,7 +5,6 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"slices"
"strings" "strings"
"testing" "testing"
"time" "time"
@@ -357,37 +356,20 @@ func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) } p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
one := map[string]neitherOne{"g14.screen-lock": {node: "g14", module: "screen-lock", one := map[string]neitherOne{"g14.screen-lock": {node: "g14", module: "screen-lock",
holding: assign.Holding{Rule: delivery.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}}} holding: assign.Holding{Rule: delivery.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}}}
p.sayNeither(one, nil, nil) // the say fails p.sayNeither(one, nil) // the say fails
p.sayNeither(one, nil, nil) p.sayNeither(one, nil)
p.sayNeither(one, nil, nil) p.sayNeither(one, nil)
if len(said) != 1 || said[0].State != asks.StateRaised || said[0].Subject.Machine != "g14" || said[0].Subject.ID != "screen-lock" { if len(said) != 1 || said[0].State != asks.StateRaised || said[0].Subject.Machine != "g14" || said[0].Subject.ID != "screen-lock" {
t.Fatalf("said = %+v, want one raised condition of screen-lock on g14", said) t.Fatalf("said = %+v, want one raised condition of screen-lock on g14", said)
} }
if !strings.Contains(strings.Join(logged, "\n"), "screen-lock") { if !strings.Contains(strings.Join(logged, "\n"), "screen-lock") {
t.Errorf("the journal does not name the module on neither list: %q", logged) t.Errorf("the journal does not name the module on neither list: %q", logged)
} }
wants := map[string]string{"g14.screen-lock": ""} p.sayNeither(map[string]neitherOne{}, nil)
p.sayNeither(map[string]neitherOne{}, nil, wants) p.sayNeither(map[string]neitherOne{}, nil)
p.sayNeither(map[string]neitherOne{}, nil, wants)
if len(said) != 2 || said[1].State != asks.StateCleared || said[1].Key != said[0].Key { if len(said) != 2 || said[1].State != asks.StateCleared || said[1].Key != said[0].Key {
t.Errorf("said = %+v, want the condition cleared once under its key", said) t.Errorf("said = %+v, want the condition cleared once under its key", said)
} }
if last := logged[len(logged)-1]; last != "screen-lock on g14 has a want again" {
t.Errorf("the clear said %q", last)
}
// 1945 (the bus bar): a module that leaves neither list with no want is cleared in words saying so, never "has a want again".
bus := map[string]neitherOne{"novox.nats": {node: "novox", module: "nats", holding: assign.Holding{Bus: true, Words: "the bus moves only by the operator's put"}}}
p.sayNeither(bus, nil, nil)
if c := said[len(said)-1]; c.State != asks.StateRaised || !strings.HasPrefix(c.Explanation, asks.NeedsYou) || !strings.Contains(c.Explanation, "the bus moves only by the operator's put") {
t.Errorf("the bus's neither condition = %+v", c)
}
p.sayNeither(map[string]neitherOne{}, nil, map[string]string{"novox.nats": "it is held, and its held condition says so"})
if c := said[len(said)-1]; c.State != asks.StateCleared || strings.Contains(c.Resolved, "has a want") {
t.Errorf("a module that left neither list with no want was cleared as %+v", c)
}
if last := logged[len(logged)-1]; last != "nats on novox is no longer on neither list and has no want: it is held, and its held condition says so" {
t.Errorf("the clear without a want said %q", last)
}
} }
// 1945/AC21: the pass gathers the modules on neither list of every module's outcome, keyed by node and module. // 1945/AC21: the pass gathers the modules on neither list of every module's outcome, keyed by node and module.
@@ -398,51 +380,6 @@ func TestThePassGathersTheModulesOnNeitherList(t *testing.T) {
if len(got) != 1 || got["g14.screen-lock"].node != "g14" || got["g14.screen-lock"].module != "screen-lock" || got["g14.screen-lock"].holding.Rule != delivery.RuleGrant { if len(got) != 1 || got["g14.screen-lock"].node != "g14" || got["g14.screen-lock"].module != "screen-lock" || got["g14.screen-lock"].holding.Rule != delivery.RuleGrant {
t.Errorf("gathered = %+v", got) t.Errorf("gathered = %+v", got)
} }
// 1945 (the bus bar): a node off neither list is cleared in words that say why it has no want; an operator's put running
// through its steps owns the want, so it is named, never "the node does not run it".
for _, c := range []struct {
name string
puts map[string]delivery.Put
before map[string]delivery.Want
why string
}{
{"a live want", nil, map[string]delivery.Want{"novox": {FP: fpTest}}, ""},
{"a put on its steps", map[string]delivery.Put{"novox": {FP: fpTest, Step: delivery.PutPerson}}, nil, "its put waits on its own steps and the operator's answer"},
{"nothing", nil, nil, "the node does not run it"},
{"an ended put", map[string]delivery.Put{"novox": {FP: fpTest, Step: delivery.PutRefused}}, nil, "the node does not run it"},
} {
left := map[string]string{}
leftNeither(left, assign.Module{Name: "nats", FP: fpTest, Nodes: []string{"novox"}, Puts: c.puts}, c.before, assign.Outcome{})
if why, there := left["novox.nats"]; !there || why != c.why {
t.Errorf("%s: left %q (%v), want %q", c.name, why, there, c.why)
}
}
// 1945 (a first want held back silently): the journal says why a node gets no want when the reason appears or changes, and
// when it has a want again: one line per change, not per pass.
p := newPasser(buckets{}, time.Date(2026, 10, 25, 21, 30, 0, 0, time.UTC))
var logged []string
p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
notDone := assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: delivery.RuleBuildDone, Words: "the build of rofi at 4e7b118e is not done"}}}
wanted := assign.Outcome{Wants: map[string]delivery.Want{"shanks": {FP: fpTest, Composed: "sha256:" + strings.Repeat("c", 64)}}}
for i, c := range []struct {
before map[string]delivery.Want
out assign.Outcome
line string // the line said in this pass, "" for none
}{
{nil, notDone, "rofi on shanks gets no want: rule 1: the build of rofi at 4e7b118e is not done"},
{nil, notDone, ""},
{map[string]delivery.Want{"shanks": {Removed: true}}, notDone, ""},
{nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: delivery.RuleGrant, Words: "no credential"}}}, "rofi on shanks gets no want: rule 6: no credential"},
{nil, wanted, "rofi on shanks has a want"},
{map[string]delivery.Want{"shanks": wanted.Wants["shanks"]}, assign.Outcome{}, ""},
{nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Bus: true, Words: "the bus moves only by the operator's put"}}}, "rofi on shanks gets no want: the bus moves only by the operator's put"},
} {
logged = nil
p.sayNoWant("rofi", []string{"shanks"}, c.before, c.out)
if got := strings.Join(logged, "\n"); got != c.line {
t.Errorf("pass %d said %q, want %q", i, got, c.line)
}
}
} }
// A neither or held condition a previous process raised is cleared by the first pass after a start: a fresh pass remembers // A neither or held condition a previous process raised is cleared by the first pass after a start: a fresh pass remembers
@@ -533,220 +470,3 @@ func TestAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
t.Errorf("after a start: %d raised, %d cleared, want 0 and 1: %+v", r, c, *said) t.Errorf("after a start: %d raised, %d cleared, want 0 and 1: %+v", r, c, *said)
} }
} }
// 1906 note 28: an ask outlives its fingerprint. Each pass takes away the hold, restore and record asks of a module whose
// fingerprint (or composition) is not the trunk's now, or whose module left the trunk; a trunk not read takes nothing away; a
// backup ask is left; and a lift takes the restore asks of its fingerprint on every node.
func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
fpOld, fpNew := delivery.SourcePrefix2+strings.Repeat("a", 64), delivery.SourcePrefix2+strings.Repeat("c", 64)
digOld, digNew := "sha256:"+strings.Repeat("d", 64), "sha256:"+strings.Repeat("e", 64)
trunkOf := func(fp string) map[string]delivery.Module {
return map[string]delivery.Module{"web": {FP2: fp, Manifest: "sha256:" + strings.Repeat("1", 64), Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath}}
}
type ask struct{ name, key string }
holdOld, _ := delivery.HoldAskKey("web", fpOld)
holdNew, _ := delivery.HoldAskKey("web", fpNew)
holdGone, _ := delivery.HoldAskKey("gone", fpOld)
restOld, _ := delivery.RestoreAskKey("novox", "web", fpOld)
restNew, _ := delivery.RestoreAskKey("novox", "web", fpNew)
recOld, _ := delivery.RecordAskKey("novox", "web", digOld)
recNew, _ := delivery.RecordAskKey("novox", "web", digNew)
backOld, _ := delivery.BackupAskKey("novox", "web", fpOld)
all := []string{holdOld, holdNew, holdGone, restOld, restNew, recOld, recNew, backOld}
for _, c := range []struct {
name string
trunk map[string]delivery.Module
gone []string // the keys taken away
noDigit bool // the controller's composition is not read
heldAt string // the node's want is at this fingerprint, whose composition is digOld
}{
{"the trunk moved on", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false, ""},
{"the trunk is at the old fingerprint", trunkOf(fpOld), []string{holdNew, holdGone, restNew, recOld}, false, ""},
{"the trunk's fingerprint is not read yet", trunkOf(""), []string{holdGone}, false, ""},
{"the composition is not read yet", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, true, ""},
{"a node held back at the old fingerprint is still asked about its composition", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, false, fpOld},
{"the trunk is not read at all", nil, nil, false, ""},
} {
t.Run(c.name, func(t *testing.T) {
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
var said []asks.Condition
p.say = func(c asks.Condition) error { said = append(said, c); return nil }
for _, key := range all {
k, _ := delivery.ParseAnyAskKey(key)
var a delivery.AssignerAsk
switch k.Kind {
case delivery.AskKindHold:
a = assign.HoldAsk(key, k.Module, k.FP, "novox", "w", 1, time.Now())
case delivery.AskKindRestore:
a = assign.RestoreAsk(key, k.Node, k.Module, k.FP, "w", time.Now())
case delivery.AskKindRecord:
a = assign.RecordAsk(key, k.Node, k.Module, fpNew, assign.RecordNeed{Node: k.Node, Digest: k.Composed}, time.Now())
default:
a = assign.BackupAsk(key, k.Node, k.Module, k.FP, digOld, "w", 2, time.Now())
}
v, err := delivery.EncodeAssignerAsk(key, a)
if err != nil {
t.Fatal(err)
}
asksKV.values[key] = v
p.w.put(delivery.AssignerAsksBucket, key, v)
}
if !c.noDigit {
ckey, _ := delivery.ComposedKey("novox", "web", fpNew)
v, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digNew, Judgement: delivery.JudgedUnchanged})
p.w.put(delivery.ControllerComposedBucket, ckey, v)
okey, _ := delivery.ComposedKey("novox", "web", fpOld)
old, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digNew, Judgement: delivery.JudgedUnchanged})
if c.heldAt != "" {
old, _ = delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digOld, Judgement: delivery.JudgedUnchanged})
}
p.w.put(delivery.ControllerComposedBucket, okey, old)
}
if c.heldAt != "" {
wkey, _ := delivery.WantKey("novox", "web")
w, _ := delivery.EncodeWant(delivery.Want{FP: c.heldAt, Composed: digOld})
p.w.put(delivery.AssignerWantsBucket, wkey, w)
}
heldOld, _ := delivery.HeldKey("web", fpOld)
wantsKV.values[heldOld] = []byte("x")
p.w.put(delivery.AssignerWantsBucket, heldOld, []byte("x"))
firstAsk, _ := delivery.DecodeAssignerAsk(holdOld, asksKV.values[holdOld])
p.staleAsks(context.Background(), c.trunk)
// the held record and condition of a fingerprint whose hold ask went are taken away with it
heldOld, _ = delivery.HeldKey("web", fpOld)
_, heldThere := wantsKV.values[heldOld]
cleared := slices.ContainsFunc(said, func(c asks.Condition) bool {
return c.Key == assign.HeldCondition("web", fpOld, "", "", false, false).Key && c.State == asks.StateCleared
})
if gone := slices.Contains(c.gone, holdOld); !heldThere || cleared != gone {
t.Errorf("hold ask of the old fingerprint gone = %v: its held record stays = %v (must), its condition cleared = %v", gone, heldThere, cleared)
}
for _, key := range all {
_, there := asksKV.values[key]
want := !slices.Contains(c.gone, key)
if there != want {
t.Errorf("%s: left in the bucket = %v, want %v", key, there, want)
}
}
if slices.Contains(c.gone, holdOld) {
// the trunk returns to the fingerprint (a revert): the held record still holds it, and it is asked and said again
p.now = func() time.Time { return time.Now().Add(time.Hour) }
p.holdAsk(context.Background(), "web", fpOld, nil)
p.heldCondition(context.Background(), "web", fpOld, nil, false)
again, err := delivery.DecodeAssignerAsk(holdOld, asksKV.values[holdOld])
raised := slices.ContainsFunc(said, func(c asks.Condition) bool { return c.State == asks.StateRaised })
if err != nil || again.Ask.ID == firstAsk.Ask.ID || !raised {
t.Errorf("the trunk returned to the held fingerprint: ask %q (first %q, %v), condition raised again = %v", again.Ask.ID, firstAsk.Ask.ID, err, raised)
}
}
})
}
t.Run("a clear that failed to be said is said at the next pass, and the ask goes after it", func(t *testing.T) {
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
var said []asks.Condition
fail := true
p.say = func(c asks.Condition) error {
if fail {
fail = false
return errors.New("router unreachable")
}
said = append(said, c)
return nil
}
a := assign.HoldAsk(holdOld, "web", fpOld, "novox", "w", 1, time.Now())
v, err := delivery.EncodeAssignerAsk(holdOld, a)
if err != nil {
t.Fatal(err)
}
asksKV.values[holdOld] = v
p.w.put(delivery.AssignerAsksBucket, holdOld, v)
p.staleAsks(context.Background(), trunkOf(fpNew))
if _, there := asksKV.values[holdOld]; !there || len(said) != 0 {
t.Fatalf("after a failed clear: ask left = %v, conditions said = %d; want the ask kept and nothing said", there, len(said))
}
p.staleAsks(context.Background(), trunkOf(fpNew))
if _, there := asksKV.values[holdOld]; there || len(said) != 1 || said[0].State != asks.StateCleared {
t.Errorf("at the next pass: ask left = %v, conditions said = %v; want the ask gone and the clear said", there, said)
}
})
t.Run("a lift takes the restore asks of its fingerprint on every node", func(t *testing.T) {
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
keep, _ := delivery.RestoreAskKey("ace", "web", fpNew)
keepOther, _ := delivery.RestoreAskKey("ace", "other", fpOld)
for _, key := range []string{restOld, mustKey(delivery.RestoreAskKey("ace", "web", fpOld)), keep, keepOther} {
asksKV.values[key] = []byte("x")
p.w.put(delivery.AssignerAsksBucket, key, []byte("x"))
}
p.lift(context.Background(), "web", fpOld)
for key, gone := range map[string]bool{restOld: true, mustKey(delivery.RestoreAskKey("ace", "web", fpOld)): true, keep: false, keepOther: false} {
if _, there := asksKV.values[key]; there == gone {
t.Errorf("%s: left in the bucket = %v after the lift", key, there)
}
}
})
}
func mustKey(key string, err error) string {
if err != nil {
panic(err)
}
return key
}
// 1906 note 28, review: a node held back at an earlier fingerprint is asked about the revised composition of that fingerprint
// (ADR 0545). That record ask is not about the trunk's fingerprint: it survives every pass, so the operator is asked, and on a
// yes the want names the revised digest.
func TestARecordAskOfAHeldBackNodeSurvivesPassesAndItsYesMovesTheWant(t *testing.T) {
fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64)
revised, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64)
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
p.logf = t.Logf
put := func(bucket, key string, v []byte) { p.w.put(bucket, key, v) }
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
put(delivery.ModulesBucket, "web", mod) // the trunk's build is not done: rule 1 holds g14 back at fpOld
put(provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-1"}`))
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
put(delivery.PresenceBucket, "g14", on)
have, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
put(delivery.HaveBucket, "g14.web", have)
wkey, _ := delivery.WantKey("g14", "web")
want, _ := delivery.EncodeWant(delivery.Want{FP: fpOld, Composed: was})
wantsKV.values[wkey] = want
put(delivery.AssignerWantsBucket, wkey, want)
ckey, _ := delivery.ComposedKey("g14", "web", fpOld)
cd, cerr := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: revised, Judgement: delivery.JudgedRecreates, Policy: delivery.PolicyRecord, PolicyFrom: "default", Against: was, Words: "recreates"})
if cerr != nil {
t.Fatal(cerr)
}
put(delivery.ControllerComposedBucket, ckey, cd)
tkey, _ := delivery.ComposedKey("g14", "web", fpTest) // the trunk's own composition, another digest than the revised one
tcd, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("f", 64), Judgement: delivery.JudgedUnchanged})
put(delivery.ControllerComposedBucket, tkey, tcd)
akey, _ := delivery.RecordAskKey("g14", "web", revised)
for i := 0; i < 2; i++ {
p.wants(context.Background())
if _, there := asksKV.values[akey]; !there {
t.Fatalf("pass %d: the record ask of the held-back node is not there", i+1)
}
}
a, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
if err != nil {
t.Fatal(err)
}
yes, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a.Ask.ID + "@2026-10-26T01:00:00Z", Words: "the operator"})
put(delivery.AnswersBucket, akey, yes)
p.wants(context.Background())
got, err := delivery.DecodeWant(wantsKV.values[wkey])
if err != nil || got.FP != fpOld || got.Composed != revised {
t.Errorf("after the yes the want is %+v (%v), want it at %s naming %s", got, err, fpOld, revised)
}
if _, there := asksKV.values[akey]; !there {
t.Error("the answered record ask was taken away")
}
}
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
go 1.26.0 go 1.26.0
require ( require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5
github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0 github.com/nats-io/nats.go v1.54.0
) )
-2
View File
@@ -6,8 +6,6 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8 h1:vhYk/iv4
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 h1:gE80bR0tPME7MTMBLcdP62v+F/9soVXmdcBYNMOye44= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 h1:gE80bR0tPME7MTMBLcdP62v+F/9soVXmdcBYNMOye44=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -96,14 +96,6 @@ func NeitherCondition(node, module string, h Holding, raised bool) asks.Conditio
if n := len([]rune(headline)); n > asks.HeadlineLength { if n := len([]rune(headline)); n > asks.HeadlineLength {
headline = string([]rune(headline)[:asks.HeadlineLength]) headline = string([]rune(headline)[:asks.HeadlineLength])
} }
if h.Bus {
needs := fmt.Sprintf("put %s on %s with assigner_put", module, node)
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
Summary: headline, Headline: headline, Needs: needs,
Explanation: fmt.Sprintf("%s%s. %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
"hold it either, so nothing applies it there. The assigner never moves the bus: %s. This clears once the node has a want for it.",
asks.NeedsYou, needs, module, node, h.Words)}
}
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised, return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
Summary: headline, Headline: headline, Summary: headline, Headline: headline,
Explanation: fmt.Sprintf("%s %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+ Explanation: fmt.Sprintf("%s %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
@@ -111,17 +103,6 @@ func NeitherCondition(node, module string, h Holding, raised bool) asks.Conditio
"rule holds, and clears this.", asks.NothingToDo, module, node, h.Rule, h.Words)} "rule holds, and clears this.", asks.NothingToDo, module, node, h.Rule, h.Words)}
} }
// NeitherLeftCondition clears the neither condition of module on node when it is no longer on neither list: why is "" when the
// node has a live want for it again, else why it has none (it is held, its move waits for an ask, the node no longer runs it), so
// the clear never says a want the node does not hold (issue 1945: "nats on novox has a want again" while it had none).
func NeitherLeftCondition(node, module, why string) asks.Condition {
c := NeitherCondition(node, module, Holding{}, false)
if why != "" {
c.Resolved = fmt.Sprintf("%s on %s is no longer on neither list and has no want: %s.", module, node, why)
}
return c
}
// LiftedCondition clears the condition of a held fingerprint the assigner lifted by itself, saying why (issue 1906). // LiftedCondition clears the condition of a held fingerprint the assigner lifted by itself, saying why (issue 1906).
func LiftedCondition(module, fp string) asks.Condition { func LiftedCondition(module, fp string) asks.Condition {
c := HeldCondition(module, fp, "", "", false, false) c := HeldCondition(module, fp, "", "", false, false)
@@ -192,27 +192,10 @@ type RecordNeed struct {
Words string Words string
} }
// Holding is a rule holding a node back, and why. Bus marks the bar on the bus instead of one of the seven rules (Rule is then // Holding is a rule holding a node back, and why.
// 0): the bus moves only by the operator's put (ADR 0539 §15, ADR 0546 §8), never by the assigner (issue 1945: nats on novox,
// running an old-form build, got no want and the pass said nothing).
type Holding struct { type Holding struct {
Rule int Rule int
Words string Words string
Bus bool
}
// Reason is the holding in words: the rule and its words, or for the bus bar its words alone.
func (h Holding) Reason() string {
if h.Bus {
return h.Words
}
return fmt.Sprintf("rule %d: %s", h.Rule, h.Words)
}
// BusBarWords say why the assigner writes no want moving the bus on node: it runs prev, the trunk builds fp, and only the
// operator's put moves it (ADR 0539 §15, ADR 0546 §8). The words reach the operator, so they name no record.
func BusBarWords(node, prev, fp string) string {
return fmt.Sprintf("only your put moves the bus: %s runs %s, the trunk builds %s; put it with assigner_put", node, short(prev), short(fp))
} }
// Wants is the pass of one module m: for each of its nodes, the want by the six rules, starting from the want written // Wants is the pass of one module m: for each of its nodes, the want by the six rules, starting from the want written
@@ -243,15 +226,7 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
} }
before, had := wants[node] before, had := wants[node]
var next delivery.Want var next delivery.Want
p, put := m.Puts[node] if p, put := m.Puts[node]; put {
if !put && had && before.Put && before.Hold {
// The want is the truth (ADR 0546 clause 5): a want marked put and held stands while its put is not heard (the
// puts not open, or not yet handed after a start); only a put heard released gives it back to the rules. A put
// without hold is not kept this way: its put takes it further once heard.
p, put = delivery.Put{FP: before.FP, Composed: before.Composed, Hold: true, Why: strings.TrimPrefix(before.Words, "put: "),
Step: delivery.PutHolding}, true
}
if put {
if want, owns := PutOwns(p); owns { if want, owns := PutOwns(p); owns {
if want != nil { if want != nil {
w := *want w := *want
@@ -320,14 +295,6 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
if need, wait := at.mustAsk(node, asked, hadLive, ran, running.FP, running.Composed == "", d); wait { if need, wait := at.mustAsk(node, asked, hadLive, ran, running.FP, running.Composed == "", d); wait {
if need != nil { if need != nil {
out.Record = append(out.Record, *need) out.Record = append(out.Record, *need)
} else if prev, barred := at.busBar(asked, hadLive, running.FP); barred {
// The bus is barred: no want moves it but the operator's put. Said, never silent (issue 1945): its holding is named,
// and a node that runs it with no live want has it on neither list, so its condition stays raised.
h := Holding{Bus: true, Words: BusBarWords(node, prev, at.FP)}
out.Rules[node] = h
if !hadLive && running.Phase != delivery.PhaseRemoved && !held {
out.Neither[node] = h
}
} }
continue continue
} }
@@ -358,18 +325,8 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
// the pass says so. A new assignment, with no have, waits for the rules and is on no list yet by design. A held module (rule 2, // the pass says so. A new assignment, with no have, waits for the rules and is on no list yet by design. A held module (rule 2,
// held before or by this pass) is on the held list instead, whose hold is said and answered on its own: it is never on neither // held before or by this pass) is on the held list instead, whose hold is said and answered on its own: it is never on neither
// list (issue 1945: systemd-resolved, held on shanks by the operator, was said as urgent every hour). // list (issue 1945: systemd-resolved, held on shanks by the operator, was said as urgent every hour).
h, there := n.Haves[node][m.Name] if h, there := n.Haves[node][m.Name]; there && h.Phase != delivery.PhaseRemoved && !held {
switch {
case there && h.Phase != delivery.PhaseRemoved && !held:
out.Neither[node] = Holding{Rule: rule, Words: words} out.Neither[node] = Holding{Rule: rule, Words: words}
case had && (!there || h.Phase == delivery.PhaseRemoved):
// A removed want of a node that does not run the module says why no want follows it (issue 1945), so the view's
// assigner_want answers it; it stays removed. A node with no key gets none: a removed want written where none was
// would have its node agent remove the module and stop holding its composition's resources back.
next = delivery.Want{Removed: true, Words: NoWantWords(rule, words)}
if !delivery.SameWant(normal(before), normal(next)) {
out.Wants[node] = next
}
} }
continue continue
} }
@@ -405,11 +362,6 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
return out return out
} }
// NoWantWords say why a node gets no want for a module: the rule that holds its first want back and why (issue 1945).
func NoWantWords(rule int, words string) string {
return fmt.Sprintf("no want: rule %d: %s", rule, words)
}
// failed says whether a have's health check failed once the node is done moving: phase running or failed (never removed, held, or // failed says whether a have's health check failed once the node is done moving: phase running or failed (never removed, held, or
// moving). A failed health while it // moving). A failed health while it
// backs up, applies or checks is a step of the move, not its outcome (on novox at 15:27:36 on 2026-10-25, "health failed, phase // backs up, applies or checks is a step of the move, not its outcome (on novox at 15:27:36 on 2026-10-25, "health failed, phase
@@ -491,7 +443,12 @@ func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, runnin
if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == delivery.JudgedUnchanged { if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == delivery.JudgedUnchanged {
return nil, false return nil, false
} }
if _, barred := m.busBar(before, had, running); barred { // The fingerprint the bus ran before: its want's, or with no readable want the one its have names (ADR 0539 §15 bars a new one).
prev := before.FP
if !had {
prev = running
}
if catalogue.ProvidesBus(m.Manifest) && prev != m.FP {
return nil, true // the bus is never moved by the assigner; the operator writes its want (§15) return nil, true // the bus is never moved by the assigner; the operator writes its want (§15)
} }
if !m.recorded(d) || d.Judgement == delivery.JudgedNeither || d.Judgement == delivery.JudgedUnchanged { if !m.recorded(d) || d.Judgement == delivery.JudgedNeither || d.Judgement == delivery.JudgedUnchanged {
@@ -503,16 +460,6 @@ func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, runnin
return &RecordNeed{Node: node, Digest: d.Digest, Judgement: d.Judgement, Words: d.Words}, true return &RecordNeed{Node: node, Digest: d.Digest, Judgement: d.Judgement, Words: d.Words}, true
} }
// busBar says whether m is the bus and the move would take it off the fingerprint it ran before, prev: its want's, or with no
// readable want the one its have names (ADR 0539 §15 bars a new one).
func (m Module) busBar(before delivery.Want, had bool, running string) (prev string, barred bool) {
prev = before.FP
if !had {
prev = running
}
return prev, catalogue.ProvidesBus(m.Manifest) && prev != m.FP
}
// recorded says whether the module is of policy record: as the controller holds it beside the composition's digest, the // recorded says whether the module is of policy record: as the controller holds it beside the composition's digest, the
// operator's own choice included (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it. // operator's own choice included (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it.
func (m Module) recorded(d delivery.ComposedDigest) bool { func (m Module) recorded(d delivery.ComposedDigest) bool {
@@ -397,15 +397,10 @@ func TestTheBusIsNotMovedButItsUserListRevisionIsWrittenAtOnce(t *testing.T) {
{"wanted", delivery.PutWanted, wantedOld(), &delivery.Want{FP: fNew, Composed: dig(fNew), Put: true, Hold: true, Words: "put: the bus's user list"}}, {"wanted", delivery.PutWanted, wantedOld(), &delivery.Want{FP: fNew, Composed: dig(fNew), Put: true, Hold: true, Words: "put: the bus's user list"}},
{"holding, written already", delivery.PutHolding, map[string]delivery.Want{"ace": PutWant(put)}, nil}, {"holding, written already", delivery.PutHolding, map[string]delivery.Want{"ace": PutWant(put)}, nil},
{"released: the rules again", delivery.PutReleased, map[string]delivery.Want{"ace": PutWant(put)}, &delivery.Want{FP: fNew, Composed: dig(fNew)}}, {"released: the rules again", delivery.PutReleased, map[string]delivery.Want{"ace": PutWant(put)}, &delivery.Want{FP: fNew, Composed: dig(fNew)}},
{"not heard: a held want at an older fingerprint stands", "", map[string]delivery.Want{"ace": {FP: fOld, Composed: dig(fOld), Put: true, Hold: true, Words: "put: back"}}, nil},
{"not heard: a held want the rules would rewrite stands", "", map[string]delivery.Want{"ace": PutWant(put)}, nil},
} { } {
p := put p := put
p.Step = c.step p.Step = c.step
m.Puts = map[string]delivery.Put{"ace": p} m.Puts = map[string]delivery.Put{"ace": p}
if c.step == "" {
m.Puts = nil
}
out := Wants(m, n, c.wanted, nil) out := Wants(m, n, c.wanted, nil)
got, wrote := out.Wants["ace"] got, wrote := out.Wants["ace"]
switch { switch {
@@ -427,23 +422,6 @@ func TestTheBusIsNotMovedButItsUserListRevisionIsWrittenAtOnce(t *testing.T) {
t.Errorf("a held bus did not take its user-list revision as a put: %+v", w) t.Errorf("a held bus did not take its user-list revision as a put: %+v", w)
} }
m.Puts = nil m.Puts = nil
// 1945 (the bus bar made loud): a node that runs the bus at an old-form build and holds no want for it gets none, since only
// the operator's put moves the bus, and the pass says so: a holding naming the bar, and the node on neither list so its
// condition stays raised. A node with a live want keeps it and is not on neither list.
m.Composed = composedEverywhere
n.Haves["ace"]["mail"] = delivery.Have{FP: "b-1700000000-oldform", Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
out = Wants(m, n, map[string]delivery.Want{"g14": {FP: fOld, Composed: dig(fOld)}}, nil)
if w, wrote := out.Wants["ace"]; wrote {
t.Errorf("the bus bar wrote ace's want %+v", w)
}
h, on := out.Neither["ace"]
if !on || !h.Bus || out.Rules["ace"] != h || !strings.Contains(h.Words, "only your put moves the bus") || !strings.Contains(h.Words, "ace runs b-1700000000-oldform") ||
!strings.Contains(h.Words, "assigner_put") {
t.Errorf("the barred bus on ace: neither %+v (%v), rules %+v", h, on, out.Rules["ace"])
}
if _, on := out.Neither["g14"]; on || out.Wants["g14"].FP == fNew {
t.Errorf("g14 has a live want: neither %+v, its want %+v", out.Neither, out.Wants["g14"])
}
stepsOfAPut(t) stepsOfAPut(t)
} }
@@ -884,33 +862,6 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) {
if out := Wants(m, n, nil, nil); len(out.Neither) != 1 || out.Neither["g14"].Rule != delivery.RuleBuildDone { if out := Wants(m, n, nil, nil); len(out.Neither) != 1 || out.Neither["g14"].Rule != delivery.RuleBuildDone {
t.Errorf("with no build: neither = %+v, want g14 alone (it runs the module), by rule 1", out.Neither) t.Errorf("with no build: neither = %+v, want g14 alone (it runs the module), by rule 1", out.Neither)
} }
// 1945 (the assigner holds a first want back silently): with no build, a node whose want was removed and that does not run
// the module has the reason written into its removed want, once; a node with no key gets no want written, and a node that
// runs the module is on neither list, its removed want left as it is.
gone := delivery.Want{Removed: true, Words: "screen-lock is no longer assigned to shanks"}
for _, c := range []struct {
name string
before map[string]delivery.Want
want map[string]delivery.Want
}{
{"no key: nothing written", nil, map[string]delivery.Want{}},
{"a removed want: it says why", map[string]delivery.Want{"shanks": gone},
map[string]delivery.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is not done"}}},
{"a removed want that says why already: nothing written", map[string]delivery.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is not done"}},
map[string]delivery.Want{}},
{"a node that runs it: on neither list, its removed want left", map[string]delivery.Want{"g14": gone}, map[string]delivery.Want{}},
} {
out := Wants(m, n, c.before, nil)
if len(out.Wants) != len(c.want) {
t.Errorf("%s: wants = %+v, want %+v", c.name, out.Wants, c.want)
continue
}
for node, w := range c.want {
if !delivery.SameWant(normal(out.Wants[node]), w) {
t.Errorf("%s: want of %s = %+v, want %+v", c.name, node, out.Wants[node], w)
}
}
}
m.FP = "" m.FP = ""
if out := Wants(m, n, nil, nil); len(out.Neither) != 0 { if out := Wants(m, n, nil, nil); len(out.Neither) != 0 {
t.Errorf("a module of the node's own composition named on neither list: %+v", out.Neither) t.Errorf("a module of the node's own composition named on neither list: %+v", out.Neither)
@@ -1,181 +0,0 @@
// Command mesh-delivery-builder-tools serves the builder's tools (novox/hq issue 1945): the runtime launches it for a call.
// builder_stuck reads the builder's builds and reasks states and the build modules' steps through the builder's state and
// reads; builder_ask_again writes one value of the reasks state, which the builder watches and answers. The builder is the
// other program: it serves no tool (ADR 0437 §4).
package main
import (
"context"
"encoding/json"
"fmt"
"os"
"strings"
"sync"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"mesh-delivery-builder/internal/build"
)
// queueAsker writes a queues value and waits for the builder's answer to it.
type queueAsker func(key string, value []byte) (json.RawMessage, error)
// Tools are the builder's tools: stuck reads, askAgain writes the reasks value, queue writes a queues value and waits for
// the builder's read.
func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue queueAsker, now func() time.Time) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
return []stdio.Tool{
{Name: build.StuckTool,
Description: "Every build key that waits (building or queued), oldest first: its module, fingerprint, step words and since when, and each ask it " +
"waits on by language with its id, when it was put, its asking (1 for the first; the builder asks an ask again when it can no longer be " +
"answered, at most 3 times) and what its build module's steps hold of it (none, or its state on its node since its start). The " +
"seat workers' delivery counts are not readable by the builder. dead_letters says whether the builder hears the asks the build seats give up " +
"on (issue 734), and why not: heard, a given-up ask is asked again at once; deaf, at its check bound. With module, that module's keys only. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "only this module's keys"}}},
Run: func(args map[string]any) (any, error) { return r.Stuck(str(args, "module")) }},
{Name: build.AskAgainTool,
Description: "Ask a waiting build key's asks again now, under new ids the key then names, as the builder does by itself for an ask its build " +
"seat's stream gave up on. The builder answers in the reasks state, which " + build.StuckTool + " shows beside the key; a key no ask of " +
"which waits is answered so and nothing is asked. An ask whose step already runs on a node is asked again too: it is then " +
"built twice, the first run's work wasted and its end read by nobody, never a wrong key; builder_stuck shows whether a step runs.",
Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "the module's name"},
"fp": map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>, as builder_stuck shows it"},
"why": map[string]any{"type": "string", "description": "why it is asked again, in plain words"}}},
Run: func(args map[string]any) (any, error) {
key, value, err := build.NewReAsk(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
if err != nil {
return nil, err
}
if err := askAgain(key, value); err != nil {
return nil, fmt.Errorf("the ask could not be written to %s: %w", build.ReAsksBucket, err)
}
return map[string]any{"key": key, "asked": true,
"next": "the builder answers in " + build.ReAsksState + "." + key + "; " + build.StuckTool + " shows the answer and the new ask"}, nil
}},
{Name: build.SeatQueueTool,
Description: "What waits on a build seat's stream (SEAT_NODE_BUILD_<LANGUAGE>), as the builder reads it: the stream's message count and " +
"sequences, and per kind (delivery, check, recheck) its messages oldest first, at most limit each, with sequence, ask id, when asked " +
"and put, age, the state of the step a build node wrote for it, and whether the builder counts it as still waiting or as given up " +
"on (its worker delivered it its most times) or taken; and the ask asked last that a node took. The workers' delivery counts are " +
"not readable by the builder (nats.nats_streams shows them). Read by the builder, which alone may read the stream; read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"language": map[string]any{"type": "string", "description": "go, node or image (default go)"},
"seat": map[string]any{"type": "string", "description": "or the seat: node-build-go, node-build-node, node-build-image"},
"limit": map[string]any{"type": "integer", "description": fmt.Sprintf("messages read of each kind (default %d, at most %d)", build.DefaultQueueLimit, build.MaxQueueLimit)}}},
Run: func(args map[string]any) (any, error) {
language := build.LanguageOf(str(args, "language"))
if language == "" {
language = build.LanguageOf(str(args, "seat"))
}
if language == "" {
language = delivery.AskGo
}
limit, _ := args["limit"].(float64)
key, value, err := build.NewQueueAsk(language, int(limit), stdio.CallerOf(args), now())
if err != nil {
return nil, err
}
return queue(key, value)
}},
}
}
// askQueue writes the queues value at key and waits, on a watch of the key, at most bound for the builder's answer.
func askQueue(s stdio.KeptState, bound time.Duration) queueAsker {
return func(key string, value []byte) (json.RawMessage, error) {
answered := make(chan json.RawMessage, 1)
var rev uint64
var mu sync.Mutex
if err := s.Watch(key, func(c stdio.StateChange) error {
mu.Lock()
mine := rev
mu.Unlock()
if c.Op != "put" || mine == 0 || c.Revision <= mine {
return nil
}
if a, err := build.DecodeQueueAsk(c.Value); err == nil && !a.Answered.IsZero() {
select {
case answered <- c.Value:
default:
}
}
return nil
}); err != nil {
return nil, fmt.Errorf("watching %s.%s: %w", build.QueuesState, key, err)
}
r, err := s.Put(key, json.RawMessage(value))
if err != nil {
return nil, fmt.Errorf("the read could not be written to %s: %w", build.QueuesBucket, err)
}
mu.Lock()
rev = r
mu.Unlock()
// The watch may have heard the answer before the revision was known: read it once too.
if e, err := s.Get(key); err == nil && e != nil && e.Revision > r {
if a, err := build.DecodeQueueAsk(e.Value); err == nil && !a.Answered.IsZero() {
return e.Value, nil
}
}
wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8)
defer cancel()
select {
case v := <-answered:
return v, nil
case <-wait.Done():
return nil, fmt.Errorf("the builder did not answer %s.%s within %s: it is not running, or cannot read %s", build.QueuesState, key, bound, build.QueuesBucket)
}
}
}
func read(s stdio.KeptState) build.StateRead {
return build.StateRead{Keys: s.Keys, Get: func(key string) (json.RawMessage, time.Time, error) {
e, err := s.Get(key)
if err != nil || e == nil {
return nil, time.Time{}, err
}
return e.Value, time.Time{}, nil
}, Written: func(key string) time.Time {
e, err := s.Get(key)
if err != nil || e == nil {
return time.Time{}
}
return written(s, key, e.Revision)
}}
}
// written is when the key's revision was written, from its history, or zero.
func written(s stdio.KeptState, key string, revision uint64) time.Time {
h, err := s.History(key)
if err != nil {
return time.Time{}
}
for _, r := range h {
if r.Revision == revision {
return r.At
}
}
return time.Time{}
}
func main() {
steps := map[string]build.StateRead{}
for _, bm := range build.StuckBuildModules {
steps[bm] = read(stdio.State(bm + ".steps"))
}
reasks := stdio.State(build.ReAsksState)
r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps,
Hearing: read(stdio.State(buildrun.HearingState))}
put := func(key string, value []byte) error {
_, err := reasks.Put(key, json.RawMessage(value))
return err
}
if err := stdio.Serve("", Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), time.Now)); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err)
os.Exit(1)
}
}
@@ -4,303 +4,73 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"sort"
"strings"
"time"
"github.com/nats-io/nats.go/jetstream" "github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-builder/internal/build"
) )
// heldOnTheBus says whether an ask a build key names is on its build seat's stream, or a build node wrote a step for it // heldOnTheBus says whether an ask a build key names is on its build seat's stream, or a build node wrote a step for it
// (issue 1910), read once at the take-over of a key at start; and, at the builder's check bound, whether an ask can still // (issue 1910): read once, at the take-over of a key at start, never polled.
// be answered (issue 1945). Never polled.
type heldOnTheBus struct{ js jetstream.JetStream } type heldOnTheBus struct{ js jetstream.JetStream }
// Held is build.Builder's Held: a step key `<node>.<ask id>` in the build module's steps, or a message of the stream under // Held is build.Builder's Held: a step key `<node>.<ask id>` in the build module's steps, or a message of the stream under
// the ask's subject whose Nats-Msg-Id is the ask's id. // the ask's subject whose Nats-Msg-Id is the ask's id.
func (h heldOnTheBus) Held(ctx context.Context, a delivery.StepAsk) (bool, error) { func (h heldOnTheBus) Held(ctx context.Context, a delivery.StepAsk) (bool, error) {
seen, err := h.seen(ctx, a, false) module, err := delivery.BuildModuleOf(a.Language)
if err != nil { if err != nil {
return false, err return false, err
} }
return seen.Step || seen.OnStream, nil pattern, err := delivery.AskStepWatch(a.ID)
}
// Gone is build.Builder's Gone: why the ask can no longer be answered (build.AskSeen.Why), or "".
func (h heldOnTheBus) Gone(ctx context.Context, a delivery.StepAsk) (string, error) {
seen, err := h.seen(ctx, a, true)
if err != nil { if err != nil {
return "", err return false, err
}
return seen.Why(), nil
}
// maxOnSubject bounds the messages of one subject a read walks (each a read by sequence, next by subject), and maxSteps
// the steps it reads whole to find the last ask taken: past them the read says it cannot tell, and the ask is waited for.
const (
maxOnSubject = 5000
maxSteps = 200
)
// seen reads what the bus holds of the ask; with later, the ask asked last that a build node took too (AskSeen.Last), when
// the ask is on its stream.
func (h heldOnTheBus) seen(ctx context.Context, a delivery.StepAsk, later bool) (build.AskSeen, error) {
seen := build.AskSeen{Kind: a.Kind, At: build.AskedAt(a.ID)}
module, err := delivery.BuildModuleOf(a.Language)
if err != nil {
return seen, err
} }
steps, err := h.js.KeyValue(ctx, delivery.StepsBucket(module)) steps, err := h.js.KeyValue(ctx, delivery.StepsBucket(module))
if err != nil { if err != nil {
return seen, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err) return false, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
}
pattern, err := delivery.AskStepWatch(a.ID)
if err != nil {
return seen, err
} }
keys, err := steps.ListKeysFiltered(ctx, pattern) keys, err := steps.ListKeysFiltered(ctx, pattern)
if err != nil { if err != nil {
return seen, err return false, err
} }
for range keys.Keys() { for range keys.Keys() {
keys.Stop() keys.Stop()
seen.Step = true // a node took it return true, nil // a node took it
return seen, nil
}
stream, name, subject, err := h.stream(ctx, a.Language, a.Kind)
seen.Stream = name
if errors.Is(err, jetstream.ErrStreamNotFound) {
return seen, nil
} }
subject, err := delivery.AskSubject(a.Language, a.Kind)
if err != nil { if err != nil {
return seen, err return false, err
}
var ahead []build.Queued
err = onSubject(ctx, stream, subject, func(q build.Queued) bool {
if q.Ask == a.ID {
seen.OnStream, seen.PutAt = true, q.PutAt
return false
}
ahead = append(ahead, q)
return true
})
if err != nil || !seen.OnStream || !later {
return seen, err
}
taken, err := stepKeys(ctx, steps)
if err != nil {
return seen, err
}
for i := range ahead {
ahead[i].Step = stepState(ctx, steps, taken[ahead[i].Ask])
}
seen.Ahead = ahead
since := seen.At
if since.IsZero() {
since = seen.PutAt
}
for _, q := range ahead {
if q.At.Before(since) {
since = q.At
}
}
seen.Last = lastTaken(ctx, steps, taken, a.Kind, since)
return seen, nil
}
// stream is the build seat's stream of language, its name and the subject of kind on it.
func (h heldOnTheBus) stream(ctx context.Context, language, kind string) (jetstream.Stream, string, string, error) {
subject, err := delivery.AskSubject(language, kind)
if err != nil {
return nil, "", "", err
} }
// The seat's stream by its name (delivery.AskStream), as the broker grants the builder its information and its // The seat's stream by its name (delivery.AskStream), as the broker grants the builder its information and its
// messages by sequence and nothing wider (issue 1910). // messages by sequence and nothing wider (issue 1910).
name, err := delivery.AskStream(language) name, err := delivery.AskStream(a.Language)
if err != nil { if err != nil {
return nil, "", "", err return false, err
} }
stream, err := h.js.Stream(ctx, name) stream, err := h.js.Stream(ctx, name)
return stream, name, subject, err
}
// onSubject hands each message of subject on stream to each, in order, until each answers false or the subject's
// messages end. A seat's stream is a work queue, which takes no second consumer over its subjects: its messages are read
// by sequence instead, each the next of the subject (one read per message, none for the sequences taken between), and
// none consumed.
func onSubject(ctx context.Context, stream jetstream.Stream, subject string, each func(build.Queued) bool) error {
for seq, n := uint64(1), 0; ; n++ {
if n >= maxOnSubject {
return fmt.Errorf("%s holds more than %d messages before the one asked after: not read further", subject, maxOnSubject)
}
m, err := stream.GetMsg(ctx, seq, jetstream.WithGetMsgSubject(subject))
if errors.Is(err, jetstream.ErrMsgNotFound) {
return nil
}
if err != nil {
return err
}
q := build.Queued{Seq: m.Sequence, Ask: m.Header.Get(jetstream.MsgIDHeader), PutAt: m.Time}
if _, kind, err := delivery.ParseAskSubject(m.Subject); err == nil {
q.Kind = kind
}
if q.At = build.AskedAt(q.Ask); q.At.IsZero() {
q.At = q.PutAt
}
if !each(q) {
return nil
}
seq = m.Sequence + 1
}
}
// stepKeys are the build module's step keys by ask id: `<node>.<ask id>` (a node's own key, such as its pause, has no
// ask id and is left out).
func stepKeys(ctx context.Context, steps jetstream.KeyValue) (map[string][]string, error) {
all, err := steps.ListKeys(ctx)
if err != nil {
if errors.Is(err, jetstream.ErrNoKeysFound) {
return map[string][]string{}, nil
}
return nil, err
}
defer all.Stop()
out := map[string][]string{}
for k := range all.Keys() {
if _, id, ok := strings.Cut(k, "."); ok && !build.AskedAt(id).IsZero() {
out[id] = append(out[id], k)
}
}
return out, nil
}
// stepState is the state of the first of keys a node wrote, "" for none.
func stepState(ctx context.Context, steps jetstream.KeyValue, keys []string) string {
for _, k := range keys {
e, err := steps.Get(ctx, k)
if err != nil {
continue
}
if s, err := delivery.DecodeStep(e.Value()); err == nil {
return s.State
}
}
return ""
}
// lastTaken is the ask asked last after since that a build node took, of a kind whose taking proves kind passed
// (build.Proves), read newest first and at most maxSteps of them; nil for none. A step's tool is "<kind> ask"
// (mesh-sdk buildstep).
func lastTaken(ctx context.Context, steps jetstream.KeyValue, taken map[string][]string, kind string, since time.Time) *build.Taken {
var ids []string
for id := range taken {
if build.AskedAt(id).After(since) {
ids = append(ids, id)
}
}
sort.Slice(ids, func(i, j int) bool { return build.AskedAt(ids[i]).After(build.AskedAt(ids[j])) })
read := 0
for _, id := range ids {
for _, k := range taken[id] {
if read >= maxSteps {
return nil
}
read++
e, err := steps.Get(ctx, k)
if err != nil {
continue
}
s, err := delivery.DecodeStep(e.Value())
if err != nil {
continue
}
stepKind, ok := strings.CutSuffix(s.Tool, " ask")
if !ok || !build.Proves(stepKind, kind) {
continue
}
return &build.Taken{Ask: id, Kind: stepKind, At: build.AskedAt(id), Node: s.Node, Started: s.Started}
}
}
return nil
}
// Drop is build.Dropper's Drop: it removes the ask's message from its build seat's stream, found by its Nats-Msg-Id on its
// subject, when the stream still holds it. The builder's account must be granted $JS.API.STREAM.MSG.DELETE.<stream>.
func (h heldOnTheBus) Drop(ctx context.Context, a delivery.StepAsk) error {
stream, name, subject, err := h.stream(ctx, a.Language, a.Kind)
if errors.Is(err, jetstream.ErrStreamNotFound) { if errors.Is(err, jetstream.ErrStreamNotFound) {
return nil return false, nil
} }
if err != nil { if err != nil {
return err return false, err
}
var seq uint64
if err := onSubject(ctx, stream, subject, func(q build.Queued) bool {
if q.Ask == a.ID {
seq = q.Seq
return false
}
return true
}); err != nil || seq == 0 {
return err
}
if err := stream.DeleteMsg(ctx, seq); err != nil && !errors.Is(err, jetstream.ErrMsgNotFound) {
return fmt.Errorf("deleting message %d of %s (is $JS.API.STREAM.MSG.DELETE.%s granted to the builder?): %w", seq, name, name, err)
}
return nil
}
// Queue is what the tool builder_seat_queue answers (build.SeatQueue): the messages of language's build seat's stream, of
// every kind, oldest first and at most limit of each kind, each with its step's state and whether it still waits.
func (h heldOnTheBus) Queue(ctx context.Context, language string, limit int) (build.SeatQueue, error) {
q := build.SeatQueue{Language: language, Note: build.QueueNote}
module, err := delivery.BuildModuleOf(language)
if err != nil {
return q, err
}
stream, name, _, err := h.stream(ctx, language, delivery.AskDelivery)
q.Stream = name
if errors.Is(err, jetstream.ErrStreamNotFound) {
q.Note = name + " does not exist"
return q, nil
}
if err != nil {
return q, err
} }
// A seat's stream is a work queue, which takes no second consumer over its subjects: its messages are read by sequence
// instead, each a direct read that consumes nothing.
info, err := stream.Info(ctx) info, err := stream.Info(ctx)
if err != nil { if err != nil {
return q, err return false, err
} }
q.Messages, q.FirstSeq, q.LastSeq = info.State.Msgs, info.State.FirstSeq, info.State.LastSeq for seq := info.State.FirstSeq; seq != 0 && seq <= info.State.LastSeq; seq++ {
steps, err := h.js.KeyValue(ctx, delivery.StepsBucket(module)) m, err := stream.GetMsg(ctx, seq)
if err != nil { if errors.Is(err, jetstream.ErrMsgNotFound) {
return q, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err) continue // taken and acknowledged, or removed
}
taken, err := stepKeys(ctx, steps)
if err != nil {
return q, err
}
for _, kind := range delivery.AskKinds {
subject, _ := delivery.AskSubject(language, kind)
var ahead []build.Queued
if err := onSubject(ctx, stream, subject, func(m build.Queued) bool {
ahead = append(ahead, m)
return len(ahead) < limit
}); err != nil {
return q, err
} }
if len(ahead) == 0 { if err != nil {
continue return false, err
} }
for i := range ahead { if m.Subject == subject && m.Header.Get(jetstream.MsgIDHeader) == a.ID {
ahead[i].Step = stepState(ctx, steps, taken[ahead[i].Ask]) return true, nil
} }
seen := build.AskSeen{Kind: kind, Ahead: ahead, Last: lastTaken(ctx, steps, taken, kind, ahead[0].At)}
q.Kinds = append(q.Kinds, build.NewKindQueue(kind, seen, time.Now()))
} }
return q, nil return false, nil
} }
@@ -24,8 +24,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-builder/internal/build"
) )
// busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the // busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the
@@ -824,34 +822,6 @@ func TestARestartedBuilderPutsNoSecondAsk(t *testing.T) {
t.Fatalf("asks put = %+v within 20 s, want the two the key names", asked()) t.Fatalf("asks put = %+v within 20 s, want the two the key names", asked())
} }
} }
// 1945 note 62, on the real bus: the stream read as builder_seat_queue reads it shows the Go ask waiting; the ask can
// still be answered (nothing was taken since); and a message dropped as an ask asked again is gone from the stream.
held := heldOnTheBus{b.js}
goAsk := delivery.StepAsk{ID: v.Asks[delivery.AskGo], Language: delivery.AskGo, Kind: delivery.AskDelivery}
stale := delivery.StepAsk{ID: "mvo53jcm70b5f7", Language: delivery.AskGo, Kind: delivery.AskDelivery}
subject, _ := delivery.AskSubject(delivery.AskGo, delivery.AskDelivery)
if _, err := b.js.Publish(b.ctx, subject, []byte("{}"), jetstream.WithMsgID(stale.ID)); err != nil {
t.Fatal(err)
}
queued := func() []build.QueuedState {
q, err := held.Queue(b.ctx, delivery.AskGo, 10)
if err != nil || len(q.Kinds) != 1 || q.Kinds[0].Kind != delivery.AskDelivery {
t.Fatalf("the Go seat's queue = %+v, %v; want its delivery asks", q, err)
}
return q.Kinds[0].Messages
}
if m := queued(); len(m) != 2 || m[0].Ask != goAsk.ID || !m[0].Waits || m[1].Ask != stale.ID {
t.Fatalf("the Go delivery queue = %+v, want the key's ask waiting, then the stale one", m)
}
if why, err := held.Gone(b.ctx, goAsk); why != "" || err != nil {
t.Errorf("Gone(the waiting ask) = %q, %v; want it answerable", why, err)
}
if err := held.Drop(b.ctx, stale); err != nil {
t.Fatal(err)
}
if m := queued(); len(m) != 1 || m[0].Ask != goAsk.ID {
t.Fatalf("after the drop the Go delivery queue = %+v, want the key's ask alone", m)
}
b.stop() b.stop()
b.start(t, "n1", nil) // the same builder, started again, while both asks wait b.start(t, "n1", nil) // the same builder, started again, while both asks wait
b.fake.open() b.fake.open()
@@ -13,6 +13,7 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"io" "io"
"log" "log"
"maps" "maps"
@@ -24,8 +25,6 @@ import (
"syscall" "syscall"
"time" "time"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"github.com/nats-io/nats.go" "github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream" "github.com/nats-io/nats.go/jetstream"
@@ -125,20 +124,29 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil { if err != nil {
return err return err
} }
builds, err := process.EssentialBucket(ctx, js, "builder", delivery.BuilderBuildsBucket, "claim or write a build key") bucket := func(name string) (jetstream.KeyValue, error) {
kv, err := js.KeyValue(ctx, name)
if err != nil {
return nil, fmt.Errorf("the bucket %s: %w", name, err)
}
return kv, nil
}
builds, err := bucket(delivery.BuilderBuildsBucket)
if err != nil { if err != nil {
return err return err
} }
modules, err := process.EssentialBucket(ctx, js, "builder", delivery.ModulesBucket, "know any fingerprint to build") modules, err := bucket(delivery.ModulesBucket)
if err != nil { if err != nil {
return err return err
} }
manifests, err := process.EssentialBucket(ctx, js, "builder", delivery.ManifestsBucket, "read what a module builds") manifests, err := bucket(delivery.ManifestsBucket)
if err != nil {
return err
}
candidates, err := bucket(delivery.CandidatesBucket)
if err != nil { if err != nil {
return err return err
} }
// The candidates' fingerprints are built too (issue 1684): a builder that cannot read them builds the trunk's.
candidates := process.OptionalBucket(ctx, js, delivery.CandidatesBucket, "no candidate's fingerprint is built until the builder starts again; the trunk's are", logger.Printf)
bound := defaultStepBound bound := defaultStepBound
if v := strings.TrimSpace(getenv("MESH_BUILDER_STEP_BOUND")); v != "" { if v := strings.TrimSpace(getenv("MESH_BUILDER_STEP_BOUND")); v != "" {
if bound, err = time.ParseDuration(v); err != nil { if bound, err = time.ParseDuration(v); err != nil {
@@ -149,18 +157,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// Each build module's part of a build is an ask on its build seat's stream, pulled by a build node with a free slot // Each build module's part of a build is an ask on its build seat's stream, pulled by a build node with a free slot
// (novox/hq ADR 0525): the builder chooses no node, so it reads neither presence nor have and holds no capacity. // (novox/hq ADR 0525): the builder chooses no node, so it reads neither presence nor have and holds no capacity.
asks := buildrun.JetStreamAsks{JS: js} asks := buildrun.JetStreamAsks{JS: js}
// The asks the build seats gave up on (issue 734): heard from DEAD_LETTERS, each ends the wait for its step at once,
// and the lane asks its part again (Builder.waitOnce) instead of waiting for the check bound.
// Heard in the background, and again at a reconnect or an ask while it is not; whether it is heard is kept in the
// state hearing, which builder_stuck says (dead_letters).
dead := &buildrun.DeadLetters{}
// The state may not be on the bus yet when a delivery started the builder (the grants part creates it, issue 1790): it
// is bound when the bus says it was created, and the latest hearing is written then.
hearingKV := process.AwaitBucket(ctx, js, conn, delivery.BuilderModule+"_"+buildrun.HearingState, "builder_stuck cannot say whether DEAD_LETTERS is heard", logger.Printf)
hearing := buildrun.NewDeadLetterHearing(ctx, buildrun.JetStreamDeadLetters{JS: js}, delivery.AskLanguages, dead, logger.Printf,
buildrun.SayHearing(ctx, hearingKV, logger.Printf))
defer hearing.Stop()
conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again() })
// The builder asks the controller, as its own account, whether the artifact store still holds a done key's blobs // The builder asks the controller, as its own account, whether the artifact store still holds a done key's blobs
// (the seat verb held, novox/hq issue 1937) before it leaves the key done. // (the seat verb held, novox/hq issue 1937) before it leaves the key done.
account, err := process.ReadAccount(path) account, err := process.ReadAccount(path)
@@ -173,36 +169,44 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
} }
defer calls.Close() defer calls.Close()
b := &build.Builder{ b := &build.Builder{
Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module}, Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
Asks: buildrun.Asker{Publisher: asks, Waiter: buildrun.LostAskWaiter{Inner: asks, Dead: dead, Hearing: hearing}, Bound: bound}, Asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: bound},
Held: heldOnTheBus{js}, Held: heldOnTheBus{js},
// An ask that can no longer be answered is asked again (issue 1945), read at the step bound: a context's bound.
Gone: heldOnTheBus{js},
CheckAfter: bound,
Toolchains: tools, Toolchains: tools,
Manifests: manifestReader{manifests}, Manifests: manifestReader{manifests},
Logf: logger.Printf, Logf: logger.Printf,
Stored: storedVia(calls.Call), Stored: storedVia(calls.Call),
} }
// A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base // A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base
// revision (the gate's source state). The source's revisions are watched with their history (a direct get of one revision // revision (the gate's source state). Where the bucket is not there yet no candidate's fingerprint is built, and said.
// is not what a read grants): a revision heard after the candidate that names it builds that candidate then. Where it // The source's revisions are watched with their history (a direct get of one revision is not what a read grants): a
// cannot be read the trunk's keys are built and the candidates wait (issue 1945). // revision heard after the candidate that names it builds that candidate then.
errs := make(chan error, 6) trunks := &sourceHeads{}
trunks := &trunkSource{ctx: ctx, js: js, heads: &sourceHeads{}, moved: b.TrunkMoved, errs: errs, logf: logger.Printf} var sw jetstream.KeyWatcher
trunks.start() switch kv, err := js.KeyValue(ctx, delivery.SourceBucket); {
b.Trunk = trunks.head case errors.Is(err, jetstream.ErrBucketNotFound):
b.Said = func(e build.AskedAgain) { logger.Printf("the bucket %s does not exist yet: no candidate's fingerprint is built until the builder starts again", delivery.SourceBucket)
if err := emitAskedAgain(context.WithoutCancel(ctx), js, cred, e); err != nil { case err != nil:
logger.Printf("%v", err) return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err)
default:
w, err := kv.Watch(ctx, delivery.SourceMainKey, jetstream.IncludeHistory())
if err != nil {
return fmt.Errorf("the bucket %s cannot be watched: %w", delivery.SourceBucket, err)
} }
if sw, err = settled(ctx, w, func(e jetstream.KeyValueEntry) { trunks.put(e) }); err != nil {
return err
}
b.Trunk = trunks.head
} }
// **Said once every build module answered a reference, taken away when stopped** (issue 1886, ADR 0522): the // **Said once every build module answered a reference, taken away when stopped** (issue 1886, ADR 0522): the
// controller delivers from this builder's keys only while an instance says it writes references. A bucket not there // controller delivers from this builder's keys only while an instance says it writes references. A bucket not there
// yet (the manifest's state not made) is said, and the controller's plan waits for the keys, named (no build-agent // yet (the manifest's state not made) is said, and the controller's plan waits for the keys, named (no build-agent
// is asked since R4g, ADR 0524 §7). // is asked since R4g, ADR 0524 §7).
switch kv := process.OptionalBucket(ctx, js, instancesBucket, "the controller is not told that this builder writes references until it starts again", logger.Printf); { switch kv, err := js.KeyValue(ctx, instancesBucket); {
case kv == nil: case errors.Is(err, jetstream.ErrBucketNotFound):
logger.Printf("the bucket %s does not exist yet: the controller is not told that this builder writes references until it starts again", instancesBucket)
case err != nil:
return fmt.Errorf("the bucket %s: %w", instancesBucket, err)
default: default:
says := newSayer(ctx, kv, cred.Node, []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, build.ImageBuildModule}, logger.Printf) says := newSayer(ctx, kv, cred.Node, []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, build.ImageBuildModule}, logger.Printf)
b.Answered = says.Answered b.Answered = says.Answered
@@ -282,37 +286,23 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
b.ForgetCandidate(e.Key()) b.ForgetCandidate(e.Key())
} }
} }
if candidates != nil { cw, err := settle(ctx, candidates, eachCandidate)
cw, err := settle(ctx, candidates, eachCandidate) if err != nil {
if err != nil { return err
process.NotWatched(logger.Printf, delivery.CandidatesBucket, err, "no candidate's fingerprint is built until the builder starts again")
} else {
go func() { errs <- follow(ctx, cw, eachCandidate) }()
}
} }
// A person asks a key's build again with builder_ask_again, which writes the reasks state (issue 1945): watched, as
// every other input. The builder asks again by itself without it.
// A state not on the bus yet at this start is watched once it appears (issue 1790).
process.AwaitBucket(ctx, js, conn, build.ReAsksBucket, "builder_ask_again is not heard; an ask that can no longer be answered is still asked again by itself", logger.Printf).Ready(func(kv jetstream.KeyValue) {
eachReAsk := func(e jetstream.KeyValueEntry) { reAsked(ctx, b, kv, e, logger.Printf) }
if rw, err := settle(ctx, kv, eachReAsk); err != nil {
process.NotWatched(logger.Printf, build.ReAsksBucket, err, "builder_ask_again is not heard until the builder starts again")
} else {
go func() { errs <- follow(ctx, rw, eachReAsk) }()
}
})
// A person reads a build seat's stream with builder_seat_queue, which writes the queues state (issue 1945 note 62): the
// builder alone may read the stream, so it answers there.
process.AwaitBucket(ctx, js, conn, build.QueuesBucket, "builder_seat_queue is not answered", logger.Printf).Ready(func(kv jetstream.KeyValue) {
eachRead := func(e jetstream.KeyValueEntry) { queueRead(ctx, heldOnTheBus{js}, kv, e, logger.Printf) }
if qw, err := settle(ctx, kv, eachRead); err != nil {
process.NotWatched(logger.Printf, build.QueuesBucket, err, "builder_seat_queue is not answered until the builder starts again")
} else {
go func() { errs <- follow(ctx, qw, eachRead) }()
}
})
logger.Printf("started on %s", cred.Node) logger.Printf("started on %s", cred.Node)
errs := make(chan error, 3)
go func() { errs <- follow(ctx, w, each) }() go func() { errs <- follow(ctx, w, each) }()
go func() { errs <- follow(ctx, cw, eachCandidate) }()
if sw != nil {
go func() {
errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) {
if trunks.put(e) {
b.TrunkMoved(ctx)
}
})
}()
}
select { select {
case <-ctx.Done(): case <-ctx.Done():
b.Wait() b.Wait()
@@ -1,78 +0,0 @@
package main
import (
"context"
"fmt"
"sync"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// **A part does not stop for a bucket it can work without** (mesh-sdk process.EssentialBucket, OptionalBucket) (novox/hq issue 1945, met twice on 2026-10-25: the builder of
// mesh-delivery #48 exited on a refused read of the gate's source, and its reasks bucket would have done the same). A grant
// reaches the bus with a delivery, so a bucket the bus refuses, or times out on, at start is often only not granted yet:
// exiting for it stops every build until a person's hand. A bucket the builder cannot build at all without (its builds,
// the indexer's modules and manifests) still stops the start, naming the grant to look at; every other is said and left,
// and opened again at the next event that needs it where there is one.
// trunkSource is the gate's source history the candidates' previews are merged onto (sourceHeads), opened at start and,
// while it cannot be, again at the next candidate that asks a trunk head: a builder without it builds the trunk's keys,
// and a candidate's preview waits, saying why.
type trunkSource struct {
ctx context.Context
js bucketOpener
heads *sourceHeads
moved func(context.Context)
errs chan<- error
logf func(string, ...any)
mu sync.Mutex
opened bool
}
// ensure opens and settles the source's watch once, and follows it; an error leaves it closed for the next ask.
func (t *trunkSource) ensure() error {
t.mu.Lock()
defer t.mu.Unlock()
if t.opened {
return nil
}
kv, err := t.js.KeyValue(t.ctx, delivery.SourceBucket)
if err != nil {
return fmt.Errorf("the bucket %s cannot be read (%v)", delivery.SourceBucket, err)
}
w, err := kv.Watch(t.ctx, delivery.SourceMainKey, jetstream.IncludeHistory())
if err != nil {
return fmt.Errorf("the bucket %s cannot be watched (%v)", delivery.SourceBucket, err)
}
sw, err := settled(t.ctx, w, func(e jetstream.KeyValueEntry) { t.heads.put(e) })
if err != nil {
return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err)
}
t.opened = true
go func() {
t.errs <- follow(t.ctx, sw, func(e jetstream.KeyValueEntry) {
if t.heads.put(e) && t.moved != nil {
t.moved(t.ctx)
}
})
}()
return nil
}
// start opens the source at start; one that cannot be is said, and the builder goes on.
func (t *trunkSource) start() {
if err := t.ensure(); err != nil {
t.logf("%v: the trunk's keys are built, and a candidate's preview waits until the source can be read, tried again at its next candidate", err)
}
}
// head is Builder.Trunk: the head of repository at source revision rev, opening the source first if it is not yet.
func (t *trunkSource) head(ctx context.Context, rev uint64, repository string) (string, error) {
if err := t.ensure(); err != nil {
return "", err
}
return t.heads.head(ctx, rev, repository)
}
@@ -1,49 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"time"
"github.com/nats-io/nats.go/jetstream"
"mesh-delivery-builder/internal/build"
)
// queueReader is the part of the builder a queues value reads through: heldOnTheBus.
type queueReader interface {
Queue(ctx context.Context, language string, limit int) (build.SeatQueue, error)
}
// queueRead answers one value of the queues state (the tool builder_seat_queue, issue 1945 note 62): a value not yet
// answered is read now, from the stream the builder's account may read, and written back by compare-and-set at its
// revision with what was read or why it could not be. A value answered already (this builder's own write, or another
// instance's) or deleted is left.
func queueRead(ctx context.Context, r queueReader, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) {
if e.Operation() != jetstream.KeyValuePut {
return
}
a, err := build.DecodeQueueAsk(e.Value())
if err != nil {
logf("%s.%s holds what is not a queue read, left: %v", build.QueuesState, e.Key(), err)
return
}
if !a.Answered.IsZero() {
return
}
read, cancel := context.WithTimeout(ctx, 20*time.Second)
q, err := r.Queue(read, e.Key(), max(1, min(a.Limit, build.MaxQueueLimit)))
cancel()
if err != nil {
a.Error = err.Error()
}
a.Queue, a.Answered = &q, time.Now().UTC()
data, err := json.Marshal(a)
if err != nil {
return
}
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
logf("%s.%s: the read could not be written: %v", build.QueuesState, e.Key(), err)
}
}
@@ -1,75 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"mesh-delivery-builder/internal/build"
)
// askAgainer is the part of the builder a reasks value asks through.
type askAgainer interface {
AskAgain(key, why string) []string
}
// reAsked answers one value of the reasks state (the tool builder_ask_again, issue 1945): a value not yet answered asks the
// key's waiting lanes again and is written back, by compare-and-set at its revision, with what the builder did. A value
// answered already (this builder's own write) or deleted is left.
func reAsked(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) {
if e.Operation() != jetstream.KeyValuePut {
return
}
r, err := build.DecodeReAsk(e.Value())
if err != nil {
logf("%s.%s holds what is not an ask to ask again, left: %v", build.ReAsksState, e.Key(), err)
return
}
if r.Answer != "" {
return
}
asked := b.AskAgain(e.Key(), r.Why)
if len(asked) == 0 {
r.Answer = "no ask of this key waits in this builder now (it is not building, or its build has not reached its ask yet): nothing was asked; builder_stuck shows where it stands"
} else {
r.Answer = "asked again: the " + strings.Join(asked, ", ") + " ask, under a new id the key names"
}
r.Answered = time.Now().UTC()
logf("%s: asked again by %s (%s): %s", e.Key(), r.By, r.Why, r.Answer)
data, err := json.Marshal(r)
if err != nil {
return
}
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err)
}
}
// emitAskedAgain says the builder's event build.asked-again (issue 1945): mesh.mod.<module>.event.build.asked-again,
// with the SDK's envelope headers and the new ask's id as the bus's dedup id.
func emitAskedAgain(ctx context.Context, js jetstream.JetStream, cred credential, e build.AskedAgain) error {
data, err := json.Marshal(e)
if err != nil {
return err
}
msg := nats.NewMsg("mesh.mod." + cred.Module + ".event." + build.AskedAgainEvent)
id := "asked-again-" + e.New
msg.Header.Set("x-event-id", id)
msg.Header.Set("x-source", cred.Module)
msg.Header.Set("x-node", cred.Node)
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
msg.Header.Set("content-type", "application/json")
msg.Data = data
pub, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
if _, err := js.PublishMsg(pub, msg, jetstream.WithMsgID(id)); err != nil {
return fmt.Errorf("saying %s for %s: %w", build.AskedAgainEvent, e.Key, err)
}
return nil
}
@@ -193,9 +193,7 @@ func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate,
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) } each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
w, err := settle(ctx, kv, each) w, err := settle(ctx, kv, each)
if err != nil { if err != nil {
// Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read. return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err)
logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
return errs, nil
} }
if gate.Settled() { if gate.Settled() {
changed(ctx) changed(ctx)
@@ -3,19 +3,11 @@ package main
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940). // The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
import ( import (
"bytes"
"context"
"errors"
"fmt" "fmt"
"log"
"strings" "strings"
"testing" "testing"
"time" "time"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
) )
@@ -97,61 +89,6 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
if n := len(set.Entries("typescript")); n != 2 { if n := len(set.Entries("typescript")); n != 2 {
t.Fatalf("novox and shanks are left; got %d entries", n) t.Fatalf("novox and shanks are left; got %d entries", n)
} }
// 1945 (a part exits at start for a bucket it works without): a bus that refuses the assignments' watch, or every
// toolchains bucket, leaves the watches unstarted and the part running, said; a refused bucket it cannot work without
// is an error naming the bucket and its grant.
var logs bytes.Buffer
logger := log.New(&logs, "", 0)
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments")}
if _, err := watchAssignments(context.Background(), refused, newToolchainGate(nil, set.Put, t.Logf), func(context.Context) {}, logger); err != nil {
t.Errorf("a refused watch of the assignments stopped the start: %v", err)
}
if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil {
t.Errorf("a refused toolchains bucket stopped the start: %v", err)
}
if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") {
t.Errorf("the refusals were not said: %q", logs.String())
}
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil ||
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
}
// The gate's source (the previews' trunk heads): refused at start, the builder goes on, and a candidate asking a trunk
// head is told why and opens it again then.
src := &trunkSource{ctx: context.Background(), js: refusingBus{refuseOpen: true, err: refused.err}, heads: &sourceHeads{}, errs: make(chan error, 1), logf: logger.Printf}
src.start()
if _, err := src.head(context.Background(), 7, "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), delivery.SourceBucket) {
t.Errorf("a trunk head with the source refused: %v, want why", err)
}
if src.opened {
t.Error("a refused source counts as opened, so it is never opened again")
}
}
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
type refusingBus struct {
refuseOpen bool
err error
}
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
if r.refuseOpen {
return nil, r.err
}
return refusedWatch{err: r.err}, nil
}
type refusedWatch struct {
jetstream.KeyValue
err error
}
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
return nil, r.err
}
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
return nil, r.err
} }
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not // A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
@@ -4,13 +4,11 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"log" "log"
"strings" "strings"
"sync" "sync"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream" "github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery"
@@ -64,14 +62,11 @@ func (w *toolchainWatch) ensure(module string) error {
w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module) w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module)
return nil return nil
case err != nil: case err != nil:
// Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the module's next report. return fmt.Errorf("the bucket %s: %w", name, err)
w.logger.Printf("the bucket %s cannot be read (%v): %s's toolchains are not heard until its next report", name, err, module)
return nil
} }
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.put(module, e.Key(), e.Value(), !opOf(e)) }) watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.put(module, e.Key(), e.Value(), !opOf(e)) })
if err != nil { if err != nil {
process.NotWatched(w.logger.Printf, name, err, fmt.Sprintf("%s's toolchains are not heard until its next report", module)) return err
return nil
} }
w.watched[module] = true w.watched[module] = true
go func() { go func() {
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-builder
go 1.26.0 go 1.26.0
require ( require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142434-65ee43f07583 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1
github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0 github.com/nats-io/nats.go v1.54.0
) )
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142434-65ee43f07583 h1:ikD/ggf9/5k23sxJOX1PFcGzb4M3+QeC66AT0MyQ9Zs= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1 h1:fzOkrz9CSp3mEjeV7NfZx8yoivI1sD1AOtIPtyyF8RI=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142434-65ee43f07583/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -1,344 +0,0 @@
package build
import (
"context"
"errors"
"fmt"
"regexp"
"slices"
"strconv"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// **An ask that can no longer be answered is asked again** (novox/hq issue 1945). A build seat's stream gives an ask up
// once it was delivered as often as its worker may (delivery.AskMaxDeliver), and a message given up on, dropped or
// terminated with no step written is answered by no build node: the lane waiting for its step would wait for ever, and on
// restart take over the same id and wait again. The builder notices it at the check bound (CheckAfter, the step bound),
// a context's bound and no timer of its own: it asks Gone, which reads the bus, and asks such an ask again under a new
// id, written into the key, said in its log and as the event build.asked-again, at most Askings times; then the key is
// failed with why. An ask DEAD_LETTERS says the seat gave up (buildrun.LostAskWaiter, issue 734) is asked again at once.
// A person asks the same by hand with the tool builder_ask_again (AskAgain).
// DefaultAskings is how many times a build module's part of a build is asked, the first included, when Builder.Askings is 0.
const DefaultAskings = 3
// AskedAgainEvent is the event the builder says when it asked an ask again: mesh.mod.mesh-delivery-builder.event.<this>.
const AskedAgainEvent = "build.asked-again"
// AskedAgain is the body of AskedAgainEvent.
type AskedAgain struct {
Key string `json:"key"`
Module string `json:"module"`
FP string `json:"fp"`
BuildModule string `json:"build_module"`
Language string `json:"language"`
Old string `json:"old"`
New string `json:"new"`
Asking int `json:"asking"`
Askings int `json:"askings"`
Why string `json:"why"`
At time.Time `json:"at"`
}
// AskSeen is what the bus says of an ask whose step has not ended, as the builder may read it: its build seat's stream
// (information and messages by sequence) and its build module's steps. The builder may not read the seat workers'
// delivery counts or ack floors (granted to the delivery view alone: $JS.API.CONSUMER.INFO.<stream>.<worker>), so a
// message given up on is told by what the workers did after it: a build node took an ask asked later, of the same kind or
// of one pulled after it (issue 1945 note 62: seven messages SEAT_NODE_BUILD_GO had given up on sat ahead of a Go ask for
// five hours, each counted as waiting, so the ask was never asked again).
type AskSeen struct {
Stream string
// Kind is the ask's kind, At when it was asked (its id's time).
Kind string
At time.Time
// Step is whether a build node wrote a step for the ask (running: the step bound of buildrun.Asker judges it).
Step bool
// OnStream is whether the stream still holds the ask's message; PutAt is when it was put.
OnStream bool
PutAt time.Time
// Ahead are the messages of its subject before it on the stream.
Ahead []Queued
// Last is the ask asked last that a build node took (wrote a step for) of the ask's kind or of a kind pulled after
// it, when one was taken: every message of the kind asked before it was then delivered already, or given up on.
Last *Taken
}
// Queued is one message of a build seat's stream: its sequence, its ask's id and kind, when the ask was asked (its id's
// time, the put time when the id says none) and put, and the state of the step a build node wrote for it ("" for none).
type Queued struct {
Seq uint64 `json:"seq"`
Ask string `json:"ask"`
Kind string `json:"kind"`
At time.Time `json:"asked"`
PutAt time.Time `json:"put"`
Step string `json:"step,omitempty"`
}
// Taken is an ask a build node took: its id, kind and when it was asked, and the step's node and start.
type Taken struct {
Ask string `json:"ask"`
Kind string `json:"kind"`
At time.Time `json:"asked"`
Node string `json:"node"`
Started time.Time `json:"started"`
}
// Proves is whether a taken ask of kind taken shows that a message of kind waiting asked before it is no longer
// delivered: a holder pulls the kinds in delivery.AskKinds' order, so taking one of the same kind or of a later one means
// its worker had none of waiting's kind left to give.
func Proves(taken, waiting string) bool {
t, w := slices.Index(delivery.AskKinds, taken), slices.Index(delivery.AskKinds, waiting)
return t >= 0 && w >= 0 && t >= w
}
// Waits is whether a message ahead still waits to be delivered: no build node wrote a step for it, and no ask of its
// kind asked after it was taken. A message its worker gave up on (delivered its most times) stays on a work queue until
// its age, and is not counted.
func (s AskSeen) Waits(q Queued) bool {
return q.Step == "" && !s.passed(q.At)
}
func (s AskSeen) passed(at time.Time) bool {
return s.Last != nil && Proves(s.Last.Kind, s.Kind) && s.Last.At.After(at)
}
// Waiting is how many messages ahead still wait to be delivered (Waits).
func (s AskSeen) Waiting() int {
n := 0
for _, q := range s.Ahead {
if s.Waits(q) {
n++
}
}
return n
}
// Why is why the ask can no longer be answered, or "" when it can: it has a step, waits behind others of its kind still
// deliverable, or no build node took a later ask of its kind (the nodes are busy or paused: a backlog, never a fault,
// ADR 0525 §9).
func (s AskSeen) Why() string {
switch {
case s.Step:
return ""
case !s.OnStream:
return fmt.Sprintf("no build node wrote a step for it and %s no longer holds it: it was given up on or dropped", s.Stream)
case s.Waiting() > 0 || !s.passed(s.at()):
return ""
default:
behind := ""
if n := len(s.Ahead); n > 0 {
behind = fmt.Sprintf(" behind %d given up on or taken", n)
}
return fmt.Sprintf("it waits on %s since %s%s while %s took the %s ask %s asked later at %s: its stream no longer delivers it (given up after its most deliveries)",
s.Stream, s.PutAt.UTC().Format(time.RFC3339), behind, s.Last.Node, s.Last.Kind, s.Last.Ask, s.Last.At.UTC().Format(time.RFC3339))
}
}
func (s AskSeen) at() time.Time {
if s.At.IsZero() {
return s.PutAt
}
return s.At
}
// AskedAt is when an ask id was made (buildstep.NewID: its milliseconds in base 36 before six hex digits), or zero.
func AskedAt(id string) time.Time {
if len(id) <= 6 {
return time.Time{}
}
ms, err := strconv.ParseInt(id[:len(id)-6], 36, 64)
if err != nil || ms <= 0 {
return time.Time{}
}
return time.UnixMilli(ms)
}
// Dropper removes an ask's message from its build seat's stream (Builder.Gone may be one): the old message of an ask asked
// again, so a message given up on does not sit ahead of every later ask of its kind until its age (issue 1945 note 62).
type Dropper interface {
Drop(ctx context.Context, a delivery.StepAsk) error
}
// drop removes the old ask's message when Gone can; a refusal is said and changes nothing else: the new ask is put.
func (b *Builder) drop(ctx context.Context, ask delivery.StepAsk, old string) {
d, ok := b.Gone.(Dropper)
if !ok {
return
}
prior := ask
prior.ID = old
if err := d.Drop(ctx, prior); err != nil {
b.logf("the message of the ask %s asked again as %s stays on its stream: %v", old, ask.ID, err)
}
}
func (b *Builder) askings() int {
if b.Askings > 0 {
return b.Askings
}
return DefaultAskings
}
var askingWords = regexp.MustCompile(`asked (\S+) again \(asking (\d+)\)`)
// askingOf is the asking a key's step words name for a build module's ask, 1 when none.
func askingOf(step, buildModule string) int {
for _, m := range askingWords.FindAllStringSubmatch(step, -1) {
if m[1] == buildModule {
if n, err := strconv.Atoi(m[2]); err == nil && n > 0 {
return n
}
}
}
return 1
}
// AskAgain asks the waiting asks of a build key again now, under new ids (the tool builder_ask_again): it answers the
// languages whose lane waits, none when no lane of this builder waits on the key.
func (b *Builder) AskAgain(key, why string) []string {
b.init()
b.mu.Lock()
defer b.mu.Unlock()
var asked []string
for language, c := range b.waiting[key] {
select {
case c <- why:
default: // one is already asked
}
asked = append(asked, language)
}
return asked
}
// Waiting are the build keys whose lanes wait for a step now, each with its languages.
func (b *Builder) Waiting() map[string][]string {
b.init()
b.mu.Lock()
defer b.mu.Unlock()
out := map[string][]string{}
for key, ls := range b.waiting {
for l := range ls {
out[key] = append(out[key], l)
}
}
return out
}
func (b *Builder) listen(key, language string) chan string {
b.init()
c := make(chan string, 1)
b.mu.Lock()
if b.waiting[key] == nil {
b.waiting[key] = map[string]chan string{}
}
b.waiting[key][language] = c
b.mu.Unlock()
return c
}
func (b *Builder) unlisten(key, language string, c chan string) {
b.mu.Lock()
if b.waiting[key][language] == c {
delete(b.waiting[key], language)
if len(b.waiting[key]) == 0 {
delete(b.waiting, key)
}
}
b.mu.Unlock()
}
var (
errCheck = errors.New("the check bound passed")
errAgain = errors.New("asked again")
)
type byHand struct{ why string }
func (h byHand) Error() string { return "asked again by hand: " + h.why }
// answer waits for the step of the lane's ask, and asks it again while it can no longer be answered (Gone) or a person
// asked it again (AskAgain); it answers the step and the ask that has it.
func (b *Builder) answer(ctx context.Context, l *lane, ask delivery.StepAsk, asked *askNotes, module, fp string) (delivery.Step, delivery.StepAsk, error) {
key, _ := delivery.BuilderBuildKey(module, fp)
again := b.listen(key, ask.Language)
defer b.unlisten(key, ask.Language, again)
asking := askingOf(asked.step, l.module)
for {
step, why, err := b.waitOnce(ctx, ask, again)
switch {
case errors.Is(err, errCheck):
continue
case errors.Is(err, errAgain):
if asking >= b.askings() {
return step, ask, fmt.Errorf("the ask %s could not be answered and was asked %d times, the most it is: %s", ask.ID, asking, why)
}
asking++
old := ask.ID
ask.ID, ask.At = buildstep.NewID(), time.Now()
if err := asked.note(ctx, ask.Language, ask.ID, fmt.Sprintf("asked %s again (asking %d): %s", l.module, asking, why)); err != nil {
return step, ask, fmt.Errorf("the ask of %s asked again could not be written to the key: %w", l.module, err)
}
if err := b.Asks.Put(ctx, ask); err != nil {
return step, ask, err
}
b.drop(ctx, ask, old)
b.logf("%s: the ask %s of %s is asked again as %s (asking %d of %d): %s", key, old, l.module, ask.ID, asking, b.askings(), why)
if b.Said != nil {
b.Said(AskedAgain{Key: key, Module: module, FP: fp, BuildModule: l.module, Language: ask.Language, Old: old, New: ask.ID,
Asking: asking, Askings: b.askings(), Why: why, At: ask.At})
}
default:
return step, ask, err
}
}
}
// waitOnce waits for the ask's step until it ends, a person asks it again, or the check bound passes: then it asks Gone
// and answers errAgain with why when the ask can no longer be answered, errCheck when it waits on.
func (b *Builder) waitOnce(ctx context.Context, ask delivery.StepAsk, again <-chan string) (delivery.Step, string, error) {
hand, cancel := context.WithCancelCause(ctx)
defer cancel(nil)
go func() {
select {
case why := <-again:
cancel(byHand{why})
case <-hand.Done():
}
}()
wait := hand
if b.Gone != nil && b.CheckAfter > 0 {
var stop context.CancelFunc
wait, stop = context.WithTimeoutCause(hand, b.CheckAfter, errCheck)
defer stop()
}
step, err := b.Asks.Wait(wait, ask)
if err == nil && buildrun.IsLost(step) {
// The build seat gave the ask up (issue 734): it can no longer be answered, so it is asked again now, with the
// words of why, not at the check bound.
return step, step.Reason, errAgain
}
if err == nil || ctx.Err() != nil {
return step, "", err
}
var h byHand
if errors.As(context.Cause(hand), &h) {
return step, strings.TrimSpace("asked again by hand: " + h.why), errAgain
}
if errors.Is(context.Cause(wait), errCheck) {
why, gerr := b.Gone.Gone(ctx, ask)
if gerr != nil {
b.logf("whether the ask %s can still be answered cannot be read, so it is waited for: %v", ask.ID, gerr)
return step, "", errCheck
}
if why == "" {
return step, "", errCheck
}
return step, why, errAgain
}
return step, "", err
}
@@ -1,115 +0,0 @@
package build
import (
"encoding/json"
"fmt"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// **The tool builder_seat_queue reads a build seat's stream as the builder reads it** (novox/hq issue 1945 note 62; the
// instruction "tools are how a module is spoken to"): the messages waiting on SEAT_NODE_BUILD_<LANGUAGE>, bounded, each
// with its ask, its age, the state of the step a build node wrote for it, and whether the builder counts it as still
// waiting (AskSeen.Waits) or as given up on. Only the builder's own account may read the stream (issue 1910): the tools
// bundle writes the read into the builder's state queues, keyed by the language, which the builder watches and answers
// there, as builder_ask_again does through reasks.
const (
SeatQueueTool = "builder_seat_queue"
// QueuesState is the builder's state of queue reads: the bucket mesh-delivery-builder_queues, keyed by the language.
QueuesState = "queues"
QueuesBucket = delivery.BuilderModule + "_" + QueuesState
// DefaultQueueLimit and MaxQueueLimit bound the messages read of each kind.
DefaultQueueLimit = 50
MaxQueueLimit = 500
)
// QueueNote says what the read cannot show.
const QueueNote = "the builder may not read the seat workers' delivery counts or ack floors ($JS.API.CONSUMER.INFO.<stream>.<worker> is " +
"granted to the delivery view alone): a message counts as given up on when no node wrote its step and a node took an ask of its kind " +
"asked after it; nats.nats_streams shows the workers' pending and waiting pulls"
// QueueAsk is a value of the queues state: a read asked (limit, who, when) and, once the builder read the stream, its answer.
type QueueAsk struct {
V int `json:"v"`
Limit int `json:"limit"`
By string `json:"by,omitempty"`
At time.Time `json:"at"`
Answered time.Time `json:"answered,omitzero"`
Error string `json:"error,omitempty"`
Queue *SeatQueue `json:"queue,omitempty"`
}
// SeatQueue is what the builder read of one build seat's stream.
type SeatQueue struct {
Language string `json:"language"`
Stream string `json:"stream"`
Messages uint64 `json:"messages"`
FirstSeq uint64 `json:"first_seq"`
LastSeq uint64 `json:"last_seq"`
Kinds []KindQueue `json:"kinds"`
Note string `json:"note"`
}
// KindQueue is one kind's messages on the stream, oldest first: how many the builder counts as waiting, the ask asked
// last a build node took of the kind (or of one pulled after it), and each message.
type KindQueue struct {
Kind string `json:"kind"`
Read int `json:"read"`
Waiting int `json:"waiting"`
GivenUp int `json:"given_up_or_taken"`
Last *Taken `json:"last_taken,omitempty"`
Messages []QueuedState `json:"messages"`
}
// QueuedState is one message as the tool shows it.
type QueuedState struct {
Queued
AgeSeconds int64 `json:"age_s"`
Waits bool `json:"waits"`
}
// NewKindQueue is seen's messages ahead as one kind's queue at now.
func NewKindQueue(kind string, seen AskSeen, now time.Time) KindQueue {
k := KindQueue{Kind: kind, Read: len(seen.Ahead), Last: seen.Last}
for _, q := range seen.Ahead {
w := seen.Waits(q)
if w {
k.Waiting++
} else {
k.GivenUp++
}
k.Messages = append(k.Messages, QueuedState{Queued: q, AgeSeconds: int64(now.Sub(q.PutAt) / time.Second), Waits: w})
}
return k
}
// NewQueueAsk is the value builder_seat_queue writes for a language: its key and the value.
func NewQueueAsk(language string, limit int, by string, at time.Time) (string, []byte, error) {
if _, err := delivery.AskStream(language); err != nil {
return "", nil, fmt.Errorf("language: %w", err)
}
if limit <= 0 {
limit = DefaultQueueLimit
}
data, err := json.Marshal(QueueAsk{V: 1, Limit: min(limit, MaxQueueLimit), By: by, At: at.UTC()})
return language, data, err
}
// DecodeQueueAsk reads a queues value.
func DecodeQueueAsk(data []byte) (QueueAsk, error) {
var a QueueAsk
if err := json.Unmarshal(data, &a); err != nil {
return a, err
}
if a.V != 1 {
return a, fmt.Errorf("a queues value of version %d, not 1", a.V)
}
return a, nil
}
// LanguageOf is the language a tool's seat or language argument names: go, node or image, or node-build-<language>.
func LanguageOf(arg string) string {
return strings.TrimPrefix(strings.TrimSpace(arg), "node-build-")
}
@@ -56,18 +56,6 @@ type Builder struct {
// Trunk answers the head of a repository in the gate's source revision rev (source.main at that revision): the trunk a // Trunk answers the head of a repository in the gate's source revision rev (source.main at that revision): the trunk a
// candidate's preview is merged onto (Candidate.Base). Nil builds no candidate's fingerprint, and says why. // candidate's preview is merged onto (Candidate.Base). Nil builds no candidate's fingerprint, and says why.
Trunk func(ctx context.Context, rev uint64, repository string) (string, error) Trunk func(ctx context.Context, rev uint64, repository string) (string, error)
// Gone says why an ask with no ended step can no longer be answered, or "" when it can (issue 1945): asked once the
// ask has waited CheckAfter, and again each CheckAfter it goes on waiting. An ask that cannot be answered is asked
// again under a new id, at most Askings times. Nil, or CheckAfter 0, never asks it.
Gone interface {
Gone(ctx context.Context, a delivery.StepAsk) (string, error)
}
CheckAfter time.Duration
// Askings is how many times one build module's part of a build is asked, the first included, before its key is failed
// with why; DefaultAskings when 0.
Askings int
// Said is told each ask asked again (the builder's event build.asked-again). Nil tells nobody.
Said func(AskedAgain)
once sync.Once once sync.Once
sem chan struct{} sem chan struct{}
@@ -87,8 +75,6 @@ type Builder struct {
// held are the done keys' references the store answered held, by `<key> <references>`, asked once per process: a // held are the done keys' references the store answered held, by `<key> <references>`, asked once per process: a
// blob collected later is found by the take-in, whose artifacts-gone forgets the key here (issue 1937). // blob collected later is found by the take-in, whose artifacts-gone forgets the key here (issue 1937).
held map[string]bool held map[string]bool
// waiting are the lanes that wait for an ask's step, by build key and language: AskAgain asks through them.
waiting map[string]map[string]chan string
} }
type planned struct { type planned struct {
@@ -105,7 +91,7 @@ func (b *Builder) init() {
b.sem = make(chan struct{}, limit) b.sem = make(chan struct{}, limit)
b.cands, b.via, b.trunkFP = map[string]delivery.Candidate{}, map[string]via{}, map[string]bool{} b.cands, b.via, b.trunkFP = map[string]delivery.Candidate{}, map[string]via{}, map[string]bool{}
b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{} b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{}
b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{} b.published, b.held = map[string]bool{}, map[string]bool{}
}) })
} }
@@ -687,14 +673,12 @@ type askNotes struct {
b *Builder b *Builder
cl *Claim cl *Claim
resumed map[string]string resumed map[string]string
// step is the step words of a key taken over at start: an ask asked again says its asking there (askingOf).
step string
} }
func (b *Builder) notes(cl *Claim) *askNotes { func (b *Builder) notes(cl *Claim) *askNotes {
n := &askNotes{b: b, cl: cl} n := &askNotes{b: b, cl: cl}
if cl.Build.State == delivery.BuildBuilding && len(cl.Build.Asks) > 0 { if cl.Build.State == delivery.BuildBuilding && len(cl.Build.Asks) > 0 {
n.resumed, n.step = maps.Clone(cl.Build.Asks), cl.Build.Step n.resumed = maps.Clone(cl.Build.Asks)
} }
return n return n
} }
@@ -792,7 +776,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del
return return
} }
} }
step, ask, err := b.answer(ctx, l, ask, asked, module, m.FP2) step, err := b.Asks.Wait(ctx, ask)
if err != nil { if err != nil {
l.err = fmt.Errorf("%s: %w", l.module, err) l.err = fmt.Errorf("%s: %w", l.module, err)
return return
@@ -6,10 +6,8 @@ import (
"errors" "errors"
"fmt" "fmt"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"strconv"
"strings" "strings"
"sync" "sync"
"sync/atomic"
"testing" "testing"
"time" "time"
@@ -618,304 +616,27 @@ func TestAFailedPublishOfACandidatesImageIsWrittenToTheKey(t *testing.T) { // re
} }
} }
// goneAsks answers Gone from a function of the ask, and counts how often it was asked; it keeps the ids it was asked to
// drop from their stream (Dropper).
type goneAsks struct {
mu sync.Mutex
n int
why func(a delivery.StepAsk) string
dropped []string
}
func (g *goneAsks) Drop(_ context.Context, a delivery.StepAsk) error {
g.mu.Lock()
defer g.mu.Unlock()
g.dropped = append(g.dropped, a.ID)
return nil
}
func (g *goneAsks) Gone(_ context.Context, a delivery.StepAsk) (string, error) {
g.mu.Lock()
g.n++
g.mu.Unlock()
return g.why(a), nil
}
func (g *goneAsks) asked() int {
g.mu.Lock()
defer g.mu.Unlock()
return g.n
}
// waitForAsks waits until n asks were put.
func waitForAsks(t *testing.T, r *rig, n int) []delivery.StepAsk {
t.Helper()
for deadline := time.Now().Add(2 * time.Second); len(r.asks.Asks()) < n; time.Sleep(time.Millisecond) {
if time.Now().After(deadline) {
t.Fatalf("%d asks put, want %d", len(r.asks.Asks()), n)
}
}
return r.asks.Asks()
}
// Review round 2 item 1: a build ask that waits in its stream longer than the step bound is not failed and not asked // Review round 2 item 1: a build ask that waits in its stream longer than the step bound is not failed and not asked
// again; the key is done when a node takes it. Issue 1945: an ask that can no longer be answered (its stream gave it up) // again; the key is done when a node takes it.
// is asked again under a new id, said, at most Askings times; a person asks it again by hand the same way.
func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
t.Run("waits in its stream: never asked again", func(t *testing.T) { r := newRig(t, goBundle)
r := newRig(t, goBundle) r.asks.Answer = nil
r.asks.Answer = nil r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 20 * time.Millisecond}
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 20 * time.Millisecond} r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
gone := &goneAsks{why: func(delivery.StepAsk) string { return "" }} for deadline := time.Now().Add(2 * time.Second); len(r.asks.Asks()) == 0; time.Sleep(time.Millisecond) {
r.b.Gone, r.b.CheckAfter = gone, 5*time.Millisecond if time.Now().After(deadline) {
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) t.Fatal("no ask was put")
waitForAsks(t, r, 1)
time.Sleep(100 * time.Millisecond)
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding {
t.Fatalf("the key = %+v while its ask waits, want building", v)
} }
if gone.asked() == 0 {
t.Fatal("whether the ask can still be answered was never read at the check bound") // 1945
}
r.asks.End(r.answer(r.asks.Asks()[0]))
r.b.Wait()
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(r.asks.Asks()) != 1 {
t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks()))
}
})
for _, c := range []struct {
name string
askings int
// step is the key's step words at start when it names an ask (a key taken over), "" for a fresh build.
step string
byHand bool
always bool // Gone says every ask cannot be answered, not only the first
dead bool // DEAD_LETTERS keeps the first ask: the seat gave it up (issue 734)
puts int
done bool
asking int // the asking the event says, 0 when none is said
}{
{name: "given up by its stream: asked again under a new id and done from it", puts: 2, done: true, asking: 2},
{name: "asked again by hand: the same, with no check bound", byHand: true, puts: 2, done: true, asking: 2},
{name: "given up into DEAD_LETTERS: its wait ends failed with the words and it is asked again at once", dead: true, puts: 2, done: true, asking: 2},
{name: "never answerable: failed after its askings", askings: 2, always: true, puts: 2},
{name: "taken over at its last asking: failed, not asked again", askings: 3, always: true, step: "asked mesh-build-go again (asking 3): x", puts: 0},
} {
t.Run(c.name, func(t *testing.T) { // 1945
r := newRig(t, goBundle)
r.asks.Answer = nil
r.b.Askings = c.askings
var said []AskedAgain
var mu sync.Mutex
r.b.Said = func(e AskedAgain) { mu.Lock(); said = append(said, e); mu.Unlock() }
first := ""
if c.step != "" {
first = "mvl0frv18d1960"
data, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD,
Step: c.step, Asks: map[string]string{delivery.AskGo: first}})
r.st.vals[r.key] = data
}
gone := &goneAsks{}
dead := &buildrun.DeadLetters{}
if c.dead {
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: buildrun.LostAskWaiter{Inner: r.asks, Dead: dead}, Bound: 2 * time.Second}
}
if !c.byHand && !c.dead {
gone.why = func(a delivery.StepAsk) string {
if c.always || a.ID == first || first == "" {
return "it waits first in line while a node took a later ask"
}
return ""
}
r.b.Gone, r.b.CheckAfter = gone, 5*time.Millisecond
}
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
if c.dead {
dead.Heard(buildrun.GivenUp{AskID: waitForAsks(t, r, 1)[0].ID, Stream: "SEAT_NODE_BUILD_GO", Deliveries: 5,
At: time.Date(2026, 10, 26, 12, 26, 40, 0, time.UTC)})
}
if c.byHand {
waitForAsks(t, r, 1)
for deadline := time.Now().Add(2 * time.Second); len(r.b.AskAgain(r.key, "stuck since 19:06")) == 0; time.Sleep(time.Millisecond) {
if time.Now().After(deadline) {
t.Fatal("no lane of the key waits")
}
}
}
if c.done {
asks := waitForAsks(t, r, c.puts)
if first == "" && !c.byHand {
first = asks[0].ID
}
last := asks[len(asks)-1]
if v := valueOf(t, r.st, r.key); v.Asks[delivery.AskGo] != last.ID || !strings.Contains(v.Step, "again (asking 2)") {
t.Fatalf("the key = %+v, want building naming the ask asked again %s", v, last.ID)
}
r.asks.End(r.answer(last))
}
r.b.Wait()
v := valueOf(t, r.st, r.key)
if n := len(r.asks.Asks()); n != c.puts {
t.Errorf("%d asks put, want %d", n, c.puts)
}
if c.done && v.State != delivery.BuildDone {
t.Errorf("the key = %+v, want done from the ask asked again", v)
}
if !c.done && (v.State != delivery.BuildFailed || !strings.Contains(v.Error, "could not be answered and was asked")) {
t.Errorf("the key = %+v, want failed saying the ask could not be answered", v)
}
mu.Lock()
defer mu.Unlock()
// Note 62: the old message of an ask asked again because its stream gave it up is dropped from the stream.
if gone.mu.Lock(); c.done && !c.byHand && !c.dead && (len(gone.dropped) != 1 || gone.dropped[0] != first) {
t.Errorf("dropped %v from the stream, want the old ask %s", gone.dropped, first)
}
gone.mu.Unlock()
if c.asking > 0 && (len(said) != 1 || said[0].Asking != c.asking || said[0].New == said[0].Old || said[0].Key != r.key || said[0].Why == "") {
t.Errorf("said %+v, want once at asking %d", said, c.asking)
}
if c.dead && (len(said) != 1 || !strings.Contains(said[0].Why, buildrun.LostWords+" after 5 deliveries at 2026-10-26T12:26:40Z")) {
t.Errorf("said %+v, want the ask asked again saying the seat gave it up", said)
}
})
} }
t.Run("what the bus says of an ask", func(t *testing.T) { // 1945 time.Sleep(100 * time.Millisecond)
put := time.Date(2026, 10, 25, 17, 6, 0, 0, time.UTC) if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding {
taken := func(kind string, at time.Time) *Taken { t.Fatalf("the key = %+v while its ask waits, want building", v)
return &Taken{Ask: "later", Kind: kind, At: at, Node: "g14", Started: at} }
} r.asks.End(r.answer(r.asks.Asks()[0]))
later := taken(delivery.AskDelivery, put.Add(time.Minute)) r.b.Wait()
// ahead is n messages put before the ask, a minute apart, each with step (its state, "" for none). if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(r.asks.Asks()) != 1 {
ahead := func(n int, step string) []Queued { t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks()))
var qs []Queued }
for i := range n {
at := put.Add(-time.Duration(n-i) * time.Minute)
qs = append(qs, Queued{Seq: uint64(i + 1), Ask: fmt.Sprint("a", i), Kind: delivery.AskDelivery, At: at, PutAt: at, Step: step})
}
return qs
}
seen := func(qs []Queued, last *Taken) AskSeen {
return AskSeen{Stream: "S", Kind: delivery.AskDelivery, At: put, OnStream: true, PutAt: put, Ahead: qs, Last: last}
}
for _, c := range []struct {
name string
seen AskSeen
want string // a part of the why, "" when it can be answered
waiting int
}{
{"a node took it", AskSeen{Stream: "S", Step: true}, "", 0},
{"no longer on its stream", AskSeen{Stream: "S"}, "no longer holds it", 0},
{"first in line, a later ask taken", seen(nil, later), "no longer delivers it", 0},
{"first in line, nothing taken since", seen(nil, nil), "", 0},
{"first in line, the ask taken last was asked before it", seen(nil, taken(delivery.AskDelivery, put.Add(-time.Second))), "", 0},
{"two wait ahead, nothing taken since", seen(ahead(2, ""), nil), "", 2},
{"two ahead passed over, the ask not yet", seen(ahead(2, ""), taken(delivery.AskDelivery, put.Add(-time.Second))), "", 0},
// Note 62: seven messages SEAT_NODE_BUILD_GO gave up on sat ahead of a Go ask for five hours.
{"seven given up on ahead, a later ask taken", seen(ahead(7, ""), later), "behind 7 given up on or taken", 0},
{"seven taken ahead (failed as poison), a later ask taken", seen(ahead(7, delivery.StepFailed), later), "no longer delivers it", 0},
{"a check ask taken later proves a delivery ask passed", seen(ahead(7, ""), taken(delivery.AskCheck, put.Add(time.Minute))), "no longer delivers it", 0},
{"a delivery ask taken later does not prove a check ask passed", func() AskSeen { s := seen(ahead(3, ""), later); s.Kind = delivery.AskCheck; return s }(), "", 3},
} {
got := c.seen.Why()
if (c.want == "") != (got == "") || !strings.Contains(got, c.want) || c.seen.Waiting() != c.waiting {
t.Errorf("%s: why %q and %d waiting, want %q and %d", c.name, got, c.seen.Waiting(), c.want, c.waiting)
}
}
if at := time.UnixMilli(1793120000123); !AskedAt(strconv.FormatInt(at.UnixMilli(), 36)+"0a1b2c").Equal(at) || !AskedAt("x").IsZero() {
t.Errorf("AskedAt does not read an ask id's time")
}
})
t.Run("the tool builder_stuck", func(t *testing.T) { // 1945
at := time.Date(2026, 10, 25, 17, 6, 0, 0, time.UTC)
state := func(vals map[string]string) StateRead {
return StateRead{Keys: func() ([]string, error) {
var ks []string
for k := range vals {
ks = append(ks, k)
}
return ks, nil
}, Get: func(k string) (json.RawMessage, time.Time, error) {
if v, ok := vals[k]; ok {
return json.RawMessage(v), at, nil
}
return nil, time.Time{}, nil
}}
}
key, _ := delivery.BuilderBuildKey("m", fp1)
done, _ := delivery.BuilderBuildKey("d", fp1)
building, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD,
Step: "asked mesh-build-go again (asking 2): x", Asks: map[string]string{delivery.AskGo: "mvo53jcm70b5f7", delivery.AskImage: "mvo53jcm70b5f8"}})
finished, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: manifestD, Step: "built"})
running, _ := json.Marshal(delivery.Step{V: delivery.Version, ID: "mvo53jcm70b5f8", Tool: "delivery ask", Node: "g14", State: delivery.StepRunning, Started: at})
_, reask, _ := NewReAsk("m", fp1, "stuck", "shanks.lead", at)
r := StuckReader{Builds: state(map[string]string{key: string(building), done: string(finished)}), ReAsks: state(map[string]string{key: string(reask)}),
Steps: map[string]StateRead{buildrun.GoBuildModule: state(map[string]string{}), ImageBuildModule: state(map[string]string{"g14.mvo53jcm70b5f8": string(running)})},
Hearing: state(map[string]string{buildrun.HearingKey: `{"heard":false,"streams":["SEAT_NODE_BUILD_GO"],"why":"DEAD_LETTERS is not heard: SEAT_NODE_BUILD_GO: permissions violation","at":"2026-10-26T13:00:00Z"}`}),
Now: func() time.Time { return at.Add(time.Hour) }}
out, err := r.Stuck("")
if err != nil {
t.Fatal(err)
}
// Issue 734, review round 1: a builder that does not hear DEAD_LETTERS says so, with why.
if dl := out["dead_letters"].(map[string]any); dl["heard"] != false || !strings.Contains(dl["why"].(string), "permissions violation") {
t.Errorf("dead_letters = %v, want not heard, saying why", dl)
}
stuck := out["stuck"].([]Stuck)
if len(stuck) != 1 || stuck[0].Key != key || stuck[0].Age != "1h0m0s" || stuck[0].NoneStep != 1 || stuck[0].ReAsk == nil || stuck[0].ReAsk.Why != "stuck" {
t.Fatalf("stuck = %+v, want the building key alone, an hour old, one ask with no step, with its ask again", stuck)
}
a := stuck[0].Asks
if len(a) != 2 || a[0].Language != delivery.AskGo || a[0].Asking != 2 || a[0].Step != "none" || a[0].Asked.IsZero() ||
a[1].Asking != 1 || !strings.HasPrefix(a[1].Step, "running on g14") {
t.Fatalf("asks = %+v, want go at asking 2 with no step and image at asking 1 running on g14", a)
}
// 1827: on the live shape (3000 build keys over 100 modules, 1 ms a Get, three keys of each module building) a module reads only its
// own keys, the whole answer reads MaxReads values in parallel and says it was cut, and the history (Written) is asked
// only for the rows kept.
var gets, written, inFlight, maxFlight atomic.Int64
vals := map[string]string{}
for i := 0; i < 3000; i++ {
k, _ := delivery.BuilderBuildKey(fmt.Sprintf("mod%02d", i%100), fmt.Sprintf("src2:%064x", i))
vals[k] = string(finished)
if (i/100)%10 == 0 {
vals[k] = string(building)
}
}
slow := state(vals)
get := slow.Get
slow.Get = func(k string) (json.RawMessage, time.Time, error) {
gets.Add(1)
n := inFlight.Add(1)
for m := maxFlight.Load(); n > m && !maxFlight.CompareAndSwap(m, n); m = maxFlight.Load() {
}
time.Sleep(time.Millisecond)
inFlight.Add(-1)
raw, _, err := get(k)
return raw, time.Time{}, err
}
slow.Written = func(string) time.Time { written.Add(1); return at }
for _, row := range []struct {
module string
stuck int
gets int64
cut bool
}{{"mod10", 3, 30, false}, {"", 100, MaxReads, true}} {
gets.Store(0)
written.Store(0)
out, err := StuckReader{Builds: slow, ReAsks: state(map[string]string{}), Steps: map[string]StateRead{}, Now: func() time.Time { return at }}.Stuck(row.module)
if err != nil {
t.Fatal(err)
}
_, cut := out["cut"]
if n := len(out["stuck"].([]Stuck)); n != row.stuck || cut != row.cut || written.Load() != int64(n) || gets.Load() != row.gets {
t.Errorf("%q: %d stuck, %d gets, %d written, cut %v; want %d stuck and gets, written for each, cut %v", row.module, n, gets.Load(), written.Load(), cut, row.stuck, row.cut)
}
}
if maxFlight.Load() < 2 {
t.Errorf("the values were read one at a time")
}
})
} }
// heldAsks says an ask is held when its id is among held. // heldAsks says an ask is held when its id is among held.
@@ -1,307 +0,0 @@
package build
import (
"encoding/json"
"errors"
"fmt"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// The builder's tools (novox/hq issue 1945; the instruction "tools are how a module is spoken to"): builder_stuck reads
// every key that waits on an ask, builder_ask_again asks a key's build again now. The tools bundle serves them through
// the tool runner; the builder's process serves no tool (ADR 0437 §4), so builder_ask_again writes the ask into the
// builder's state reasks, which the builder watches and answers there.
const (
StuckTool = "builder_stuck"
AskAgainTool = "builder_ask_again"
// ReAsksState is the builder's state of asks to ask again: the bucket mesh-delivery-builder_reasks, keyed as the build key.
ReAsksState = "reasks"
ReAsksBucket = delivery.BuilderModule + "_" + ReAsksState
)
// ReAsk is a value of the reasks state: why a person asked the key's build again, who and when, and, once the builder
// read it, its answer.
type ReAsk struct {
V int `json:"v"`
Why string `json:"why"`
By string `json:"by,omitempty"`
At time.Time `json:"at"`
Answer string `json:"answer,omitempty"`
Answered time.Time `json:"answered,omitzero"`
}
// DecodeReAsk reads a reasks value.
func DecodeReAsk(data []byte) (ReAsk, error) {
var r ReAsk
if err := json.Unmarshal(data, &r); err != nil {
return r, err
}
if r.V != 1 {
return r, fmt.Errorf("a reasks value of version %d, not 1", r.V)
}
return r, nil
}
// NewReAsk is the value builder_ask_again writes for a module's fingerprint: its key and the value.
func NewReAsk(module, fp, why, by string, at time.Time) (string, []byte, error) {
key, err := delivery.BuilderBuildKey(module, fp)
if err != nil {
return "", nil, err
}
if strings.TrimSpace(why) == "" {
return "", nil, errors.New("say why the build is asked again (why)")
}
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
return key, data, err
}
// StateRead is a state the tools read: its keys, and a key's value with when it was written (zero when not known).
type StateRead struct {
Keys func() ([]string, error)
Get func(key string) (json.RawMessage, time.Time, error)
// Written is when the key's current value was written, from the state's history: one more round trip, so the tools ask it
// only for a row they keep, and only when Get did not say.
Written func(key string) time.Time
}
// StuckAsk is one ask a key waits on.
type StuckAsk struct {
Language string `json:"language"`
BuildModule string `json:"build_module"`
ID string `json:"id"`
Asking int `json:"asking"`
// Asked is when the ask was put, read from its id; Step is what its build module's steps hold of it: none, or its
// state on its node since its start.
Asked time.Time `json:"asked,omitzero"`
Step string `json:"step"`
}
// Stuck is one build key that waits on an ask.
type Stuck struct {
Key string `json:"key"`
Module string `json:"module"`
FP string `json:"fp"`
State string `json:"state"`
Step string `json:"step"`
By string `json:"by"`
Since time.Time `json:"since,omitzero"`
Age string `json:"age,omitempty"`
Asks []StuckAsk `json:"asks"`
ReAsk *ReAsk `json:"reask,omitempty"`
Unasked bool `json:"unasked,omitempty"`
NoneStep int `json:"asks_with_no_step"`
}
// StuckReader reads what builder_stuck answers.
type StuckReader struct {
Builds StateRead
ReAsks StateRead
// Steps are the build modules' steps states, by build module.
Steps map[string]StateRead
// Hearing is the builder's state hearing: whether it hears DEAD_LETTERS (issue 734); without it none is said.
Hearing StateRead
Now func() time.Time
// Logf, when set, gets the one line a call logs: the values it read and the time it took.
Logf func(format string, args ...any)
}
// ReadWorkers is how many values builder_stuck asks the bus for at once, and MaxReads how many it reads in a call (issue 1827:
// every value is a round trip through the runtime, so one after the other a few thousand keys outlast the call's 30 s).
// A copy of the delivery view's and the checker's: a few lines are not worth a group delivery through mesh-sdk.
const (
ReadWorkers = 16
MaxReads = 1000
)
// parallel runs read(i) for i in [0, n), at most ReadWorkers at a time, and answers the first error.
func parallel(n int, read func(i int) error) error {
var (
next atomic.Int64
failed atomic.Bool
once sync.Once
first error
wg sync.WaitGroup
)
for w := 0; w < min(ReadWorkers, n); w++ {
wg.Add(1)
go func() {
defer wg.Done()
for !failed.Load() {
i := int(next.Add(1) - 1)
if i >= n {
return
}
if err := read(i); err != nil {
once.Do(func() { first = err })
failed.Store(true)
}
}
}()
}
wg.Wait()
return first
}
// Stuck is every key building or queued with its asks, oldest first; with module, that module's only.
// deadLetters says whether the builder hears the asks the build seats give up on (issue 734), and why not: deaf, a lane
// whose ask is given up waits for its check bound (buildrun.HearingHealth over the kept hearing).
func (r StuckReader) deadLetters() map[string]any {
if r.Hearing.Get == nil {
return map[string]any{"heard": false, "why": "the builder's state hearing is not readable here"}
}
raw, _, err := r.Hearing.Get(buildrun.HearingKey)
if err != nil {
return map[string]any{"heard": false, "why": "the builder's state hearing cannot be read: " + err.Error()}
}
a := buildrun.HearingHealth(raw)
return map[string]any{"heard": a.Healthy, "why": a.Why}
}
func (r StuckReader) Stuck(module string) (map[string]any, error) {
started := time.Now()
reads := 0
defer func() {
if r.Logf != nil {
r.Logf("builder_stuck read %d values in %s", reads, time.Since(started).Round(time.Millisecond))
}
}()
keys, err := r.Builds.Keys()
if err != nil {
return nil, err
}
now := time.Now
if r.Now != nil {
now = r.Now
}
// Steps are found by the ask's id, which their key ends in: one index per build module, no value read until an ask names it.
stepKeys := map[string]map[string]string{}
for bm, s := range r.Steps {
ks, err := s.Keys()
if err != nil {
return nil, fmt.Errorf("the steps of %s: %w", bm, err)
}
byID := make(map[string]string, len(ks))
for _, k := range ks {
if _, id, err := delivery.ParseStepKey(k); err == nil {
byID[id] = k
}
}
stepKeys[bm] = byID
}
// Narrowed by the key (the module) before any value is read; the state is in the value, so every matching key is read, in
// parallel, at most MaxReads of them.
type parsed struct{ key, mod, fp string }
var matching []parsed
for _, key := range keys {
mod, fp, err := delivery.ParseBuilderBuildKey(key)
if err != nil || (module != "" && mod != module) {
continue
}
matching = append(matching, parsed{key, mod, fp})
}
sort.Slice(matching, func(i, j int) bool { return matching[i].key < matching[j].key })
cut := len(matching) > MaxReads
total := len(matching)
if cut {
matching = matching[:MaxReads]
}
values := make([]json.RawMessage, len(matching))
ats := make([]time.Time, len(matching))
if err := parallel(len(matching), func(i int) error {
raw, at, err := r.Builds.Get(matching[i].key)
if err == nil {
values[i], ats[i] = raw, at
}
return nil
}); err != nil {
return nil, err
}
reads += len(matching)
out := []Stuck{}
for i, m := range matching {
key, mod, fp := m.key, m.mod, m.fp
raw, at := values[i], ats[i]
if raw == nil {
continue
}
v, err := delivery.DecodeBuilderBuild(raw)
if err != nil || (v.State != delivery.BuildBuilding && v.State != delivery.BuildQueued) {
continue
}
if at.IsZero() && r.Builds.Written != nil {
at = r.Builds.Written(key)
}
s := Stuck{Key: key, Module: mod, FP: fp, State: v.State, Step: v.Step, By: v.By, Since: at, Asks: []StuckAsk{}}
if !at.IsZero() {
s.Age = now().Sub(at).Round(time.Second).String()
}
for language, id := range v.Asks {
bm, _ := delivery.BuildModuleOf(language)
a := StuckAsk{Language: language, BuildModule: bm, ID: id, Asking: askingOf(v.Step, bm), Step: "none"}
if t, ok := idTime(id); ok {
a.Asked = t
}
if st, ok := r.Steps[bm]; ok {
if k, ok := stepKeys[bm][id]; ok {
reads++
if raw, _, err := st.Get(k); err == nil && raw != nil {
if step, err := delivery.DecodeStep(raw); err == nil {
a.Step = fmt.Sprintf("%s on %s since %s", step.State, step.Node, step.Started.UTC().Format(time.RFC3339))
}
}
}
}
if a.Step == "none" {
s.NoneStep++
}
s.Asks = append(s.Asks, a)
}
sort.Slice(s.Asks, func(i, j int) bool { return s.Asks[i].Language < s.Asks[j].Language })
s.Unasked = v.State == delivery.BuildBuilding && len(v.Asks) == 0
if r.ReAsks.Get != nil {
reads++
if raw, _, err := r.ReAsks.Get(key); err == nil && raw != nil {
if ra, err := DecodeReAsk(raw); err == nil {
s.ReAsk = &ra
}
}
}
out = append(out, s)
}
sort.Slice(out, func(i, j int) bool {
if !out[i].Since.Equal(out[j].Since) {
return out[i].Since.Before(out[j].Since)
}
return out[i].Key < out[j].Key
})
answer := map[string]any{"stuck": out, "count": len(out),
"delivery_counts": "not readable by the builder: the build seats' workers' information is granted to the delivery view alone (its backlog); an ask given up on is told by the builder at its check bound and asked again"}
answer["dead_letters"] = r.deadLetters()
if cut {
answer["cut"], answer["note"] = true, fmt.Sprintf("read %d of %d build keys in key order and stopped; give a module to read its keys alone", MaxReads, total)
}
return answer, nil
}
// StuckBuildModules are the build modules whose steps builder_stuck reads.
var StuckBuildModules = []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, ImageBuildModule}
// idTime is when a step id was made (buildstep.NewID: the time in milliseconds in base 36, then six hex digits).
func idTime(id string) (time.Time, bool) {
if len(id) <= 6 {
return time.Time{}, false
}
ms, err := strconv.ParseInt(id[:len(id)-6], 36, 64)
if err != nil || ms <= 0 {
return time.Time{}, false
}
return time.UnixMilli(ms).UTC(), true
}
+11 -26
View File
@@ -23,21 +23,18 @@ type state struct {
} }
type artifact struct { type artifact struct {
Name string `json:"name"` Name string `json:"name"`
Kind string `json:"kind"` Kind string `json:"kind"`
Language string `json:"language"` Language string `json:"language"`
System string `json:"system"` System string `json:"system"`
From string `json:"from"` From string `json:"from"`
Binary string `json:"binary"` Binary string `json:"binary"`
Loads []string `json:"loads"`
} }
type manifest struct { type manifest struct {
Module string `json:"module"` Module string `json:"module"`
Version string `json:"version"` Version string `json:"version"`
Slug string `json:"slug"` Slug string `json:"slug"`
Tools []string `json:"tools"`
Emits []string `json:"emits"`
Consumes []string `json:"consumes"` Consumes []string `json:"consumes"`
State []state `json:"state"` State []state `json:"state"`
Reads []string `json:"reads"` Reads []string `json:"reads"`
@@ -87,7 +84,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
if m.Module != "mesh-delivery-builder" { if m.Module != "mesh-delivery-builder" {
t.Errorf("module = %q", m.Module) t.Errorf("module = %q", m.Module)
} }
if want := []state{{Name: "builds", History: 5}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "queues", History: 1}, {Name: "reasks", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory if want := []state{{Name: "builds", History: 5}, {Name: "instances", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
t.Errorf("state = %+v, want %+v", m.State, want) t.Errorf("state = %+v, want %+v", m.State, want)
} }
// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and // 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and
@@ -115,14 +112,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
if m.SecretsOwner != "mesh-delivery-builder" { if m.SecretsOwner != "mesh-delivery-builder" {
t.Errorf("secrets-owner = %q", m.SecretsOwner) t.Errorf("secrets-owner = %q", m.SecretsOwner)
} }
// 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again. for _, k := range []string{"streams", "logs", "emits", "claims", "provides", "requires", "defines-seats", "tools", "seat", "seats", "bus", "config"} {
if want := []string{"builder_ask_again", "builder_seat_queue", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
if want := []string{"build.asked-again"}; !reflect.DeepEqual(m.Emits, want) {
t.Errorf("emits = %v, want %v", m.Emits, want)
}
for _, k := range []string{"streams", "logs", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} {
if _, ok := top[k]; ok { if _, ok := top[k]; ok {
t.Errorf("the manifest declares %q: the builder has none", k) t.Errorf("the manifest declares %q: the builder has none", k)
} }
@@ -137,7 +127,7 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
procs = append(procs, r) procs = append(procs, r)
} }
if r["type"] == "tool" || r["type"] == "seat" { if r["type"] == "tool" || r["type"] == "seat" {
t.Errorf("resource %v: the builder's tools are its bundle's, and it holds no seat", r) t.Errorf("resource %v: the builder serves no tool and holds no seat", r)
} }
} }
if len(procs) != 1 { if len(procs) != 1 {
@@ -159,13 +149,8 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
if _, bus := p["bus"]; bus { if _, bus := p["bus"]; bus {
t.Errorf("the process declares a bus: it runs on its own broker credential") t.Errorf("the process declares a bus: it runs on its own broker credential")
} }
if len(m.Build.Artifacts) != 2 { if len(m.Build.Artifacts) != 1 {
t.Fatalf("artifacts = %+v, want the program and the tools", m.Build.Artifacts) t.Fatalf("artifacts = %+v, want the program alone", m.Build.Artifacts)
}
if a := m.Build.Artifacts[1]; a.Name != "tools" || a.Kind != "bundle" || a.Binary != "mesh-delivery-builder-tools" || !reflect.DeepEqual(a.Loads, []string{"mesh-delivery-builder-tools"}) {
t.Errorf("tools artifact = %+v", a) // 1945
} else if st, err := os.Stat(a.From); err != nil || !st.IsDir() {
t.Errorf("the tools' source %s: %v", a.From, err)
} }
a := m.Build.Artifacts[0] a := m.Build.Artifacts[0]
if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-builder" || !strings.HasSuffix(a.From, a.Binary) || a.Kind != "bundle" || a.Language != "go" { if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-builder" || !strings.HasSuffix(a.From, a.Binary) || a.Kind != "bundle" || a.Language != "go" {
+1 -23
View File
@@ -2,14 +2,6 @@
"module": "mesh-delivery-builder", "module": "mesh-delivery-builder",
"version": "1", "version": "1",
"slug": "builder", "slug": "builder",
"tools": [
"builder_ask_again",
"builder_seat_queue",
"builder_stuck"
],
"emits": [
"build.asked-again"
],
"consumes": [ "consumes": [
"mesh-build-go.toolchain.reported", "mesh-build-go.toolchain.reported",
"mesh-build-node.toolchain.reported", "mesh-build-node.toolchain.reported",
@@ -20,10 +12,7 @@
], ],
"state": [ "state": [
{ "name": "builds", "history": 5 }, { "name": "builds", "history": 5 },
{ "name": "hearing", "history": 1 }, { "name": "instances", "history": 1 }
{ "name": "instances", "history": 1 },
{ "name": "queues", "history": 1 },
{ "name": "reasks", "history": 1 }
], ],
"reads": [ "reads": [
"mesh-build-go.steps", "mesh-build-go.steps",
@@ -89,17 +78,6 @@
"system": "arch", "system": "arch",
"from": "cmd/mesh-delivery-builder", "from": "cmd/mesh-delivery-builder",
"binary": "mesh-delivery-builder" "binary": "mesh-delivery-builder"
},
{
"name": "tools",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/mesh-delivery-builder-tools",
"binary": "mesh-delivery-builder-tools",
"loads": [
"mesh-delivery-builder-tools"
]
} }
] ]
} }
@@ -7,45 +7,16 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"os" "os"
"sync/atomic"
"time" "time"
stdio "git.novox.be/novox/mesh-sdk/go" stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"mesh-delivery-checker/internal/check" "mesh-delivery-checker/internal/check"
) )
// reads counts the values this call read from the bus: the runtime launches the tools for one call, so the count is the call's.
var reads atomic.Int64
// cutAnswer says in the answer that it was cut, and how to narrow it.
func cutAnswer(answer map[string]any, cut string) map[string]any {
if cut != "" {
answer["cut"], answer["note"] = true, cut
}
return answer
}
// logged runs every tool with one line to stderr per call: the tool, the values it read and the time it took.
func logged(tools []stdio.Tool) []stdio.Tool {
for i := range tools {
name, run := tools[i].Name, tools[i].Run
tools[i].Run = func(args map[string]any) (any, error) {
reads.Store(0)
start := time.Now()
out, err := run(args)
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %s read %d values in %s (error: %v)\n", name, reads.Load(), time.Since(start).Round(time.Millisecond), err)
return out, err
}
}
return tools
}
func read(s stdio.KeptState) check.StateRead { func read(s stdio.KeptState) check.StateRead {
return check.StateRead{Keys: s.Keys, Get: func(key string) (json.RawMessage, error) { return check.StateRead{Keys: s.Keys, Get: func(key string) (json.RawMessage, error) {
reads.Add(1)
e, err := s.Get(key) e, err := s.Get(key)
if err != nil || e == nil { if err != nil || e == nil {
return nil, err return nil, err
@@ -54,36 +25,10 @@ func read(s stdio.KeptState) check.StateRead {
}} }}
} }
// runsOf says which commit the mesh runs of a module from the indexer's modules state, as the checker's own record of it does
// (the commit the module's record was computed at); "" when the record is not readable.
type runsOf struct{ modules check.StateRead }
func (r runsOf) CommitOf(module string) string {
if r.modules.Get == nil {
return ""
}
raw, err := r.modules.Get(module)
if err != nil || len(raw) == 0 {
return ""
}
m, err := delivery.DecodeModule(raw)
if err != nil {
return ""
}
return m.Commit
}
// stateStore is a state the tools write: the tries. // stateStore is a state the tools write: the tries.
type stateStore struct { type stateStore struct {
check.StateRead check.StateRead
create func(key string, value []byte) error create func(key string, value []byte) error
entry func(key string) (json.RawMessage, uint64, error)
update func(key string, value []byte, revision uint64) error
}
func (s stateStore) Entry(key string) (json.RawMessage, uint64, error) { return s.entry(key) }
func (s stateStore) Update(key string, value []byte, revision uint64) error {
return s.update(key, value, revision)
} }
func (s stateStore) Keys() ([]string, error) { return s.StateRead.Keys() } func (s stateStore) Keys() ([]string, error) { return s.StateRead.Keys() }
@@ -93,12 +38,11 @@ func (s stateStore) Create(key string, value []byte) error { return s.create(k
// Trying is what the try's tools work on: the tries, and the gate's trunk heads. // Trying is what the try's tools work on: the tries, and the gate's trunk heads.
type Trying struct { type Trying struct {
Tries check.TryStore Tries check.TryStore
Ender check.TryEnder
Trunks func() (map[string]string, error) Trunks func() (map[string]string, error)
Now func() time.Time Now func() time.Time
} }
// TryTools are the try's tools (ADR 0501 §5): ask one, read one, and end one by hand (issue 734). mesh-controller's try calls them. // TryTools are the try's two tools (ADR 0501 §5): ask one, read one. mesh-controller's try calls them.
func TryTools(tr Trying) []stdio.Tool { func TryTools(tr Trying) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return s } str := func(args map[string]any, k string) string { s, _ := args[k].(string); return s }
return []stdio.Tool{ return []stdio.Tool{
@@ -106,8 +50,7 @@ func TryTools(tr Trying) []stdio.Tool {
Description: "Try a pushed head before its pull request is opened: the checker judges it as it judges a pull request's head, merged onto its trunk's head, " + Description: "Try a pushed head before its pull request is opened: the checker judges it as it judges a pull request's head, merged onto its trunk's head, " +
"its unit and integration levels through the build modules, and owes the forge nothing. Answers the try's id at once; " + check.TryReadTool + " reads " + "its unit and integration levels through the build modules, and owes the forge nothing. Answers the try's id at once; " + check.TryReadTool + " reads " +
"where it stands. A try of the same head that waits or runs is the answer to a second ask; at most 4 wait or run at once, mesh-wide. A repository " + "where it stands. A try of the same head that waits or runs is the answer to a second ask; at most 4 wait or run at once, mesh-wide. A repository " +
"the checker checks no level of (one whose trunk the gate does not name) is refused. A try whose ask no build node took within the step " + "the checker checks no level of (one whose trunk the gate does not name) is refused.",
"bound ends as error; its ask stays in the build seat's stream and may still run later, judged by nobody (issue 734).",
Input: map[string]any{"type": "object", "required": []string{"repository", "head"}, "properties": map[string]any{ Input: map[string]any{"type": "object", "required": []string{"repository", "head"}, "properties": map[string]any{
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"}, "repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
"head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits"}, "head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits"},
@@ -144,28 +87,6 @@ func TryTools(tr Trying) []stdio.Tool {
} }
return check.ReadTry(tr.Tries, str(args, "id")) return check.ReadTry(tr.Tries, str(args, "id"))
}}, }},
{Name: check.TryEndTool,
Description: "End a try that waits or runs, as error, by hand: say why (kept in what it says). Its place among the 4 is free at once, " +
"the checker stops waiting for its asks, and the next " + check.TryTool + " of its head asks anew with a new id. For a try whose ask " +
"was lost (no step runs for it in checker_status); the checker ends one by itself when the build seat gives its ask up, or no node " +
"takes it within the step bound. A try that ended already is refused.",
Input: map[string]any{"type": "object", "required": []string{"id", "why"}, "properties": map[string]any{
"id": map[string]any{"type": "string", "description": "the try's id, try-<digits>"},
"why": map[string]any{"type": "string", "description": "why it is ended, in words"}}},
Run: func(args map[string]any) (any, error) {
if tr.Ender == nil {
return nil, fmt.Errorf("the tries state cannot be written here")
}
now := time.Now
if tr.Now != nil {
now = tr.Now
}
t, err := check.EndTry(tr.Ender, str(args, "id"), str(args, "why"), now())
if err != nil {
return nil, err
}
return map[string]any{"id": t.ID, "state": t.State, "said": t.Said}, nil
}},
} }
} }
@@ -179,8 +100,8 @@ func Tools(r check.Reader) []stdio.Tool {
"group": map[string]any{"type": "string", "description": "the delivery group's branch name"}}}, "group": map[string]any{"type": "string", "description": "the delivery group's branch name"}}},
Run: func(args map[string]any) (any, error) { Run: func(args map[string]any) (any, error) {
group, _ := args["group"].(string) group, _ := args["group"].(string)
vs, cut, err := r.Verdicts(group) vs, err := r.Verdicts(group)
return cutAnswer(map[string]any{"verdicts": vs}, cut), err return map[string]any{"verdicts": vs}, err
}}, }},
{Name: check.PullReadTool, {Name: check.PullReadTool,
Description: "The verdict that judges one pull request and the full report of its status mesh/check. Read-only.", Description: "The verdict that judges one pull request and the full report of its status mesh/check. Read-only.",
@@ -201,8 +122,8 @@ func Tools(r check.Reader) []stdio.Tool {
Run: func(args map[string]any) (any, error) { Run: func(args map[string]any) (any, error) {
level, _ := args["level"].(string) level, _ := args["level"].(string)
digest, _ := args["digest"].(string) digest, _ := args["digest"].(string)
kept, cut, err := r.Kept(level, digest) kept, err := r.Kept(level, digest)
return cutAnswer(map[string]any{"kept": kept}, cut), err return map[string]any{"kept": kept}, err
}}, }},
{Name: check.PreviewsReadTool, {Name: check.PreviewsReadTool,
Description: "The merge previews the checker holds, each as stored under previews.<repository>.<head>.<trunk>: the merge commit, per module directory " + Description: "The merge previews the checker holds, each as stored under previews.<repository>.<head>.<trunk>: the merge commit, per module directory " +
@@ -217,18 +138,15 @@ func Tools(r check.Reader) []stdio.Tool {
if head != "" && len(head) < 7 { if head != "" && len(head) < 7 {
return nil, fmt.Errorf("the head %q is too short: give at least 7 characters", head) return nil, fmt.Errorf("the head %q is too short: give at least 7 characters", head)
} }
ps, bad, cut, err := r.Previews(repository, head) ps, bad, err := r.Previews(repository, head)
return cutAnswer(map[string]any{"previews": ps, "unreadable": bad}, cut), err return map[string]any{"previews": ps, "unreadable": bad}, err
}}, }},
{Name: check.StatusReadTool, {Name: check.StatusReadTool,
Description: "What runs: the steps in flight on the build modules by node, the candidates without a verdict for their heads, " + Description: "What runs: the steps in flight on the build modules by node, the candidates without a verdict for their heads, " +
"and the candidates whose check is owed by kind (waiting_by_kind: new, nodes, and trunk, a re-judge pulled after the other two). A key a " + "and the candidates whose check is owed by kind (waiting_by_kind: new, nodes, and trunk, a re-judge pulled after the other two). A key a " +
"build module left running (started past the step bound, or its build module not running on its node) is listed apart under " + "build module left running (started past the step bound, or its build module not running on its node) is listed apart under " +
"left_running with why, and not counted. Nothing holds the checker and it has no capacity of its own: a check is an ask on the " + "left_running with why, and not counted. Nothing holds the checker and it has no capacity of its own: a check is an ask on the " +
"build seats' streams, whose backlog the delivery view shows (novox/hq ADR 0525). held_by_turn names the candidates whose check an approved " + "build seats' streams, whose backlog the delivery view shows (novox/hq ADR 0525). Read-only.",
"merge's turn holds off (novox/hq ADR 0548 §3), with the repository and the merge that waits; the gate's turns tool reads every turn and its " +
"hand-now tool steers one. dead_letters says whether the checker hears the asks the build seats give up on (issue 734), and why not: " +
"deaf, a try or check whose ask is given up waits until its bound. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{}}, Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) { return r.Status() }}, Run: func(map[string]any) (any, error) { return r.Status() }},
} }
@@ -247,9 +165,7 @@ func main() {
r := check.Reader{VerdictState: read(stdio.State("verdicts")), KeptState: read(stdio.State("kept")), CandidateState: read(stdio.State("mesh-delivery-indexer.candidates")), r := check.Reader{VerdictState: read(stdio.State("verdicts")), KeptState: read(stdio.State("kept")), CandidateState: read(stdio.State("mesh-delivery-indexer.candidates")),
PreviewState: read(stdio.State("previews")), PreviewState: read(stdio.State("previews")),
StepStates: map[string]check.StateRead{"mesh-build-go": read(stdio.State("mesh-build-go.steps")), "mesh-build-node": read(stdio.State("mesh-build-node.steps"))}, StepStates: map[string]check.StateRead{"mesh-build-go": read(stdio.State("mesh-build-go.steps")), "mesh-build-node": read(stdio.State("mesh-build-node.steps"))},
HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound(), HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound()}
TurnState: read(stdio.State("mesh-delivery-gate.turns")), HearingState: read(stdio.State(buildrun.HearingState))}
r.Runs = runsOf{read(stdio.State("mesh-delivery-indexer.modules"))} // the node-engine at the commit the mesh runs, as the checker reads it
tries := stdio.State(check.TriesState) tries := stdio.State(check.TriesState)
source := stdio.State("mesh-delivery-gate.source") source := stdio.State("mesh-delivery-gate.source")
trunks := func() (map[string]string, error) { trunks := func() (map[string]string, error) {
@@ -261,22 +177,14 @@ func main() {
return s.Heads, err return s.Heads, err
} }
r.Trunks = trunks r.Trunks = trunks
store := stateStore{StateRead: read(tries), create: func(key string, value []byte) error { trying := Trying{
_, err := tries.Create(key, json.RawMessage(value)) Tries: stateStore{StateRead: read(tries), create: func(key string, value []byte) error {
return err _, err := tries.Create(key, json.RawMessage(value))
}, entry: func(key string) (json.RawMessage, uint64, error) { return err
reads.Add(1) }},
e, err := tries.Get(key) Trunks: trunks,
if err != nil || e == nil { }
return nil, 0, err if err := stdio.Serve("", append(Tools(r), TryTools(trying)...)); err != nil {
}
return e.Value, e.Revision, nil
}, update: func(key string, value []byte, revision uint64) error {
_, err := tries.Update(key, json.RawMessage(value), revision)
return err
}}
trying := Trying{Tries: store, Ender: store, Trunks: trunks}
if err := stdio.Serve("", logged(append(Tools(r), TryTools(trying)...))); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err) fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err)
os.Exit(1) os.Exit(1)
} }
@@ -53,19 +53,13 @@ func (m *memStore) Create(key string, value []byte) error {
return nil return nil
} }
func (m *memStore) Entry(key string) (json.RawMessage, uint64, error) { return m.vals[key], 1, nil }
func (m *memStore) Update(key string, value []byte, revision uint64) error {
m.vals[key] = value
return nil
}
func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed interface with mesh-controller's try func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed interface with mesh-controller's try
store := &memStore{vals: map[string]json.RawMessage{}} store := &memStore{vals: map[string]json.RawMessage{}}
tools := TryTools(Trying{Tries: store, Ender: store, Now: func() time.Time { return time.Unix(1, 5) }, tools := TryTools(Trying{Tries: store, Now: func() time.Time { return time.Unix(1, 5) },
Trunks: func() (map[string]string, error) { Trunks: func() (map[string]string, error) {
return map[string]string{"novox/mesh-delivery": strings.Repeat("a", 40)}, nil return map[string]string{"novox/mesh-delivery": strings.Repeat("a", 40)}, nil
}}) }})
if len(tools) != 3 || tools[0].Name != "checker_try" || tools[1].Name != "checker_try_read" || tools[2].Name != "checker_try_end" { if len(tools) != 2 || tools[0].Name != "checker_try" || tools[1].Name != "checker_try_read" {
t.Fatalf("tools = %+v", tools) t.Fatalf("tools = %+v", tools)
} }
out, err := tools[0].Run(map[string]any{"repository": "novox/mesh-delivery", "head": strings.Repeat("1", 40), "trunk": "main"}) out, err := tools[0].Run(map[string]any{"repository": "novox/mesh-delivery", "head": strings.Repeat("1", 40), "trunk": "main"})
@@ -80,14 +74,6 @@ func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed i
if err != nil || !strings.Contains(string(raw.(json.RawMessage)), `"state":"queued"`) { if err != nil || !strings.Contains(string(raw.(json.RawMessage)), `"state":"queued"`) {
t.Errorf("read = %s, %v", raw, err) t.Errorf("read = %s, %v", raw, err)
} }
// Issue 734: a try whose ask was lost is ended by hand with why, and its head is asked anew.
ended, err := tools[2].Run(map[string]any{"id": id, "why": "its ask was lost"})
if err != nil || ended.(map[string]any)["state"] != "error" {
t.Errorf("end = %v, %v", ended, err)
}
if _, err := tools[2].Run(map[string]any{"id": id, "why": "again"}); err == nil {
t.Error("an ended try was ended again")
}
if _, err := tools[0].Run(map[string]any{"repository": "novox/hq", "head": strings.Repeat("1", 40)}); err == nil || if _, err := tools[0].Run(map[string]any{"repository": "novox/hq", "head": strings.Repeat("1", 40)}); err == nil ||
err.Error() != "no level of novox/hq is checked by the checker" { err.Error() != "no level of novox/hq is checked by the checker" {
t.Errorf("hq: %v", err) t.Errorf("hq: %v", err)
@@ -24,7 +24,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/announce" "git.novox.be/novox/mesh-sdk/go/announce"
"git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"mesh-delivery-checker/internal/check" "mesh-delivery-checker/internal/check"
) )
@@ -80,8 +79,6 @@ type fakeBuildModules struct {
js jetstream.JetStream js jetstream.JetStream
ctx context.Context ctx context.Context
conflict atomic.Bool conflict atomic.Bool
// hold is a commit whose fetch never answers: its ask stays taken, as one lost in a bus reload (issue 734).
hold atomic.Value
mu sync.Mutex mu sync.Mutex
calls map[string]int // by `<module>.<tool>` calls map[string]int // by `<module>.<tool>`
testTool *testing.T testTool *testing.T
@@ -101,9 +98,6 @@ func (f *fakeBuildModules) result(t *testing.T, tool string, args map[string]any
var v any var v any
switch { switch {
case strings.HasSuffix(tool, "_fetch"): case strings.HasSuffix(tool, "_fetch"):
if h, _ := f.hold.Load().(string); h != "" && args["commit"] == h {
<-f.ctx.Done()
}
name := "go-fetch-clean" name := "go-fetch-clean"
if f.conflict.Load() { if f.conflict.Load() {
name = "go-fetch-conflict" name = "go-fetch-conflict"
@@ -304,10 +298,6 @@ func startBed(t *testing.T) *bed {
} }
b.tools[module] = kv b.tools[module] = kv
} }
// What the controller keeps of the messages a consumer gave up on (issue 734).
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buildrun.DeadLettersStream, Subjects: []string{"mesh.events.dead.>"}}); err != nil {
t.Fatal(err)
}
// The checker's stream of events, which the indexer and the forge bridge hear. // The checker's stream of events, which the indexer and the forge bridge hear.
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "CHECKS", Subjects: []string{"mesh.stream.mesh-delivery-checker.event.>"}}); err != nil { if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "CHECKS", Subjects: []string{"mesh.stream.mesh-delivery-checker.event.>"}}); err != nil {
t.Fatal(err) t.Fatal(err)
@@ -315,8 +305,7 @@ func startBed(t *testing.T) *bed {
for name, history := range map[string]int{delivery.CheckerVerdictsBucket: delivery.CheckerVerdictsHistory, delivery.CheckerKeptBucket: delivery.CheckerKeptHistory, for name, history := range map[string]int{delivery.CheckerVerdictsBucket: delivery.CheckerVerdictsHistory, delivery.CheckerKeptBucket: delivery.CheckerKeptHistory,
delivery.CheckerOwedBucket: delivery.CheckerOwedHistory, delivery.PreviewsBucket: delivery.PreviewsHistory, delivery.CandidatesBucket: delivery.CandidatesHistory, delivery.CheckerOwedBucket: delivery.CheckerOwedHistory, delivery.PreviewsBucket: delivery.PreviewsHistory, delivery.CandidatesBucket: delivery.CandidatesHistory,
delivery.ModulesBucket: delivery.ModulesHistory, delivery.ManifestsBucket: delivery.ManifestsHistory, delivery.SourceBucket: delivery.SourceHistory, delivery.ModulesBucket: delivery.ModulesHistory, delivery.ManifestsBucket: delivery.ManifestsHistory, delivery.SourceBucket: delivery.SourceHistory,
delivery.PresenceBucket: delivery.PresenceHistory, delivery.HaveBucket: delivery.HaveHistory, check.TriesBucket: 1, delivery.PresenceBucket: delivery.PresenceHistory, delivery.HaveBucket: delivery.HaveHistory, check.TriesBucket: 1} {
delivery.TurnsBucket: delivery.TurnsHistory} {
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: uint8(history)}) kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: uint8(history)})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -518,12 +507,9 @@ func TestCheckerJudgesACandidateAndOwesItsStatus(t *testing.T) { // N6/AC5, 1300
} }
} }
// TestACheckOrACarryOfEqualInputsRunsNothing: a second candidate (another head, the same digests) is checked with every // TestACheckOfEqualInputsRunsNothing: a second candidate (another head, the same digests) is checked with every package
// package taken from the first's kept verdicts: no test step runs and no second kept key appears. Then, through the real bus // taken from the first's kept verdicts: no test step runs and no second kept key appears.
// (novox/hq ADR 0548 §3, §5): a third candidate is not claimed while the gate's merge turn on its repository stands, and is func TestACheckOfEqualInputsRunsNothing(t *testing.T) { // 1287/AC5, 1288/AC6, 1929/AC2
// claimed when the turn's key is deleted; and a move of the trunk carries the first verdict on the recheck stream by its
// unchanged digests, running no test, the verdict naming the trunk it was carried from.
func TestACheckOrACarryOfEqualInputsRunsNothing(t *testing.T) { // 1287/AC5, 1288/AC6, 1929/AC2, ADR 0548 §3, §5
b := startBed(t) b := startBed(t)
first := b.candidate(t, "feat/x", pull(repo, 7, head1)) first := b.candidate(t, "feat/x", pull(repo, 7, head1))
b.start(t, nil) b.start(t, nil)
@@ -540,42 +526,6 @@ func TestACheckOrACarryOfEqualInputsRunsNothing(t *testing.T) { // 1287/AC5, 128
if got := keptCount(t, b); got != kept { if got := keptCount(t, b); got != kept {
t.Errorf("%d kept keys, was %d", got, kept) t.Errorf("%d kept keys, was %d", got, kept)
} }
// §3: the gate's turn on the repository holds a new check off; its delete claims it.
at := time.Now().UTC()
turn, err := delivery.EncodeTurn(delivery.Turn{Repository: repo, Waiting: []delivery.TurnWait{{Group: "fix/other", Heads: "0123456789ab", Approved: at, Since: at}}})
if err != nil {
t.Fatal(err)
}
tk, _ := delivery.TurnsKey(repo)
if _, err := b.buckets[delivery.TurnsBucket].Put(b.ctx, tk, turn); err != nil {
t.Fatal(err)
}
time.Sleep(300 * time.Millisecond) // the checker hears the turn
third := b.candidate(t, "feat/z", pull(repo, 9, "3333333333333333333333333333333333333333"))
time.Sleep(time.Second)
if _, err := b.buckets[delivery.CheckerVerdictsBucket].Get(b.ctx, third); !errors.Is(err, jetstream.ErrKeyNotFound) {
t.Fatalf("verdicts.%s exists (%v) while a merge turn holds its repository", third, err)
}
if err := b.buckets[delivery.TurnsBucket].Delete(b.ctx, tk); err != nil {
t.Fatal(err)
}
b.verdict(t, third, ended)
// §5: the trunk moves; the first verdict is carried by its digests, nothing runs.
tests = b.fake.count("mesh-build-go.build_go_test")
const moved = "abababababababababababababababababababab"
b.setTrunk(t, map[string]string{repo: moved})
c := b.verdict(t, first, func(v delivery.Verdict) bool { return ended(v) && v.Trunk[repo] == moved })
if c.State != delivery.VerdictPass || c.CarriedFrom[repo] != trunk || c.Levels[delivery.LevelUnit].Run != 0 || c.Levels[delivery.LevelIntegration].Run != 0 {
t.Fatalf("the carried verdict = %+v, want a pass carried from %s with no test run", c, trunk[:12])
}
if got := b.fake.count("mesh-build-go.build_go_test"); got != tests {
t.Errorf("build_go_test called %d times on the carry, was %d: a carry of unchanged digests runs nothing", got, tests)
}
if o := b.owed(t, 7, head1, "success"); !strings.HasPrefix(o.Text, "pass 111111111111 on abababababab | carried from aaaaaaaaaaaa") {
t.Errorf("the carried status says %q", o.Text)
}
} }
func keptCount(t *testing.T, b *bed) int { func keptCount(t *testing.T, b *bed) int {
@@ -726,49 +676,6 @@ func TestATryIsJudgedAndItsEndWrittenWithNoStatusOwed(t *testing.T) {
t.Errorf("a try wrote %s: %v", name, keys) t.Errorf("a try wrote %s: %v", name, keys)
} }
} }
// Issue 734: a try whose ask the build seat gives up on (the controller keeps its copy in DEAD_LETTERS with the ask's
// message id) ends as error saying so, and its head asked again is a new try.
b.fake.hold.Store(head2)
lost, err := check.AskTry(store, map[string]string{repo: trunk}, check.TryAsk{Repository: repo, Head: head2}, time.Now())
if err != nil {
t.Fatal(err)
}
steps, err := b.js.KeyValue(b.ctx, delivery.StepsBucket("mesh-build-go"))
if err != nil {
t.Fatal(err)
}
askID := ""
for deadline := time.Now().Add(20 * time.Second); askID == "" && time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
keys, _ := steps.Keys(b.ctx)
for _, k := range keys {
if e, err := steps.Get(b.ctx, k); err == nil && strings.Contains(string(e.Value()), `"running"`) {
askID = strings.TrimPrefix(k, "n1.")
}
}
}
if askID == "" {
t.Fatal("the held try's ask was never taken")
}
dead := &nats.Msg{Subject: "mesh.events.dead.SEAT_NODE_BUILD_GO.SEAT_NODE_BUILD_GO_CHECK_worker", Header: nats.Header{}}
dead.Header.Set(buildrun.DeadHeaderMsgID, askID)
dead.Header.Set(buildrun.DeadHeaderDeliveries, "5")
dead.Header.Set(buildrun.DeadHeaderStream, "SEAT_NODE_BUILD_GO")
dead.Header.Set(buildrun.DeadHeaderGaveUp, time.Now().UTC().Format(time.RFC3339Nano))
if _, err := b.js.PublishMsg(b.ctx, dead); err != nil {
t.Fatal(err)
}
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
raw, _ := store.Get(lost)
if last, err = check.DecodeTry(raw); err == nil && last.Over() {
break
}
}
if last.State != check.TryError || !strings.Contains(last.Said, buildrun.LostWords+" after 5 deliveries") {
t.Fatalf("the try whose ask was given up = %+v, want error saying so", last)
}
if again, err := check.AskTry(store, map[string]string{repo: trunk}, check.TryAsk{Repository: repo, Head: head2}, time.Now()); err != nil || again == lost {
t.Errorf("its head asked again answered %q, %v, want a new try", again, err)
}
} }
// TestTheCheckerSaysOwedReadableOnTheBus: started, the checker says on its own states subject what it keeps, owed among // TestTheCheckerSaysOwedReadableOnTheBus: started, the checker says on its own states subject what it keeps, owed among
@@ -795,7 +702,7 @@ func TestTheCheckerSaysOwedReadableOnTheBus(t *testing.T) { // R2-1
break break
} }
if said.Module != "mesh-delivery-checker" || said.Node != "n1" || !said.Keeps("owed") || !said.Readable("owed") || if said.Module != "mesh-delivery-checker" || said.Node != "n1" || !said.Keeps("owed") || !said.Readable("owed") ||
!said.Readable("previews") || !said.Readable("verdicts") || !said.Readable("kept") || !said.Readable("tries") || said.Readable("hearing") || len(said.States) != 6 { !said.Readable("previews") || !said.Readable("verdicts") || !said.Readable("kept") || said.Readable("tries") || len(said.States) != 5 {
t.Fatalf("the checker said %+v, want its six states with kept, owed, previews, tries and verdicts readable, tries by the delivery view (1929/AC4; 1887 A5b; build_usage)", said) t.Fatalf("the checker said %+v, want its five states with kept, owed, previews and verdicts readable (1929/AC4; 1887 A5b)", said)
} }
} }
@@ -37,8 +37,8 @@ func TestTheCheckerAnnouncesEveryStateItKeepsAndOwedReadable(t *testing.T) { //
if !equal(manifest, announced) { if !equal(manifest, announced) {
t.Errorf("announced %v, the manifest keeps %v", announced, manifest) t.Errorf("announced %v, the manifest keeps %v", announced, manifest)
} }
// kept is read by the Go build module for a check ask (issue 1887 design limit 1); tries by the delivery view (build_usage). // kept is read by the Go build module for a check ask (issue 1887 design limit 1).
if want := []string{"kept", "owed", "previews", "tries", "verdicts"}; !equal(readable, want) { if want := []string{"kept", "owed", "previews", "verdicts"}; !equal(readable, want) {
t.Errorf("readable %v, want %v", readable, want) t.Errorf("readable %v, want %v", readable, want)
} }
} }
@@ -1,50 +0,0 @@
package main
import (
"sync"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-checker/internal/check"
)
// lazyTurns is the gate's merge turns as the checker reads them (novox/hq ADR 0548 §3), opened when the gate's bucket is
// there: before each claim it is opened again until it is, so a checker that started before the gate made the bucket holds
// checks from the gate's first turn on, with no timer and no restart. Until then it holds nothing, and says so once, loudly,
// and once again when it opens.
type lazyTurns struct {
turns *check.Turns
open func() error
logf func(format string, args ...any)
mu sync.Mutex
opened bool
said bool
}
// ensure opens the bucket once it is there; a failure is said once.
func (l *lazyTurns) ensure() {
l.mu.Lock()
defer l.mu.Unlock()
if l.opened || l.open == nil {
return
}
if err := l.open(); err != nil {
if !l.said {
l.said = true
l.logf("NO MERGE TURN HOLDS A CHECK: the gate's bucket %s cannot be read (%v); every check is claimed as if no approved merge waited, "+
"and the bucket is opened again before each claim until it is there", delivery.TurnsBucket, err)
}
return
}
l.opened = true
if l.said {
l.logf("the gate's merge turns are read from %s from now on: an approved merge's turn holds checks off its repositories again", delivery.TurnsBucket)
}
}
// Holding opens the bucket when it is not yet, then answers what the turns hold.
func (l *lazyTurns) Holding(repositories []string, heads string) (string, delivery.TurnWait, bool) {
l.ensure()
return l.turns.Holding(repositories, heads)
}
@@ -112,17 +112,14 @@ func durationEnv(getenv func(string) string, name string, def time.Duration) (ti
// run is the program: it returns when ctx ends or the bus cannot be used. // run is the program: it returns when ctx ends or the bus cannot be used.
// keptStates is every state the checker keeps and which others may read (ADR 0495 §2-§3): owed by the forge module, // keptStates is every state the checker keeps and which others may read (ADR 0495 §2-§3): owed by the forge module,
// which writes mesh/check from it (ADR 0521), and by the forge bridge; previews by the indexer; verdicts by the gate; tries by the delivery view. // which writes mesh/check from it (ADR 0521), and by the forge bridge; previews by the indexer; verdicts by the gate.
var keptStates = []announce.State{ var keptStates = []announce.State{
// The kept verdicts are read by the Go build module, which takes them for a check ask's unchanged packages (issue 1887's // The kept verdicts are read by the Go build module, which takes them for a check ask's unchanged packages (issue 1887's
// design limit 1); the checker alone writes them (ADR 0444 clause 5). // design limit 1); the checker alone writes them (ADR 0444 clause 5).
// Whether DEAD_LETTERS is heard (issue 734): read by the checker's own status tool alone.
{Name: "hearing"},
{Name: "kept", Readable: true}, {Name: "kept", Readable: true},
{Name: "owed", Readable: true}, {Name: "owed", Readable: true},
{Name: "previews", Readable: true}, {Name: "previews", Readable: true},
// The tries are read by the delivery view, which tells a try's asks from a candidate's by the asks each names (build_usage). {Name: "tries"},
{Name: "tries", Readable: true},
{Name: "verdicts", Readable: true}, {Name: "verdicts", Readable: true},
} }
@@ -155,12 +152,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil { if err != nil {
return err return err
} }
// Its own states, the candidates it checks, and the modules and manifests that say which build modules judge a repository: bucket := func(name string) (jetstream.KeyValue, error) {
// a check without one of them would be wrong or unwritten, so each stops the start, naming its grant (issue 1945). kv, err := js.KeyValue(ctx, name)
if err != nil {
return nil, fmt.Errorf("the bucket %s: %w", name, err)
}
return kv, nil
}
buckets := map[string]jetstream.KeyValue{} buckets := map[string]jetstream.KeyValue{}
for _, name := range []string{delivery.CheckerVerdictsBucket, delivery.CheckerKeptBucket, delivery.CheckerOwedBucket, delivery.PreviewsBucket, for _, name := range []string{delivery.CheckerVerdictsBucket, delivery.CheckerKeptBucket, delivery.CheckerOwedBucket, delivery.PreviewsBucket,
delivery.CandidatesBucket, delivery.ModulesBucket, delivery.ManifestsBucket, check.TriesBucket} { delivery.CandidatesBucket, delivery.ModulesBucket, delivery.ManifestsBucket, check.TriesBucket} {
if buckets[name], err = process.EssentialBucket(ctx, js, "checker", name, "judge or keep a check"); err != nil { if buckets[name], err = bucket(name); err != nil {
return err return err
} }
} }
@@ -196,28 +198,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
tools := &buildrun.ToolchainSet{} tools := &buildrun.ToolchainSet{}
presence := &buildrun.PresenceSet{} presence := &buildrun.PresenceSet{}
trunks := &check.Trunks{} trunks := &check.Trunks{}
turns := &check.Turns{}
lazy := &lazyTurns{turns: turns, logf: logger.Printf}
// A check is one ask per pull request and build module on the build seats' check stream, pulled by a build node with a // A check is one ask per pull request and build module on the build seats' check stream, pulled by a build node with a
// free slot (novox/hq ADR 0525 §1, §3): the checker chooses no node and starts no step. // free slot (novox/hq ADR 0525 §1, §3): the checker chooses no node and starts no step.
asks := buildrun.JetStreamAsks{JS: js} asks := buildrun.JetStreamAsks{JS: js}
// The asks the build seats gave up on (issue 734): heard from DEAD_LETTERS, each ends the step its asker waits for.
// Heard in the background, and again at a reconnect or an ask while it is not; whether it is heard is kept in the
// state hearing, which checker_status says (dead_letters).
dead := &buildrun.DeadLetters{}
// The state may not be on the bus yet when a delivery started the checker (the grants part creates it, issue 1790): it
// is bound when the bus says it was created, and the latest hearing is written then.
hearingKV := process.AwaitBucket(ctx, js, conn, check.HearingBucket, "checker_status cannot say whether DEAD_LETTERS is heard", logger.Printf)
hearing := buildrun.NewDeadLetterHearing(ctx, buildrun.JetStreamDeadLetters{JS: js}, []string{delivery.AskGo, delivery.AskNode}, dead,
logger.Printf, buildrun.SayHearing(ctx, hearingKV, logger.Printf))
defer hearing.Stop()
conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again() })
waiter := buildrun.LostAskWaiter{Inner: asks, Dead: dead, Hearing: hearing}
ch := &check.Checker{ ch := &check.Checker{
Verdicts: kvStore{buckets[delivery.CheckerVerdictsBucket]}, Verdicts: kvStore{buckets[delivery.CheckerVerdictsBucket]},
Owed: kvStore{buckets[delivery.CheckerOwedBucket]}, Owed: kvStore{buckets[delivery.CheckerOwedBucket]},
Trunks: trunks, Trunks: trunks,
Turns: lazy,
Bound: checkBound, Bound: checkBound,
Link: link, Link: link,
Logf: logger.Printf, Logf: logger.Printf,
@@ -225,7 +212,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// The builder's builds say how the candidates' fingerprints stand (issue 1684); where its bucket is not there yet the line // The builder's builds say how the candidates' fingerprints stand (issue 1684); where its bucket is not there yet the line
// says nothing of the builds, until the checker starts again. // says nothing of the builds, until the checker starts again.
var builds check.BuildStates var builds check.BuildStates
if kv := process.OptionalBucket(ctx, js, delivery.BuilderBuildsBucket, "the line says nothing of the builds until the checker starts again", logger.Printf); kv != nil { switch kv, err := js.KeyValue(ctx, delivery.BuilderBuildsBucket); {
case errors.Is(err, jetstream.ErrBucketNotFound):
logger.Printf("the bucket %s does not exist yet: the line says nothing of the builds until the checker starts again", delivery.BuilderBuildsBucket)
case err != nil:
return fmt.Errorf("the bucket %s: %w", delivery.BuilderBuildsBucket, err)
default:
builds = buildReader{kvStore{kv}} builds = buildReader{kvStore{kv}}
} }
// The assigner's compositions (ADR 0528 clause 5, issue 1297): their bucket is looked up at each check until it is // The assigner's compositions (ADR 0528 clause 5, issue 1297): their bucket is looked up at each check until it is
@@ -246,7 +238,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return cw, nil return cw, nil
}} }}
// The tries (ADR 0501 §5) are judged by the same judge, held by the same deliveries. // The tries (ADR 0501 §5) are judged by the same judge, held by the same deliveries.
tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, NotTaken: stepBound, Logf: logger.Printf} tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, Logf: logger.Printf}
announce := func(ctx context.Context, event, key string) { announce := func(ctx context.Context, event, key string) {
body, _ := json.Marshal(map[string]string{"key": key}) body, _ := json.Marshal(map[string]string{"key": key})
if _, err := js.Publish(ctx, eventSubject(event), body); err != nil { if _, err := js.Publish(ctx, eventSubject(event), body); err != nil {
@@ -254,9 +246,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
} }
} }
repos := &repoModules{manifests: kvStore{buckets[delivery.ManifestsBucket]}, logf: logger.Printf} repos := &repoModules{manifests: kvStore{buckets[delivery.ManifestsBucket]}, logf: logger.Printf}
ch.Runs = repos // the node-engine a check reads beside at its running commit (ADR 0548 §6)
ch.Judge = newJudge(judgeDeps{ ch.Judge = newJudge(judgeDeps{
asks: buildrun.Asker{Publisher: asks, Waiter: waiter, Bound: stepBound}, tools: tools, trunks: trunks, asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: stepBound}, tools: tools, trunks: trunks,
kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]}, kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]},
repos: repos, builds: builds, forge: forgeReads{call: caller.Call}, logf: logger.Printf, repos: repos, builds: builds, forge: forgeReads{call: caller.Call}, logf: logger.Printf,
announce: announce, compositions: compositions, announce: announce, compositions: compositions,
@@ -299,75 +290,55 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// the assigner's own presence says whether a composition is being made (compositionWatch, ADR 0537 §4). A check is no // the assigner's own presence says whether a composition is being made (compositionWatch, ADR 0537 §4). A check is no
// longer held for a node: its asks wait in the build seats' streams, which redeliver a lost node's ask (ADR 0525 §6, // longer held for a node: its asks wait in the build seats' streams, which redeliver a lost node's ask (ADR 0525 §6,
// issue 1912), so the node-engine's have is not watched here. // issue 1912), so the node-engine's have is not watched here.
if kv := process.OptionalBucket(ctx, js, delivery.PresenceBucket, "no node counts as online until the checker starts again", logger.Printf); kv != nil { switch kv, err := js.KeyValue(ctx, delivery.PresenceBucket); {
if pw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { presence.Put(e.Key(), e.Value(), !opOf(e)) }); err != nil { case errors.Is(err, jetstream.ErrBucketNotFound):
process.NotWatched(logger.Printf, delivery.PresenceBucket, err, "no node counts as online until the checker starts again") logger.Printf("the bucket %s does not exist yet: no node counts as online until the checker starts again", delivery.PresenceBucket)
} else { case err != nil:
go func() { return fmt.Errorf("the bucket %s: %w", delivery.PresenceBucket, err)
errs <- follow(ctx, pw, func(e jetstream.KeyValueEntry) { default:
if presence.Put(e.Key(), e.Value(), !opOf(e)) { pw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { presence.Put(e.Key(), e.Value(), !opOf(e)) })
ch.NodesChanged(ctx) // a node came online or went: a check whose forge was not heard may now be heard if err != nil {
} return err
})
}()
} }
go func() {
errs <- follow(ctx, pw, func(e jetstream.KeyValueEntry) {
if presence.Put(e.Key(), e.Value(), !opOf(e)) {
ch.NodesChanged(ctx) // a node came online or went: a check whose forge was not heard may now be heard
}
})
}()
} }
// The gate's source says the trunk head of each repository, which a pull request is merged onto. Where it is not there // The gate's source says the trunk head of each repository, which a pull request is merged onto. Where it is not there
// yet no trunk is known, so nothing is checked. // yet no trunk is known, so nothing is checked.
if kv := process.OptionalBucket(ctx, js, delivery.SourceBucket, "no trunk is known, so nothing is checked until the checker starts again", logger.Printf); kv != nil { switch kv, err := js.KeyValue(ctx, delivery.SourceBucket); {
case errors.Is(err, jetstream.ErrBucketNotFound):
logger.Printf("the bucket %s does not exist yet: no trunk is known until the checker starts again", delivery.SourceBucket)
case err != nil:
return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err)
default:
sw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { sw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) {
if e.Key() == delivery.SourceMainKey { if e.Key() == delivery.SourceMainKey {
trunks.Put(e.Value(), !opOf(e)) trunks.Put(e.Value(), !opOf(e))
} }
}) })
if err != nil {
process.NotWatched(logger.Printf, delivery.SourceBucket, err, "no trunk is known, so nothing is checked until the checker starts again")
} else {
go func() {
errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) {
if e.Key() == delivery.SourceMainKey && trunks.Put(e.Value(), !opOf(e)) {
ch.TrunkMoved(ctx) // the trunk moved: its statuses go pending, and a verdict for the old trunk is judged again
}
})
}()
}
}
// The gate's merge turns (novox/hq ADR 0548 §3): no check is claimed of a repository an approved merge waits on, and the
// delete of its turn claims it. Read before the candidates, so no check starts past a turn written before this start.
// Where the gate's bucket is not there yet it is read as empty, and opened again before each claim until it is there
// (lazyTurns), so a gate delivered after the checker started holds checks from its first turn on.
lazy.open = func() error {
kv, err := js.KeyValue(ctx, delivery.TurnsBucket)
if err != nil {
return err
}
tw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { turns.Put(e.Key(), e.Value(), !opOf(e)) })
if err != nil { if err != nil {
return err return err
} }
go func() { go func() {
errs <- follow(ctx, tw, func(e jetstream.KeyValueEntry) { errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) {
if turns.Put(e.Key(), e.Value(), !opOf(e)) { if e.Key() == delivery.SourceMainKey && trunks.Put(e.Value(), !opOf(e)) {
ch.Reconsider(ctx) // a turn went or changed: a check it held is claimed now ch.TrunkMoved(ctx) // the trunk moved: its statuses go pending, and a verdict for the old trunk is judged again
} }
}) })
}() }()
return nil
} }
lazy.ensure()
// The modules say which repository holds a TypeScript module, so the Node build module judges it too. // The modules say which repository holds a TypeScript module, so the Node build module judges it too.
mw, err := settle(ctx, buckets[delivery.ModulesBucket], func(e jetstream.KeyValueEntry) { repos.Put(ctx, e.Key(), e.Value(), !opOf(e)) }) mw, err := settle(ctx, buckets[delivery.ModulesBucket], func(e jetstream.KeyValueEntry) { repos.Put(ctx, e.Key(), e.Value(), !opOf(e)) })
if err != nil { if err != nil {
return err return err
} }
go func() { go func() {
errs <- follow(ctx, mw, func(e jetstream.KeyValueEntry) { errs <- follow(ctx, mw, func(e jetstream.KeyValueEntry) { repos.Put(ctx, e.Key(), e.Value(), !opOf(e)) })
before := repos.CommitOf(check.NodeEngineModule)
repos.Put(ctx, e.Key(), e.Value(), !opOf(e))
if after := repos.CommitOf(check.NodeEngineModule); after != before {
ch.TrunkMoved(ctx) // the node-engine the mesh runs moved: a verdict that read it beside is carried (ADR 0548 §6)
}
})
}() }()
each := func(e jetstream.KeyValueEntry) { each := func(e jetstream.KeyValueEntry) {
if opOf(e) { if opOf(e) {
@@ -193,9 +193,7 @@ func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate,
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) } each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
w, err := settle(ctx, kv, each) w, err := settle(ctx, kv, each)
if err != nil { if err != nil {
// Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read. return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err)
logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
return errs, nil
} }
if gate.Settled() { if gate.Settled() {
changed(ctx) changed(ctx)
@@ -3,19 +3,11 @@ package main
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940). // The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
import ( import (
"bytes"
"context"
"errors"
"fmt" "fmt"
"log"
"strings" "strings"
"testing" "testing"
"time" "time"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
) )
@@ -97,51 +89,6 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
if n := len(set.Entries("typescript")); n != 2 { if n := len(set.Entries("typescript")); n != 2 {
t.Fatalf("novox and shanks are left; got %d entries", n) t.Fatalf("novox and shanks are left; got %d entries", n)
} }
// 1945 (a part exits at start for a bucket it works without): a bus that refuses the assignments' watch, or every
// toolchains bucket, leaves the watches unstarted and the part running, said; a refused bucket it cannot work without
// is an error naming the bucket and its grant.
var logs bytes.Buffer
logger := log.New(&logs, "", 0)
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments")}
if _, err := watchAssignments(context.Background(), refused, newToolchainGate(nil, set.Put, t.Logf), func(context.Context) {}, logger); err != nil {
t.Errorf("a refused watch of the assignments stopped the start: %v", err)
}
if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil {
t.Errorf("a refused toolchains bucket stopped the start: %v", err)
}
if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") {
t.Errorf("the refusals were not said: %q", logs.String())
}
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "checker", "mesh-delivery-indexer_modules", "work"); err == nil ||
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
}
}
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
type refusingBus struct {
refuseOpen bool
err error
}
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
if r.refuseOpen {
return nil, r.err
}
return refusedWatch{err: r.err}, nil
}
type refusedWatch struct {
jetstream.KeyValue
err error
}
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
return nil, r.err
}
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
return nil, r.err
} }
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not // A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
@@ -8,8 +8,6 @@ import (
"strings" "strings"
"sync" "sync"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream" "github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery"
@@ -65,14 +63,11 @@ func (w *toolchainWatch) ensure(module string) error {
w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module) w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module)
return nil return nil
case err != nil: case err != nil:
// Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the module's next report. return fmt.Errorf("the bucket %s: %w", name, err)
w.logger.Printf("the bucket %s cannot be read (%v): %s's toolchains are not heard until its next report", name, err, module)
return nil
} }
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.put(module, e.Key(), e.Value(), !opOf(e)) }) watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.put(module, e.Key(), e.Value(), !opOf(e)) })
if err != nil { if err != nil {
process.NotWatched(w.logger.Printf, name, err, fmt.Sprintf("%s's toolchains are not heard until its next report", module)) return err
return nil
} }
w.watched[module] = true w.watched[module] = true
go func() { go func() {
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-checker
go 1.26.0 go 1.26.0
require ( require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142434-65ee43f07583 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93
github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0 github.com/nats-io/nats.go v1.54.0
) )
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142434-65ee43f07583 h1:ikD/ggf9/5k23sxJOX1PFcGzb4M3+QeC66AT0MyQ9Zs= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93 h1:FAiFY2KBKEoZSH49OZTt7/zRBIQxyl5g/KhYrpggbN4=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142434-65ee43f07583/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -1,355 +0,0 @@
package check
import (
"context"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// One check per delivery (novox/hq ADR 0548): the checker's part. A merge turn holds new checks off its repository (§3);
// a trunk or sibling move never ends a running check and carries its verdict once it ended (§1, §5, §6); only a head git
// refused gets a whole check after a move (§7); the line of a carried verdict says so (§8).
// turnOf is a turn's value on a repository for a delivery of the heads hash.
func turnOf(t testing.TB, repo, group, heads string) []byte {
t.Helper()
at := time.Date(2026, 10, 26, 9, 0, 0, 0, time.UTC)
data, err := delivery.EncodeTurn(delivery.Turn{Repository: repo, Waiting: []delivery.TurnWait{{Group: group, Heads: heads, Approved: at, Since: at}}})
if err != nil {
t.Fatal(err)
}
return data
}
func turnKey(t testing.TB, repo string) string {
t.Helper()
k, err := delivery.TurnsKey(repo)
if err != nil {
t.Fatal(err)
}
return k
}
// waitFor polls a condition the checker's goroutines make true, failing after two seconds.
func waitFor(t testing.TB, what string, ok func() bool) {
t.Helper()
for deadline := time.Now().Add(2 * time.Second); !ok(); time.Sleep(time.Millisecond) {
if time.Now().After(deadline) {
t.Fatalf("%s did not happen", what)
}
}
}
// pullWithFP is a pull request touching the module m at the fingerprint `src2:` and 64 times the hex digit d.
func pullWithFP(repo string, number int, head string, d string) delivery.CandidatePull {
p := pullOf(repo, number, head)
p.Modules = []string{"m"}
p.Fps = map[string]delivery.CandidateModule{"m": {FP2: "src2:" + strings.Repeat(d, 64)}}
return p
}
func TestOneCheckPerDelivery(t *testing.T) {
ctx := context.Background()
for name, run := range map[string]func(t *testing.T){
// §3: no claim while a turn names one of its repositories; a claim on the turn's delete event.
"a turn on its repository holds a new check off until the turn goes": func(t *testing.T) {
j := passJudge()
r := newRig(t, j)
turns := &Turns{}
r.ch.Turns = turns
turns.Put(turnKey(t, repoA), turnOf(t, repoA, "fix/other", "0123456789ab"), false)
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
if j.count() != 0 || len(r.verdicts.vals) != 0 {
t.Fatalf("judged %d times with a turn on %s, verdicts %v", j.count(), repoA, r.verdicts.vals)
}
if held := r.ch.Held()[keyOf(t, "feat/x", cand)]; !strings.Contains(held, repoA) || !strings.Contains(held, "fix/other") {
t.Errorf("the hold is said as %q, want the turn's repository and group", held)
}
if !turns.Put(turnKey(t, repoA), nil, true) {
t.Fatal("the turn's delete changed nothing")
}
r.ch.Reconsider(ctx) // what the watch of turns does on the delete
r.ch.Wait()
if j.count() != 1 || r.verdict(t, keyOf(t, "feat/x", cand)).State != delivery.VerdictPass {
t.Errorf("judged %d times after the turn went, want once", j.count())
}
},
"a turn on another repository, or of its own heads, holds nothing": func(t *testing.T) {
j := passJudge()
r := newRig(t, j)
turns := &Turns{}
r.ch.Turns = turns
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
c, _ := delivery.DecodeCandidate(cand)
turns.Put(turnKey(t, repoSDK), turnOf(t, repoSDK, "fix/other", "0123456789ab"), false)
turns.Put(turnKey(t, repoA), turnOf(t, repoA, "feat/x", GateHeads(c)), false)
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
if j.count() != 1 {
t.Errorf("judged %d times, want once: the turns hold another repository and its own merge", j.count())
}
},
// §3, the race: a check claimed before the gate's turn is judged on the commit it read, never restarted.
"a check claimed before a turn was written runs to its verdict": func(t *testing.T) {
release, started := make(chan struct{}), make(chan struct{})
j := &judgeFunc{f: func(context.Context, *Case, int) (Outcome, error) {
close(started)
<-release
return Outcome{Verdict: passing()}, nil
}}
r := newRig(t, j)
turns := &Turns{}
r.ch.Turns = turns
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
<-started
turns.Put(turnKey(t, repoA), turnOf(t, repoA, "fix/other", "0123456789ab"), false)
r.ch.Reconsider(ctx)
close(release)
r.ch.Wait()
if j.count() != 1 || r.verdict(t, keyOf(t, "feat/x", cand)).State != delivery.VerdictPass {
t.Errorf("judged %d times; the running check is not ended by the turn", j.count())
}
},
// §1, §5: a trunk move during a running check calls the judge once, writes its verdict, keeps the status pending (a pass
// on an old trunk is never shown), then asks one carry, on the recheck stream, and no check.
"a trunk move during a check ends nothing and asks one carry after it": func(t *testing.T) {
var r *rig
var kinds []string
var from []map[string]string
judged := make(chan struct{})
j := &judgeFunc{f: func(ctx context.Context, c *Case, call int) (Outcome, error) {
kinds, from = append(kinds, c.Kind), append(from, c.CarriedFrom)
if call == 1 {
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) // the trunk moves while it runs
r.ch.TrunkMoved(ctx) // as the watch of the source says it
close(judged)
} else if s := owedState(t, r, repoA, 7, head1); s != "pending" {
t.Errorf("the status during the carry is %q, want pending: the pass on the old trunk is never shown", s)
}
return Outcome{Verdict: passing()}, nil
}}
r = newRig(t, j)
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
<-judged
r.ch.Wait()
v := r.verdict(t, keyOf(t, "feat/x", cand))
if j.count() != 2 || kinds[0] != KindNew || kinds[1] != KindTrunk || from[0] != nil || from[1][repoA] != trunkA {
t.Fatalf("judged %d times as %v carried from %v; want a whole check, then one carry from %s", j.count(), kinds, from, trunkA[:12])
}
if v.State != delivery.VerdictPass || v.Trunk[repoA] != trunkA2 || v.CarriedFrom[repoA] != trunkA {
t.Errorf("the verdict is %s on %v carried from %v; want a pass on the new trunk carried from the old", v.State, v.Trunk, v.CarriedFrom)
}
if s := owedState(t, r, repoA, 7, head1); s != "success" {
t.Errorf("the status after the carry is %q, want success", s)
}
if r.verdicts.creates[keyOf(t, "feat/x", cand)] != 1 {
t.Error("the verdict key was made again: a carry rewrites it")
}
},
// §5: the composition is kept when no fingerprint of a module it composed changed, and runs again otherwise.
"a carry keeps a passed composition while the fingerprints stay the same": func(t *testing.T) {
var kept []bool
j := &judgeFunc{f: func(_ context.Context, c *Case, _ int) (Outcome, error) {
kept = append(kept, c.Composition != nil)
v := passing()
v.Composition = &delivery.Composition{State: delivery.CompositionPass, SDK: "v0.1.14-x", FPSet: c.FPSet}
return Outcome{Verdict: v}, nil
}}
r := newRig(t, j)
cand := candidateOf(t, "feat/x", pullWithFP(repoA, 7, head1, "a"))
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS})
r.ch.TrunkMoved(ctx) // the indexer's fingerprints of the candidate stay the same
r.ch.Wait()
r.ch.Candidate(ctx, "feat%2Fx", candidateOf(t, "feat/x", pullWithFP(repoA, 7, head1, "b"))) // a fingerprint changed with the move
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS})
r.ch.TrunkMoved(ctx)
r.ch.Wait()
if len(kept) != 3 || kept[0] || !kept[1] || kept[2] {
t.Errorf("the composition was kept in the checks %v; want composed, kept on the carry, composed again once a fingerprint changed", kept)
}
},
// §6: a sibling's move puts the verdict out of date, its status pending saying it is carried, and the carry's verdict
// names the sibling's old commit; the status follows the carry's verdict. Which packages run is the digest's (only
// those that name MESH_CHECK_BESIDE change with a sibling), the build module's part.
"a sibling's move is carried and the status follows the carry": func(t *testing.T) {
var r *rig
var from map[string]string
j := &judgeFunc{f: func(_ context.Context, c *Case, call int) (Outcome, error) {
if call == 2 {
from = c.CarriedFrom
if s := owedState(t, r, repoA, 7, head1); s != "pending" {
t.Errorf("the status during the sibling's carry is %q, want pending", s)
}
}
return Outcome{Verdict: passing(), Beside: map[string]string{catalogueRepository: r.trunks.Heads()[catalogueRepository]}}, nil
}}
r = newRig(t, j)
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS, catalogueRepository: trunkS})
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
if b := r.verdict(t, keyOf(t, "feat/x", cand)).Beside; b[catalogueRepository] != trunkS {
t.Fatalf("the verdict names the siblings %v, want the catalogue at %s", b, trunkS[:12])
}
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS, catalogueRepository: trunkS2})
r.ch.TrunkMoved(ctx)
r.ch.Wait()
v := r.verdict(t, keyOf(t, "feat/x", cand))
if j.count() != 2 || from[catalogueRepository] != trunkS || v.CarriedFrom[catalogueRepository] != trunkS || v.Beside[catalogueRepository] != trunkS2 {
t.Errorf("judged %d times, carried from %v, verdict beside %v carried from %v; want one carry over the catalogue's move", j.count(), from, v.Beside, v.CarriedFrom)
}
if s := owedState(t, r, repoA, 7, head1); s != "success" {
t.Errorf("the status after the carry is %q, want success", s)
}
},
// §7: a carry whose merge conflicts fails naming the files and asks no check; a later move onto which the head merges
// cleanly asks a whole check, not a carry.
"a carry git refuses fails naming the files, and the next move asks a whole check": func(t *testing.T) {
var kinds []string
var from []map[string]string
j := &judgeFunc{f: func(_ context.Context, c *Case, call int) (Outcome, error) {
kinds, from = append(kinds, c.Kind), append(from, c.CarriedFrom)
if call == 2 {
return Outcome{Verdict: delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "ace",
Words: repoA + "#7 " + ConflictWords + "go.mod"}}}, nil
}
return Outcome{Verdict: passing()}, nil
}}
r := newRig(t, j)
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS})
r.ch.TrunkMoved(ctx)
r.ch.Wait()
r.ch.Reconsider(ctx) // anything else that looks again asks nothing of a conflict
r.ch.Wait()
if v := r.verdict(t, keyOf(t, "feat/x", cand)); j.count() != 2 || !IsConflictFailure(v) || !strings.Contains(v.Failed.Words, "go.mod") {
t.Fatalf("judged %d times, verdict %+v; want the carry failed naming go.mod and nothing asked after it", j.count(), v.Failed)
}
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS})
r.ch.TrunkMoved(ctx)
r.ch.Wait()
if j.count() != 3 || kinds[1] != KindTrunk || from[1] == nil || kinds[2] != KindNew || from[2] != nil {
t.Errorf("checks %v carried from %v; want a whole check, a carry, then a whole check after the conflict", kinds, from)
}
},
// §8, §6: the line of a carried verdict names the head on the new trunk commit first, as the gate and the merge tool
// read a line, then where it was carried from and what the carry ran and kept; while it is carried it is pending and says so.
"the line of a carry says where it was carried from": func(t *testing.T) {
cs, p := lineCase(t, pullOf(repoA, 7, head1))
v := passVerdict()
v.Trunk = map[string]string{repoA: trunkA}
v.Beside = map[string]string{catalogueRepository: trunkS2}
v.CarriedFrom = map[string]string{repoA: trunkA2, catalogueRepository: trunkS}
line := Line(v, cs, p, nil, nil, nil)
if !strings.HasPrefix(line, "pass 111111111111 on aaaaaaaaaaaa | carried from abababababab, mesh-catalog 555555555555 | unit 12 run") {
t.Errorf("the carried line = %q", line)
}
if head, trunk := checkWordsOf(line); head != head1[:12] || trunk != trunkA[:12] {
t.Errorf("the line names %s on %s first, want the head on the new trunk", head, trunk)
}
cs.CarriedFrom = map[string]string{repoA: trunkA2}
pending := Line(delivery.Verdict{State: delivery.VerdictRunning}, cs, p, nil, nil, nil)
if !strings.HasPrefix(pending, "being carried to 111111111111 on aaaaaaaaaaaa | from abababababab") {
t.Errorf("the pending line of a carry = %q", pending)
}
},
// §3 (review round 1): two approved merges in one turn whose trunk moved each get their carry; neither holds the other's.
"two merges waiting in one turn each get their carry": func(t *testing.T) {
var r *rig
j := &judgeFunc{f: func(context.Context, *Case, int) (Outcome, error) { return Outcome{Verdict: passing()}, nil }}
r = newRig(t, j)
candA, candC := candidateOf(t, "feat/a", pullOf(repoA, 7, head1)), candidateOf(t, "feat/c", pullOf(repoA, 8, head2))
r.ch.Candidate(ctx, "feat%2Fa", candA)
r.ch.Candidate(ctx, "feat%2Fc", candC)
r.ch.Wait()
a, _ := delivery.DecodeCandidate(candA)
c, _ := delivery.DecodeCandidate(candC)
at := time.Date(2026, 10, 26, 9, 0, 0, 0, time.UTC)
turn, err := delivery.EncodeTurn(delivery.Turn{Repository: repoA, Waiting: []delivery.TurnWait{
{Group: "feat/a", Heads: GateHeads(a), Approved: at, Since: at}, {Group: "feat/c", Heads: GateHeads(c), Approved: at.Add(time.Minute), Since: at}}})
if err != nil {
t.Fatal(err)
}
turns := &Turns{}
turns.Put(turnKey(t, repoA), turn, false)
r.ch.Turns = turns
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) // a third merge moves the trunk under both
r.ch.TrunkMoved(ctx)
r.ch.Wait()
for _, cand := range [][]byte{candA, candC} {
group := map[bool]string{true: "feat/a", false: "feat/c"}[string(cand) == string(candA)]
if v := r.verdict(t, keyOf(t, group, cand)); v.Trunk[repoA] != trunkA2 || v.CarriedFrom[repoA] != trunkA {
t.Errorf("%s: verdict on %v carried from %v; want its carry claimed and done", group, v.Trunk, v.CarriedFrom)
}
}
},
// §6 (review round 1): the node-engine read at the commit the mesh runs is named, and a new running commit is carried.
"a new running node-engine is carried": func(t *testing.T) {
runs := runsAt{nodeEngineModule: trunkS}
j := &judgeFunc{f: func(_ context.Context, c *Case, _ int) (Outcome, error) {
return Outcome{Verdict: passing(), Beside: map[string]string{nodeEngineRepo: runs[nodeEngineModule]}}, nil
}}
r := newRig(t, j)
r.ch.Runs = runs
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
r.ch.TrunkMoved(ctx) // nothing moved: nothing is carried
r.ch.Wait()
runs[nodeEngineModule] = trunkS2
r.ch.TrunkMoved(ctx) // what the indexer's watch does when mesh-host's running commit changes
r.ch.Wait()
v := r.verdict(t, keyOf(t, "feat/x", cand))
if j.count() != 2 || v.CarriedFrom[nodeEngineRepo] != trunkS || v.Beside[nodeEngineRepo] != trunkS2 {
t.Errorf("judged %d times, verdict beside %v carried from %v; want one carry over the node-engine's new running commit", j.count(), v.Beside, v.CarriedFrom)
}
},
// §5: a carry that fails names the move it failed on.
"a carry that fails names the move": func(t *testing.T) {
j := &judgeFunc{f: func(_ context.Context, c *Case, call int) (Outcome, error) {
if call == 1 {
return Outcome{Verdict: passing()}, nil
}
return Outcome{Verdict: delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Test: "unit: m/a.TestX", Words: "want 4"}}}, nil
}}
r := newRig(t, j)
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS})
r.ch.TrunkMoved(ctx)
r.ch.Wait()
v := r.verdict(t, keyOf(t, "feat/x", cand))
if v.State != delivery.VerdictFail || v.CarriedFrom != nil || !strings.HasPrefix(v.Failed.Words, "want 4") ||
!strings.Contains(v.Failed.Words, repoA+" "+trunkA[:12]+" to "+trunkA2[:12]) {
t.Errorf("the failed carry is %s carried from %v with words %q; want the test's words, then the move", v.State, v.CarriedFrom, v.Failed.Words)
}
},
} {
t.Run(name, run)
}
}
// checkWordsOf is the head and trunk a line names first, as the gate reads it (mesh-delivery-gate onestatus.go checkLine).
func checkWordsOf(line string) (string, string) {
for _, part := range strings.Split(line, " | ") {
f := strings.Fields(part)
for i := 0; i+2 < len(f); i++ {
if f[i+1] == "on" && len(f[i]) == 12 && len(f[i+2]) == 12 {
return f[i], f[i+2]
}
}
}
return "", ""
}
@@ -127,9 +127,6 @@ func (ch *Checker) Composed(ctx context.Context, comp delivery.CompositionVerdic
return return
} }
withComposition(&v, comp) withComposition(&v, comp)
if v.Composition != nil && v.Composition.State != delivery.CompositionNotRun {
v.Composition.FPSet = cs.FPSet
}
data, err := delivery.EncodeVerdict(v) data, err := delivery.EncodeVerdict(v)
if err != nil { if err != nil {
ch.logf("%s: the verdict with its composition cannot be written: %v", cs.Key, err) ch.logf("%s: the verdict with its composition cannot be written: %v", cs.Key, err)
@@ -12,18 +12,8 @@ type BesideRef struct {
Repository string `json:"repository"` Repository string `json:"repository"`
Commit string `json:"commit"` Commit string `json:"commit"`
Ref string `json:"ref,omitempty"` Ref string `json:"ref,omitempty"`
// atTrunk says the sibling was read at its trunk's head, so a move of that trunk puts the verdict out of date and is
// carried (novox/hq ADR 0548 §6). Never sent: the node-engine at the commit the mesh runs, mesh-sdk at a pin and a
// group's member at its head do not move with a trunk.
atTrunk bool
// runs says the sibling was read at the commit the mesh runs (the node-engine, the indexer's record of mesh-host): a new
// running commit puts the verdict out of date and is carried (ADR 0548 §6). Never sent.
runs bool
} }
// NodeEngineModule is the module whose running commit the node-engine sibling is read at.
const NodeEngineModule = nodeEngineModule
const ( const (
catalogueRepository = "novox/mesh-catalog" catalogueRepository = "novox/mesh-catalog"
nodeEngineRepo = "novox/mesh-host" nodeEngineRepo = "novox/mesh-host"
@@ -61,21 +51,18 @@ func (j *Examiner) besideFor(ctx context.Context, c *Case, p Pull) ([]BesideRef,
switch { switch {
case repo == catalogueRepository: case repo == catalogueRepository:
if head := heads[repo]; head != "" { if head := heads[repo]; head != "" {
out = append(out, BesideRef{Repository: repo, Commit: head, atTrunk: c.member(repo) == nil}) out = append(out, BesideRef{Repository: repo, Commit: head})
} }
case repo == nodeEngineRepo: case repo == nodeEngineRepo:
commit := "" commit := ""
if j.Runs != nil { if j.Runs != nil {
commit = j.Runs.CommitOf(nodeEngineModule) commit = j.Runs.CommitOf(nodeEngineModule)
} }
atTrunk := false
if commit == "" { if commit == "" {
commit, atTrunk = heads[repo], true commit = heads[repo]
} }
if commit != "" { if commit != "" {
// Named in the verdict either way (ADR 0548 §6, issue 568's own example): at the commit the mesh runs, a new out = append(out, BesideRef{Repository: repo, Commit: commit})
// running commit puts the verdict out of date; at the trunk, a trunk move does (Checker.besideNow).
out = append(out, BesideRef{Repository: repo, Commit: commit, atTrunk: atTrunk, runs: !atTrunk})
} }
case repo == SDKRepository && p.Repository == controllerRepo && j.Forge != nil: case repo == SDKRepository && p.Repository == controllerRepo && j.Forge != nil:
text, err := j.Forge.File(ctx, p.Repository, "go.mod", p.Head) text, err := j.Forge.File(ctx, p.Repository, "go.mod", p.Head)
@@ -95,7 +82,7 @@ func (j *Examiner) besideFor(ctx context.Context, c *Case, p Pull) ([]BesideRef,
if member := c.member(repo); member != nil { if member := c.member(repo); member != nil {
out = append(out, BesideRef{Repository: repo, Commit: member.Head, Ref: refOf(member.Number)}) out = append(out, BesideRef{Repository: repo, Commit: member.Head, Ref: refOf(member.Number)})
} else if head := heads[repo]; head != "" { } else if head := heads[repo]; head != "" {
out = append(out, BesideRef{Repository: repo, Commit: head, atTrunk: true}) out = append(out, BesideRef{Repository: repo, Commit: head})
} else if repo == controllerRepo && p.Repository == nodeEngineRepo { } else if repo == controllerRepo && p.Repository == nodeEngineRepo {
// The node-engine's step-nine test judges the controller's own manifest and fails without it (issue 1295): a // The node-engine's step-nine test judges the controller's own manifest and fails without it (issue 1295): a
// checker that cannot place it says so, where placing none left that test to skip. // checker that cannot place it says so, where placing none left that test to skip.
@@ -121,13 +108,3 @@ func refOf(number int) string {
} }
return "refs/pull/" + strconv.Itoa(number) + "/head" return "refs/pull/" + strconv.Itoa(number) + "/head"
} }
// besideAtTrunk are the siblings of refs read at their trunk's head that the case does not merge, by repository: what a
// verdict names as Beside (ADR 0548 §6).
func besideAtTrunk(c *Case, refs []BesideRef, into map[string]string) {
for _, r := range refs {
if _, merged := c.Trunk[r.Repository]; (r.atTrunk || r.runs) && !merged {
into[r.Repository] = r.Commit
}
}
}
@@ -1,79 +0,0 @@
package check
import (
"fmt"
"sort"
"sync"
"sync/atomic"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// ReadWorkers is how many values a read tool asks the bus for at once. Each value is a round trip from the tool through the
// runtime to the bus, so reading a bucket of thousands of keys one after the other outlasts the 30 s a tool call is given
// (issue 1827, as the delivery view's issue 1945). It is a copy of the view's: a few lines are not worth a group delivery
// through mesh-sdk (rules-are-weighed).
const ReadWorkers = 16
// MaxReads bounds the values one call of a read tool reads; past it the answer says it was cut and how to narrow it.
const MaxReads = 1000
// parallel runs read(i) for i in [0, n), at most ReadWorkers at a time, and answers the first error; once one failed no
// further read starts.
func parallel(n int, read func(i int) error) error {
var (
next atomic.Int64
failed atomic.Bool
once sync.Once
first error
wg sync.WaitGroup
)
for w := 0; w < min(ReadWorkers, n); w++ {
wg.Add(1)
go func() {
defer wg.Done()
for !failed.Load() {
i := int(next.Add(1) - 1)
if i >= n {
return
}
if err := read(i); err != nil {
once.Do(func() { first = err })
failed.Store(true)
}
}
}()
}
wg.Wait()
return first
}
// capKeys keeps the first MaxReads keys and answers a note when it cut the rest, "" otherwise.
func capKeys(keys []string, narrow string) ([]string, string) {
if len(keys) <= MaxReads {
return keys, ""
}
return keys[:MaxReads], fmt.Sprintf("read %d of %d matching keys and stopped; %s", MaxReads, len(keys), narrow)
}
// newestFirst sorts step keys (<node>.<id>) by id, newest first: an id is the time in base 36, then six hex digits, so the
// longer id is the later and equal lengths compare as text. A key that is no step key goes last.
func newestFirst(keys []string) {
id := func(k string) (string, bool) {
_, id, err := delivery.ParseStepKey(k)
return id, err == nil
}
sort.Slice(keys, func(i, j int) bool {
a, aok := id(keys[i])
b, bok := id(keys[j])
switch {
case aok != bok:
return aok
case len(a) != len(b):
return len(a) > len(b)
case a != b:
return a > b
}
return keys[i] > keys[j]
})
}
@@ -44,35 +44,11 @@ type Case struct {
notesMu sync.Mutex notesMu sync.Mutex
notes []string notes []string
// asks are the ids of the asks the check put on the build seats' streams, in order: a try names them (Try.Asks).
asks []string
// Progress says what the check is doing, so a check that ran past its bound names the step it was stuck at. // Progress says what the check is doing, so a check that ran past its bound names the step it was stuck at.
Progress *Progress Progress *Progress
// Kind is what asked the check (KindNew, KindNodes, KindTrunk): a carry after a trunk or sibling move is asked on the // Kind is what asked the check (KindNew, KindNodes, KindTrunk): a re-judge after a trunk move is asked on the build
// build seats' recheck stream, every other check on check (ADR 0536 §4, ADR 0548 §5). // seats' recheck stream, every other check on check (ADR 0536 §4).
Kind string Kind string
// CarriedFrom are, for the carry of a verdict that passed, the trunk and sibling commits it was judged on (ADR 0548 §5);
// nil for a whole check.
CarriedFrom map[string]string
// FPSet is the candidate's fingerprint set (Candidate.FPSet): a composition names the set it judged.
FPSet string
// Composition is, for a carry, the composition that passed on the same fingerprint set: it is kept and not asked again,
// since no fingerprint of a module it composed changed (ADR 0548 §5). Nil composes.
Composition *delivery.Composition
}
// Asked records the id of an ask the check put.
func (c *Case) Asked(id string) {
c.notesMu.Lock()
c.asks = append(c.asks, id)
c.notesMu.Unlock()
}
// Asks are the ids of the asks the check put, in order.
func (c *Case) Asks() []string {
c.notesMu.Lock()
defer c.notesMu.Unlock()
return append([]string(nil), c.asks...)
} }
// Progress is the step a check is at, written by the check and read when its bound passes. // Progress is the step a check is at, written by the check and read when its bound passes.
@@ -104,7 +80,7 @@ func CaseOf(c delivery.Candidate, trunk map[string]string) *Case {
if err != nil { if err != nil {
return nil return nil
} }
cs := &Case{Key: key, Group: c.Group, HeadsHash: c.HeadsHash, Heads: GateHeads(c), Trunk: map[string]string{}, Progress: &Progress{}, FPSet: c.FPSet} cs := &Case{Key: key, Group: c.Group, HeadsHash: c.HeadsHash, Heads: GateHeads(c), Trunk: map[string]string{}, Progress: &Progress{}}
for _, p := range c.Pulls { for _, p := range c.Pulls {
switch head, known := trunk[p.Repository]; { switch head, known := trunk[p.Repository]; {
case !known || head == "": case !known || head == "":
@@ -4,7 +4,6 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"sort"
"strings" "strings"
"sync" "sync"
"time" "time"
@@ -43,9 +42,6 @@ type Outcome struct {
// Judged says, by pull request (pullKey), which program judged a change to a build module (ADR 0535 §5); a pull request // Judged says, by pull request (pullKey), which program judged a change to a build module (ADR 0535 §5); a pull request
// that changes none is not named. // that changes none is not named.
Judged map[string]string Judged map[string]string
// Beside are the siblings the check read at their trunk's head, by repository (ADR 0548 §6): the verdict names them, and
// a move of one is carried.
Beside map[string]string
} }
// BuildNote is the state of a candidate's module build, by module: built, failed or waiting (ADR 0484 clause 7). // BuildNote is the state of a candidate's module build, by module: built, failed or waiting (ADR 0484 clause 7).
@@ -69,14 +65,6 @@ type Checker struct {
Owed KV Owed KV
Judge Judge Judge Judge
Trunks interface{ Heads() map[string]string } Trunks interface{ Heads() map[string]string }
// Runs says which commit the mesh runs of a module (the indexer's modules): the node-engine a verdict read beside at its
// running commit is out of date when that commit moves (ADR 0548 §6); nil reads every sibling at its trunk.
Runs interface{ CommitOf(module string) string }
// Turns are the gate's merge turns (ADR 0548 §3): no check is claimed of a repository a turn names, but the check of the
// waiting delivery's own heads; nil holds none.
Turns interface {
Holding(repositories []string, heads string) (string, delivery.TurnWait, bool)
}
// Bound is how long one check may run; none when 0 (DefaultBound). // Bound is how long one check may run; none when 0 (DefaultBound).
Bound time.Duration Bound time.Duration
// Link is the base of the address a status points at; the verdict key follows. // Link is the base of the address a status points at; the verdict key follows.
@@ -95,13 +83,12 @@ type Checker struct {
tried map[string]bool // by verdict key: looked at since this process started tried map[string]bool // by verdict key: looked at since this process started
epoch uint64 // changes of the nodes seen since this process started epoch uint64 // changes of the nodes seen since this process started
nodesFail map[string]uint64 // by verdict key: the epoch a failure of the nodes was written at nodesFail map[string]uint64 // by verdict key: the epoch a failure of the nodes was written at
held map[string]string // by verdict key: the turn that holds its check off, as last said
} }
func (ch *Checker) init() { func (ch *Checker) init() {
ch.once.Do(func() { ch.once.Do(func() {
ch.known, ch.inflight, ch.again, ch.tried = map[string]delivery.Candidate{}, map[string]bool{}, map[string]bool{}, map[string]bool{} ch.known, ch.inflight, ch.again, ch.tried = map[string]delivery.Candidate{}, map[string]bool{}, map[string]bool{}, map[string]bool{}
ch.nodesFail, ch.held = map[string]uint64{}, map[string]string{} ch.nodesFail = map[string]uint64{}
}) })
} }
@@ -204,7 +191,7 @@ func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidat
// Every check is asked at once: the build seats' streams hold the backlog, a new head's check and a check after a // Every check is asked at once: the build seats' streams hold the backlog, a new head's check and a check after a
// failure of the nodes on check, a re-judge after a trunk move on recheck, pulled after them (ADR 0525 §4, §9; // failure of the nodes on check, a re-judge after a trunk move on recheck, pulled after them (ADR 0525 §4, §9;
// ADR 0536 §4). The checker holds no place and no capacity of its own. // ADR 0536 §4). The checker holds no place and no capacity of its own.
ch.prepare(ctx, cs) cs.Kind = ch.kindOf(ctx, cs)
ch.try(ctx, key, cs) ch.try(ctx, key, cs)
ch.mu.Lock() ch.mu.Lock()
more := ch.again[cs.Key] more := ch.again[cs.Key]
@@ -224,130 +211,29 @@ func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidat
} }
// The kinds of a check (ADR 0536 §4): a head never judged and a head asked again after a failure of the nodes are asked on // The kinds of a check (ADR 0536 §4): a head never judged and a head asked again after a failure of the nodes are asked on
// the build seats' check stream; a carry, a verdict moved to a newer trunk or sibling commit by its package digests (ADR 0548 // the build seats' check stream, a head asked again only because its trunk moved on the recheck stream, pulled after.
// §5), on the recheck stream, pulled after.
const ( const (
KindNew = "new" KindNew = "new"
KindNodes = "nodes" KindNodes = "nodes"
KindTrunk = "trunk" KindTrunk = "trunk"
) )
// prepare reads the case's verdict key and says what its check is (ADR 0548 §1, §5, §7): none (or one left running) is a // kindOf is the kind of a case's check, read from its verdict key: none (or one left running) is a head never judged; a
// head never judged, a whole check; a failure of the nodes is asked again after the nodes changed; a verdict judged on trunk // failure of the nodes is asked again after the nodes changed; a verdict on other trunk heads is a re-judge after a trunk
// or sibling commits that moved since is carried, its packages run only where the move changed their digest, and a pass's // move. A verdict that stands is ranked with the re-judges: its claim finds nothing to do.
// commits are named as the carry's CarriedFrom. A verdict that stands is ranked with the carries: its claim finds nothing func (ch *Checker) kindOf(ctx context.Context, cs *Case) string {
// to do.
func (ch *Checker) prepare(ctx context.Context, cs *Case) {
cs.Kind, cs.CarriedFrom, cs.Composition = KindNew, nil, nil
value, _, found, err := ch.Verdicts.Get(ctx, cs.Key) value, _, found, err := ch.Verdicts.Get(ctx, cs.Key)
if err != nil || !found { if err != nil || !found {
return return KindNew
} }
v, err := delivery.DecodeVerdict(value) v, err := delivery.DecodeVerdict(value)
switch { switch {
case err != nil || v.State == delivery.VerdictRunning: case err != nil || v.State == delivery.VerdictRunning:
return return KindNew
case IsNodesFailure(v): case IsNodesFailure(v):
cs.Kind = KindNodes return KindNodes
return
case IsConflictFailure(v):
return // a head git refused: its next check is whole (ADR 0548 §7)
} }
cs.Kind = KindTrunk return KindTrunk
if v.State == delivery.VerdictPass && ch.stale(v, cs) {
cs.CarriedFrom = judgedOn(v)
if c := v.Composition; c != nil && c.State == delivery.CompositionPass && c.FPSet != "" && c.FPSet == cs.FPSet {
kept := *c
cs.Composition = &kept
}
}
}
// stale says whether a verdict was judged on commits that are no longer the case's: a trunk it merged moved, or a sibling it
// read at its trunk's head (ADR 0548 §6).
func (ch *Checker) stale(v delivery.Verdict, cs *Case) bool {
return !sameTrunk(v.Trunk, cs.Trunk) || besideMoved(v.Beside, ch.besideNow())
}
// besideNow are the commits a sibling is read at now, by repository: its trunk head, and for the node-engine the commit the
// mesh runs when the indexer names one, as besideFor places them.
func (ch *Checker) besideNow() map[string]string {
return besideAt(ch.heads(), ch.Runs)
}
// besideAt places the commits a sibling is read at: the trunk heads, with the node-engine at the commit the mesh runs when
// runs names one (ADR 0548 §6). The checker's decision and the status read the same commits through it.
func besideAt(now map[string]string, runs interface{ CommitOf(module string) string }) map[string]string {
if runs != nil {
if c := runs.CommitOf(nodeEngineModule); c != "" {
out := make(map[string]string, len(now)+1)
for r, h := range now {
out[r] = h
}
out[nodeEngineRepo] = c
return out
}
}
return now
}
// besideMoved says whether a sibling read at these commits moved: the commit it is read at now is known and another.
func besideMoved(beside, now map[string]string) bool {
for r, c := range beside {
if h := now[r]; h != "" && h != c {
return true
}
}
return false
}
// judgedOn are the trunk and sibling commits a verdict was judged on, by repository.
func judgedOn(v delivery.Verdict) map[string]string {
out := make(map[string]string, len(v.Trunk)+len(v.Beside))
for r, c := range v.Beside {
out[r] = c
}
for r, c := range v.Trunk {
out[r] = c
}
return out
}
// carried is what a carry's verdict names as carried from (ADR 0548 §5): the commits it was judged on before, of the
// repositories the new verdict still judges or reads; nil when none of them moved.
func carried(from map[string]string, v delivery.Verdict) map[string]string {
out, moved := map[string]string{}, false
for r, c := range from {
now, ok := v.Trunk[r]
if !ok {
now, ok = v.Beside[r]
}
if ok && now != "" && c != "" {
out[r] = c
moved = moved || now != c
}
}
if !moved {
return nil
}
return out
}
// moveWords say a carry's moves, for the words of a failure: "carried from <old12> to <new12> of <repository>", each moved
// repository, sorted.
func moveWords(from map[string]string, v delivery.Verdict) string {
var moves []string
for r, c := range from {
now, ok := v.Trunk[r]
if !ok {
now = v.Beside[r]
}
if now != "" && now != c {
moves = append(moves, fmt.Sprintf("%s %s to %s", r, short12(c), short12(now)))
}
}
sort.Strings(moves)
return strings.Join(moves, ", ")
} }
func (ch *Checker) candidate(key string, fallback delivery.Candidate) delivery.Candidate { func (ch *Checker) candidate(key string, fallback delivery.Candidate) delivery.Candidate {
@@ -379,9 +265,6 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64
return nil, fmt.Errorf("%s: %w", cs.Key, err) return nil, fmt.Errorf("%s: %w", cs.Key, err)
} }
if !found { if !found {
if ch.heldByTurn(cs) {
return nil, nil
}
rev, err := ch.Verdicts.Create(ctx, cs.Key, data) rev, err := ch.Verdicts.Create(ctx, cs.Key, data)
if errors.Is(err, ErrConflict) { if errors.Is(err, ErrConflict) {
return nil, nil // another checker made it first return nil, nil // another checker made it first
@@ -402,7 +285,7 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64
return &claimed{rev}, nil return &claimed{rev}, nil
} }
return nil, nil return nil, nil
case ch.stale(cur, cs): case !sameTrunk(cur.Trunk, cs.Trunk):
retake = true retake = true
case cur.State == delivery.VerdictFail && IsNodesFailure(cur): case cur.State == delivery.VerdictFail && IsNodesFailure(cur):
ch.mu.Lock() ch.mu.Lock()
@@ -410,7 +293,7 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64
ch.mu.Unlock() ch.mu.Unlock()
retake = !ok || epoch > at retake = !ok || epoch > at
} }
if !retake || ch.heldByTurn(cs) { if !retake {
return nil, nil return nil, nil
} }
rev, err = ch.Verdicts.Update(ctx, cs.Key, data, rev) rev, err = ch.Verdicts.Update(ctx, cs.Key, data, rev)
@@ -423,59 +306,6 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64
return &claimed{rev}, nil return &claimed{rev}, nil
} }
// heldByTurn says whether a merge turn holds the case's check off (ADR 0548 §3): an approved merge waits on one of the
// repositories it merges heads into, and it is not the merge of the case's own heads. The check is claimed when the turn's
// key goes (Reconsider on the turns watch). Said once per turn in the log. A check already claimed is never held: one
// claimed in the moment before the gate wrote its turn is judged on the commit it read and carried once after.
func (ch *Checker) heldByTurn(cs *Case) bool {
if ch.Turns == nil {
return false
}
repos := make([]string, 0, len(cs.Trunk))
for r := range cs.Trunk {
repos = append(repos, r)
}
repo, w, held := ch.Turns.Holding(repos, cs.Heads)
why := ""
if held {
why = repo + " " + w.Group + " " + w.Heads
}
ch.mu.Lock()
said := ch.held[cs.Key]
if why == "" {
delete(ch.held, cs.Key)
} else {
ch.held[cs.Key] = why
}
ch.mu.Unlock()
if held && said != why {
ch.logf("%s: not claimed: the approved merge of %s (heads %s) holds a turn on %s, so its check waits for that merge (ADR 0548 §3)",
cs.Key, w.Group, w.Heads, repo)
}
return held
}
// Held is every check a merge turn holds off now, by verdict key, with the turn's repository, group and heads.
func (ch *Checker) Held() map[string]string {
ch.init()
ch.mu.Lock()
defer ch.mu.Unlock()
out := make(map[string]string, len(ch.held))
for k, v := range ch.held {
out[k] = v
}
return out
}
// ConflictWords are what the words of a head that git would not merge onto its trunk say before the files in conflict.
const ConflictWords = "does not merge into the trunk: files in conflict: "
// IsConflictFailure says whether a verdict failed because its head did not merge onto its trunk (ADR 0548 §7): it has no
// pass to carry, so its next check, at a push or at a later trunk move, is a whole check of its head.
func IsConflictFailure(v delivery.Verdict) bool {
return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, ConflictWords)
}
// IsNodesFailure says whether a verdict failed because a part of the mesh the check reads was not heard (UnheardWords): // IsNodesFailure says whether a verdict failed because a part of the mesh the check reads was not heard (UnheardWords):
// it is judged again when the nodes change. No check fails for want of a build node any more: its asks wait in the build // it is judged again when the nodes change. No check fails for want of a build node any more: its asks wait in the build
// seats' streams (ADR 0525 §6, issue 1912). // seats' streams (ADR 0525 §6, issue 1912).
@@ -502,41 +332,11 @@ func (ch *Checker) try(ctx context.Context, key string, cs *Case) {
began := ch.now() began := ch.now()
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{}) ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
out := ch.judge(ctx, cs) out := ch.judge(ctx, cs)
// An ask the build seat gave up on judged nothing (issue 734): the check is asked again under the same claim, its status
// pending, and after LostAskTries it fails as not heard, judged again when the nodes change.
for try := 1; IsLostAsk(out.Verdict) && ctx.Err() == nil; try++ {
if try >= LostAskTries {
out.Verdict.Failed.Words = strings.TrimSpace(out.Verdict.Failed.Words + "\nasked " + fmt.Sprint(LostAskTries) +
" times and given up each time: " + UnheardWords)
break
}
ch.logf("%s: %s; asked again (%d of %d)", cs.Key, out.Verdict.Failed.Words, try+1, LostAskTries)
out = ch.judge(ctx, cs)
}
if ctx.Err() != nil { if ctx.Err() != nil {
return return
} }
v := out.Verdict v := out.Verdict
v.Group, v.Heads, v.Trunk, v.Beside = cs.Group, cs.Heads, cs.Trunk, nil v.Group, v.Heads, v.Trunk = cs.Group, cs.Heads, cs.Trunk
for r, c := range out.Beside {
if _, merged := cs.Trunk[r]; !merged && c != "" {
if v.Beside == nil {
v.Beside = map[string]string{}
}
v.Beside[r] = c
}
}
if cs.CarriedFrom != nil {
// A carry (ADR 0548 §5): a pass names the commits it was carried from; a failure names the move it failed on.
switch v.State {
case delivery.VerdictPass:
v.CarriedFrom = carried(cs.CarriedFrom, v)
case delivery.VerdictFail:
if moves := moveWords(cs.CarriedFrom, v); moves != "" {
v.Failed.Words = strings.TrimSpace(v.Failed.Words + "\nfailed on the carry over the move of " + moves + ": the move changed what these tests read")
}
}
}
if v.Took.Queue == 0 && !cs.Queued.IsZero() && began.After(cs.Queued) { if v.Took.Queue == 0 && !cs.Queued.IsZero() && began.After(cs.Queued) {
v.Took.Queue = began.Sub(cs.Queued).Milliseconds() v.Took.Queue = began.Sub(cs.Queued).Milliseconds()
} }
@@ -558,11 +358,11 @@ func (ch *Checker) try(ctx context.Context, key string, cs *Case) {
ch.nodesFail[cs.Key] = epoch ch.nodesFail[cs.Key] = epoch
ch.mu.Unlock() ch.mu.Unlock()
} }
if ch.movedSince(cs) || besideMoved(v.Beside, ch.besideNow()) { if ch.movedSince(cs) {
// Judged on a trunk that moved while it ran: the verdict is kept as judged, and the status stays pending until the // Judged on a trunk that moved while it ran: the verdict is kept as judged, and the status stays pending until the
// check of the new trunk, which Reconsider starts, ends. // check of the new trunk, which Reconsider starts, ends.
ch.owePending(ctx, cs, v) ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
ch.logf("%s: %s on a trunk that moved while it ran; the status stays pending until it is carried", cs.Key, v.State) ch.logf("%s: %s on a trunk that moved while it ran; the status stays pending", cs.Key, v.State)
return return
} }
ch.writeOwed(ctx, cs, v, out) ch.writeOwed(ctx, cs, v, out)
@@ -702,37 +502,14 @@ func (ch *Checker) TrunkMoved(ctx context.Context) {
if err != nil || !found { if err != nil || !found {
continue // never judged: no status to put out of date continue // never judged: no status to put out of date
} }
v, err := delivery.DecodeVerdict(value) if v, err := delivery.DecodeVerdict(value); err == nil && sameTrunk(v.Trunk, cs.Trunk) {
if err == nil && !ch.stale(v, cs) {
continue continue
} }
if err == nil && v.State == delivery.VerdictPass {
cs.CarriedFrom = judgedOn(v) // the status says it is being carried to the new commit (ADR 0548 §6)
}
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{}) ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
} }
ch.Reconsider(ctx) ch.Reconsider(ctx)
} }
// owePending owes mesh/check pending on the heads of a verdict judged on commits that moved while it ran: a pass says it is
// being carried to the trunk's head now (ADR 0548 §6), anything else that it is checked again.
func (ch *Checker) owePending(ctx context.Context, judged *Case, v delivery.Verdict) {
next := judged
ckey, err := delivery.CandidatesKey(v.Group)
ch.mu.Lock()
c, ok := ch.known[ckey]
ch.mu.Unlock()
if err == nil && ok {
if now := CaseOf(c, ch.heads()); now != nil && now.Key == judged.Key {
next = now
}
}
if v.State == delivery.VerdictPass {
next.CarriedFrom = judgedOn(v)
}
ch.writeOwed(ctx, next, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
}
// movedSince says whether a trunk the case was judged on is no longer the trunk's head. // movedSince says whether a trunk the case was judged on is no longer the trunk's head.
func (ch *Checker) movedSince(cs *Case) bool { func (ch *Checker) movedSince(cs *Case) bool {
now := ch.heads() now := ch.heads()
@@ -28,7 +28,6 @@ func TestAPassingCheckTakesTheAssignersComposition(t *testing.T) {
f := &fakeCompositions{v: delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionPass, SDK: "v0.1.14-x", Machines: []string{"g14"}, Took: 40}} f := &fakeCompositions{v: delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionPass, SDK: "v0.1.14-x", Machines: []string{"g14"}, Took: 40}}
b.ex.Compositions = f b.ex.Compositions = f
cs := caseOf(t, pullOf(repoA, 7, head1)) cs := caseOf(t, pullOf(repoA, 7, head1))
cs.FPSet = "fps-of-the-candidate"
v := b.judge(t, cs).Verdict v := b.judge(t, cs).Verdict
if v.State != delivery.VerdictPass || v.Composition == nil || v.Composition.State != delivery.CompositionPass || v.Composition.SDK != "v0.1.14-x" || v.Took.Composition != 40 { if v.State != delivery.VerdictPass || v.Composition == nil || v.Composition.State != delivery.CompositionPass || v.Composition.SDK != "v0.1.14-x" || v.Took.Composition != 40 {
t.Fatalf("verdict = %+v, composition %+v", v, v.Composition) t.Fatalf("verdict = %+v, composition %+v", v, v.Composition)
@@ -39,15 +38,6 @@ func TestAPassingCheckTakesTheAssignersComposition(t *testing.T) {
if !compositionMentions(v, "passed") { if !compositionMentions(v, "passed") {
t.Errorf("the line says %q", compositionWords(v.Composition, v.Took.Composition)) t.Errorf("the line says %q", compositionWords(v.Composition, v.Took.Composition))
} }
if v.Composition.FPSet == "" || v.Composition.FPSet != cs.FPSet {
t.Errorf("the composition names the fingerprint set %q, want the candidate's %q (ADR 0548 §5)", v.Composition.FPSet, cs.FPSet)
}
// ADR 0548 §5: a carry whose fingerprints did not change keeps the composition it passed and asks the assigner nothing.
carry := caseOf(t, pullOf(repoA, 7, head1))
carry.CarriedFrom, carry.Composition = map[string]string{repoA: trunkA2}, v.Composition
if kept := b.judge(t, carry).Verdict; kept.Composition == nil || kept.Composition.SDK != "v0.1.14-x" || len(f.asked) != 1 {
t.Errorf("the carry's composition is %+v and the assigner was asked %d times; want the kept one and no second ask", kept.Composition, len(f.asked))
}
} }
func compositionMentions(v delivery.Verdict, word string) bool { func compositionMentions(v delivery.Verdict, word string) bool {
@@ -1,90 +0,0 @@
package check
import (
"encoding/json"
"fmt"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// failingResult is a level's test result in which the named tests failed, each in its own package.
func failingResult(t *testing.T, tests int, failed ...Failure) json.RawMessage {
t.Helper()
raw, err := json.Marshal(Tested{Level: delivery.LevelIntegration, Tests: tests, Elapsed: 2, Failed: failed, Passed: len(failed) == 0})
if err != nil {
t.Fatal(err)
}
return raw
}
func judgeLevel(t *testing.T, tests ...json.RawMessage) (*delivery.Verdict, *state) {
t.Helper()
st := &state{j: &Examiner{}, c: &Case{}, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{}}
v, _ := st.level(t.Context(), &target{pull: Pull{Repository: "novox/x"}}, delivery.LevelAnswer{Level: delivery.LevelIntegration, Tests: tests})
return v, st
}
// Issue 1921: two failing tests in two packages are two entries of the verdict's Failures and two lines of a try's
// failed list, each with its own first line (its cause, issue 1901), whether the build module answered them in one test
// result (one go test run over both packages) or in two (the changed packages first, then the rest); every result is
// counted, the ones after a failure too; and Failed keeps the first one's words whole.
func TestTwoFailingTestsInTwoPackagesAreTwoEntries(t *testing.T) {
a := Failure{Package: "x/a", Test: "TestA", Cause: "a_test.go:9: want 1, got 2", Output: " a_test.go:3: logged\n a_test.go:9: want 1, got 2\n--- FAIL: TestA (0.00s)"}
b := Failure{Package: "x/b", Test: "TestB", Cause: "b_test.go:4: boom", Output: " b_test.go:4: boom\n--- FAIL: TestB (0.00s)"}
for name, tests := range map[string][]json.RawMessage{
"one result": {failingResult(t, 10, a, b)},
"two results": {failingResult(t, 4, a), failingResult(t, 3), failingResult(t, 3, b)},
} {
v, st := judgeLevel(t, tests...)
if v == nil {
t.Fatalf("%s: the level passed", name)
}
want := []string{"integration: x/a.TestA: a_test.go:9: want 1, got 2", "integration: x/b.TestB: b_test.go:4: boom"}
if got := failedLines(*v); strings.Join(got, "\n") != strings.Join(want, "\n") {
t.Errorf("%s: failed lines\n%s\nwant\n%s", name, strings.Join(got, "\n"), strings.Join(want, "\n"))
}
if len(v.Failures) != 2 || v.Failed.Test != "integration: x/a.TestA" || !strings.Contains(v.Failed.Words, "a_test.go:3: logged") {
t.Errorf("%s: verdict %+v", name, v)
}
if run := st.levels[delivery.LevelIntegration].run; run != 10 {
t.Errorf("%s: %d tests counted, want all 10", name, run)
}
if _, err := delivery.EncodeVerdict(delivery.Verdict{Group: "g", Heads: "0123456789ab", State: v.State, Failed: v.Failed, Failures: v.Failures}); err != nil {
t.Errorf("%s: the verdict is not written: %v", name, err)
}
}
}
// Issue 1921: past delivery.MaxFailures the list names the first fifty and how many more, the status line says how many
// failed, and each entry's line is cut to MaxFailureLine.
func TestFailuresAreCappedWithHowManyMore(t *testing.T) {
var fs []Failure
for i := range 60 {
fs = append(fs, Failure{Package: fmt.Sprintf("x/p%d", i), Test: "TestT", Cause: strings.Repeat("y", 500)})
}
v, _ := judgeLevel(t, failingResult(t, 60, fs...))
if len(v.Failures) != delivery.MaxFailures+1 || v.Failures[delivery.MaxFailures].Test != "and 10 more" {
t.Fatalf("%d entries, last %+v", len(v.Failures), v.Failures[len(v.Failures)-1])
}
if len(v.Failures[0].Words) > MaxFailureLine {
t.Errorf("an entry's line is %d bytes", len(v.Failures[0].Words))
}
if n := failedCount(*v); n != 60 {
t.Errorf("failedCount = %d, want 60", n)
}
cs := &Case{Pulls: []Pull{{Repository: "novox/x", Head: "0123456789abcdef", Trunk: "fedcba9876543210"}}}
if line := Line(*v, cs, cs.Pulls[0], nil, nil, nil); !strings.HasPrefix(line, "failed (60 tests): integration: x/p0.TestT") {
t.Errorf("line %q", line)
}
}
// A verdict that names no list (a conflict, a refused step, a verdict written before issue 1921) is one failed line, as
// before.
func TestAVerdictWithoutFailuresIsOneLine(t *testing.T) {
v := delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "g14", Words: "files in conflict: go.mod\nmore"}}
if got := failedLines(v); len(got) != 1 || got[0] != "g14: files in conflict: go.mod" {
t.Fatalf("%q", got)
}
}
@@ -116,9 +116,6 @@ type bench struct {
fetch Fetched fetch Fetched
digest map[string]Digested // by level digest map[string]Digested // by level
test func(c stepCall) (any, error) test func(c stepCall) (any, error)
// declines is how many asks naming needs a node lacking them declines before shanks, which has every need, takes
// one (issue 890); an ask naming none runs on shanks, which then has none.
declines int
} }
// refsOf reads the siblings of a fetch's arguments, as they arrive in an ask (JSON) or as the checker gave them. // refsOf reads the siblings of a fetch's arguments, as they arrive in an ask (JSON) or as the checker gave them.
@@ -154,17 +151,8 @@ func (b *bench) answer(a delivery.StepAsk) delivery.Step {
if module == buildrun.GoBuildModule { if module == buildrun.GoBuildModule {
steps.Digest = tool("digest") steps.Digest = tool("digest")
} }
began := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC)
if len(a.Needs) > 0 {
if b.declines > 0 {
b.declines--
result, _ := json.Marshal(delivery.StepDeclined{Declined: true, Needs: a.Needs, Lacks: a.Needs})
return delivery.Step{ID: a.ID, Tool: a.Kind + " ask", Node: "ace", State: delivery.StepRefused, Started: began, Ended: began,
Reason: "declined", Result: result}
}
steps.Has = func(string) bool { return true }
}
out, err := buildstep.RunAsk(context.Background(), steps, a) out, err := buildstep.RunAsk(context.Background(), steps, a)
began := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC)
step := delivery.Step{ID: a.ID, Tool: a.Kind + " ask", Node: node, State: delivery.StepPassed, Started: began, Ended: began.Add(2 * time.Second)} step := delivery.Step{ID: a.ID, Tool: a.Kind + " ask", Node: node, State: delivery.StepPassed, Started: began, Ended: began.Add(2 * time.Second)}
if err != nil { if err != nil {
step.State, step.Reason = delivery.StepFailed, err.Error() step.State, step.Reason = delivery.StepFailed, err.Error()
@@ -288,49 +276,6 @@ func TestOnlyThePackageWithoutAKeptVerdictRuns(t *testing.T) { // 1288/AC1
if u := out.Verdict.Levels[delivery.LevelUnit]; u.Run != 2 || u.Kept != 40 { if u := out.Verdict.Levels[delivery.LevelUnit]; u.Run != 2 || u.Kept != 40 {
t.Errorf("unit = %+v, want 2 run and 40 kept", u) t.Errorf("unit = %+v, want 2 run and 40 kept", u)
} }
// Issue 890/AC5: a package needing node-build-node is left unmet where the node lacks it; the check keeps the passes
// and asks the integration level again for a node with it, puts it again when a node declines it, and that node tests
// the package alone, its kept passes not counted twice.
// fixture/outside has no kept verdict after its pass (it reads outside itself: its digest moves every call), so a
// placed ask that ran more than its unmet packages would run and count it again.
b = newBench(t)
b.declines = 1
outside := 0
b.steps.handlers["build_go_digest"] = func(c stepCall) (any, error) {
d := echoBeside(b, b.digest[c.args["level"].(string)])
if d.Level == "integration" {
outside++
d.Packages = map[string]string{"fixture/slow": digB, "fixture/node": digC, "fixture/outside": fmt.Sprintf("%064x", outside)}
}
return d, nil
}
b.digest[delivery.LevelIntegration] = Digested{Level: "integration",
Needs: []delivery.PackageNeeds{{Package: "fixture/node", Needs: []string{delivery.NeedNodeBuildNode}}}}
out = b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
asks := b.asks.Asks()
if out.Verdict.State != delivery.VerdictPass || len(asks) != 3 || len(asks[0].Needs) != 0 {
t.Fatalf("verdict %+v, asks %+v: want a pass of three asks, the first naming no need", out.Verdict, asks)
}
for _, a := range asks[1:] {
if strings.Join(a.Needs, ",") != delivery.NeedNodeBuildNode || strings.Join(a.Check.Levels, ",") != delivery.LevelIntegration || strings.Join(a.Check.Packages, ",") != "fixture/node" {
t.Errorf("a placed ask %+v, want the integration level needing node-build-node", a)
}
}
tests = b.steps.tool("test")
if last := tests[len(tests)-1]; len(tests) != 3 || strings.Join(packagesOf(last), ",") != "fixture/node" || last.args["level"] != "integration" {
t.Errorf("test calls = %+v, want the unit level, the integration level of fixture/slow and fixture/outside, then fixture/node alone", tests)
}
slow := captured[Tested](t, "go-test-slow")
if i := out.Verdict.Levels[delivery.LevelIntegration]; i.Run != 2*slow.Tests || i.Kept != 0 {
t.Errorf("integration = %+v, want %d run and nothing counted kept", i, 2*slow.Tests)
}
// And where no node has the need, the check fails once the declines outlast the bound, naming the need.
b.kept, b.declines = newMemKV(), 1000
b.ex.Kept, b.ex.DeclineBound = Kept{Store: b.kept}, 200*time.Millisecond
if v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict; v.State != delivery.VerdictFail || !strings.Contains(fmt.Sprint(v), "needs a node with node-build-node") {
t.Errorf("with no node having the need: %+v", v)
}
} }
func TestAReAskRunsOnlyWhatFailedAndWhatChanged(t *testing.T) { // 1288/AC2, AC3 func TestAReAskRunsOnlyWhatFailedAndWhatChanged(t *testing.T) { // 1288/AC2, AC3
@@ -396,9 +341,7 @@ func TestANodePackageIsNeverKeptAndRunsEveryCheck(t *testing.T) { // 1288/AC4, 1
} }
} }
// 1289/AC1, AC2 as issue 1921 changed it: the change's own failing test runs first, the level's other packages still run func TestTheChangesOwnFailingTestRunsFirstAndNothingElseRuns(t *testing.T) { // 1289/AC1, AC2
// after it (so one check names every failure), and no later level runs.
func TestTheChangesOwnFailingTestRunsFirstThenTheRestAndNoLaterLevel(t *testing.T) {
b := newBench(t) b := newBench(t)
b.fetch.Changed = []string{"modules/m/internal/b/b_test.go"} b.fetch.Changed = []string{"modules/m/internal/b/b_test.go"}
b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"m/internal/a": digA, "m/internal/b": digB}} b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"m/internal/a": digA, "m/internal/b": digB}}
@@ -414,8 +357,8 @@ func TestTheChangesOwnFailingTestRunsFirstThenTheRestAndNoLaterLevel(t *testing.
t.Fatalf("verdict = %+v", out.Verdict) t.Fatalf("verdict = %+v", out.Verdict)
} }
tests := b.steps.tool("test") tests := b.steps.tool("test")
if len(tests) != 2 || strings.Join(packagesOf(tests[0]), ",") != "m/internal/b" || strings.Join(packagesOf(tests[1]), ",") != "m/internal/a" { if len(tests) != 1 || strings.Join(packagesOf(tests[0]), ",") != "m/internal/b" {
t.Fatalf("test calls = %+v, want m/internal/b, then m/internal/a, each once", tests) t.Fatalf("test calls = %+v, want only m/internal/b, once", tests)
} }
if n := len(b.steps.tool("digest")); n != 1 { if n := len(b.steps.tool("digest")); n != 1 {
t.Errorf("%d digest calls, want the unit level's alone (the check stopped)", n) t.Errorf("%d digest calls, want the unit level's alone (the check stopped)", n)
@@ -675,8 +618,9 @@ func TestNothingIsKeptForAWorkspaceGivenSiblingsWhenTheDigestDoesNotSayItCoversT
// Beside): every check now places the change's own repository at its trunk (issue 1887 A12). // Beside): every check now places the change's own repository at its trunk (issue 1887 A12).
func echoBeside(b *bench, d Digested) Digested { func echoBeside(b *bench, d Digested) Digested {
var lines []string var lines []string
if fetches := b.steps.tool("fetch"); len(fetches) > 0 { // the workspace digested is the last fetch's (issue 890: a placed ask fetches again) for _, f := range b.steps.tool("fetch") {
for _, r := range refsOf(fetches[len(fetches)-1].args["beside"]) { refs, _ := refsOf(f.args["beside"]), true
for _, r := range refs {
lines = append(lines, r.Repository+" "+r.Commit) lines = append(lines, r.Repository+" "+r.Commit)
} }
} }
@@ -4,6 +4,7 @@ import (
"context" "context"
"strings" "strings"
"testing" "testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery"
) )
@@ -87,3 +88,29 @@ func TestATrunkMoveMarksEveryOpenPullRequestIntoItPendingAtOnce(t *testing.T) {
t.Errorf("mesh-sdk#13, alone into a trunk that did not move, is %q, want its success to stand", s) t.Errorf("mesh-sdk#13, alone into a trunk that did not move, is %q, want its success to stand", s)
} }
} }
func TestACheckThatEndsAfterItsTrunkMovedOwesPendingNotItsVerdict(t *testing.T) { // gate (a): a pass on an old trunk is never shown
ctx := context.Background()
var r *rig
release, judged := make(chan struct{}), make(chan struct{})
j := &judgeFunc{f: func(ctx context.Context, c *Case, call int) (Outcome, error) {
if call == 1 {
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) // the trunk moves while it runs
defer close(judged)
} else {
<-release // the check asked again for the new trunk waits in the stream meanwhile
}
return Outcome{Verdict: passing()}, nil
}}
r = newRig(t, j)
r.ch.Candidate(ctx, "feat%2Fx", candidateOf(t, "feat/x", pullOf(repoA, 7, head1)))
<-judged
for deadline := time.Now().Add(2 * time.Second); owedState(t, r, repoA, 7, head1) == "" && time.Now().Before(deadline); time.Sleep(time.Millisecond) {
}
time.Sleep(20 * time.Millisecond)
if s := owedState(t, r, repoA, 7, head1); s != "pending" {
t.Errorf("the status is %q, want pending: the pass was judged on a trunk that moved", s)
}
close(release)
r.ch.Wait()
}
@@ -80,9 +80,6 @@ var askLanguage = map[string]string{buildrun.GoBuildModule: delivery.AskGo, buil
// stopping at the first failure; it judges the step's answer as it judged the tools' answers. // stopping at the first failure; it judges the step's answer as it judged the tools' answers.
type Examiner struct { type Examiner struct {
Asks Asks Asks Asks
// DeclineBound is how long a placed ask may go on being declined, from its first decline, before the check fails
// saying no node with its need took it (issue 890, ADR 0552); DefaultDeclineBound when zero.
DeclineBound time.Duration
// Toolchains are the nodes' toolchains: a kept verdict names the toolchain the nodes agree on. // Toolchains are the nodes' toolchains: a kept verdict names the toolchain the nodes agree on.
Toolchains ToolchainEntries Toolchains ToolchainEntries
Kept Kept Kept Kept
@@ -206,15 +203,12 @@ type state struct {
nodes map[string]bool nodes map[string]bool
clone time.Duration clone time.Duration
slow []string slow []string
// beside are the siblings read at their trunk's head, by repository (ADR 0548 §6).
beside map[string]string
} }
// Judge runs the case (it is the Judge of a Checker). A failure of the change, of a merge or of a step is a verdict; an error is only a case that cannot // Judge runs the case (it is the Judge of a Checker). A failure of the change, of a merge or of a step is a verdict; an error is only a case that cannot
// be judged at all. // be judged at all.
func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) { func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) {
st := &state{j: j, c: c, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{}, st := &state{j: j, c: c, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{}}
beside: map[string]string{}}
if v := st.judgeAll(ctx); v != nil { if v := st.judgeAll(ctx); v != nil {
return st.finish(ctx, *v), nil return st.finish(ctx, *v), nil
} }
@@ -236,13 +230,6 @@ func (st *state) finish(ctx context.Context, v delivery.Verdict) Outcome {
// fails the check, naming the machine it failed at. A try is not composed (it has no candidate); a composition that // fails the check, naming the machine it failed at. A try is not composed (it has no candidate); a composition that
// cannot be had leaves it not run, said; with no assigner running it is not judged, said at once. // cannot be had leaves it not run, said; with no assigner running it is not judged, said at once.
func (st *state) composed(ctx context.Context, v *delivery.Verdict) { func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
if st.c.Composition != nil && st.c.Try == nil {
// A carry whose candidate's fingerprints did not change keeps the composition it passed (ADR 0548 §5).
kept := *st.c.Composition
v.Composition = &kept
st.c.note("the composition is kept: no fingerprint of a module it composed changed")
return
}
if st.j.Compositions == nil || st.c.Try != nil { if st.j.Compositions == nil || st.c.Try != nil {
return return
} }
@@ -256,9 +243,6 @@ func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
return return
} }
withComposition(v, comp) withComposition(v, comp)
if v.Composition != nil && v.Composition.State != delivery.CompositionNotRun {
v.Composition.FPSet = st.c.FPSet // a carry keeps it while the fingerprints stay the same (ADR 0548 §5)
}
} }
// outcome fills the verdict's levels, times, machines and composition from what ran. // outcome fills the verdict's levels, times, machines and composition from what ran.
@@ -298,9 +282,6 @@ func (st *state) outcome(v delivery.Verdict) Outcome {
} }
} }
out := Outcome{Verdict: v, Slowest: slow} out := Outcome{Verdict: v, Slowest: slow}
if len(st.beside) > 0 {
out.Beside = st.beside
}
for _, t := range st.targets { for _, t := range st.targets {
if t.judged != "" { if t.judged != "" {
if out.Judged == nil { if out.Judged == nil {
@@ -362,7 +343,6 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
if len(beside) > 0 { if len(beside) > 0 {
args["beside"] = beside args["beside"] = beside
t.beside = besideLines(beside) t.beside = besideLines(beside)
besideAtTrunk(st.c, beside, st.beside)
} }
if p.Repository == catalogueRepository { if p.Repository == catalogueRepository {
t.runner = RunnerWorkspace // used by the build module when the merge changes it (ADR 0535 §1) t.runner = RunnerWorkspace // used by the build module when the merge changes it (ADR 0535 §1)
@@ -384,25 +364,11 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
return nil return nil
} }
// DefaultDeclineBound is an Examiner's DeclineBound when it names none: longer than a check runs, so a node with the
// need that is busy is waited for, and short enough that a mesh where no node has it says so.
const DefaultDeclineBound = 30 * time.Minute
// ask puts the target's check ask, waits for the step of the node that pulled it and judges its answer: the fetch, the // ask puts the target's check ask, waits for the step of the node that pulled it and judges its answer: the fetch, the
// pins of its go.mod files, then each level's digest, kept verdicts and tests. When the answer leaves packages unmet, // pins of its go.mod files, then each level's digest, kept verdicts and tests.
// their tests needing what that node lacks (issue 890), the passes are kept and the levels from the first such one are
// asked again of a node that has it (placed).
func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *delivery.Verdict { func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *delivery.Verdict {
return st.askNeeding(ctx, t, merged, st.levelsOf(), nil, nil)
}
// askNeeding is ask with the levels to run, what the node must have and the packages it tests: none for the first ask
// of a target; for the placed one the unmet packages' needs and those packages alone, so no package the first ask judged
// is run or counted again (ADR 0552 §4).
func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, levels, needs, packages []string) *delivery.Verdict {
p := t.pull p := t.pull
placed := len(needs) > 0 job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: st.levelsOf(), Kept: true}
job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: levels, Kept: true, Packages: packages}
if t.runner != "" { if t.runner != "" {
job.Runner, job.RunnerWhen = t.runner, buildGoDir job.Runner, job.RunnerWhen = t.runner, buildGoDir
} }
@@ -414,47 +380,17 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
kind = delivery.AskRecheck // a re-judge after a trunk move goes after every new head's check (ADR 0536 §4) kind = delivery.AskRecheck // a re-judge after a trunk move goes after every new head's check (ADR 0536 §4)
} }
a := delivery.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: askLanguage[t.module], Asker: delivery.CheckerModule, a := delivery.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: askLanguage[t.module], Asker: delivery.CheckerModule,
At: st.j.now().UTC(), Check: job, Needs: needs} At: st.j.now().UTC(), Check: job}
var step delivery.Step st.c.Progress.Set(fmt.Sprintf("the %s ask %s of %s#%d", kind, a.ID, p.Repository, p.Number), "")
var firstDecline time.Time if err := st.j.Asks.Put(ctx, a); err != nil {
for { return fail(t.module, fmt.Sprintf("the check of %s#%d could not be asked: %v", p.Repository, p.Number, err))
what := fmt.Sprintf("the %s ask %s of %s#%d", kind, a.ID, p.Repository, p.Number) }
if placed { step, err := st.j.Asks.Wait(ctx, a)
what += " for a node with " + strings.Join(needs, ", ") if err != nil {
if ctx.Err() != nil {
return fail(t.module, ctx.Err().Error())
} }
st.c.Progress.Set(what, "") return fail(t.module, fmt.Sprintf("the check of %s#%d: %v", p.Repository, p.Number, err))
st.c.Asked(a.ID)
if err := st.j.Asks.Put(ctx, a); err != nil {
return fail(t.module, fmt.Sprintf("the check of %s#%d could not be asked: %v", p.Repository, p.Number, err))
}
var err error
step, err = st.j.Asks.Wait(ctx, a)
if err != nil {
if ctx.Err() != nil {
return fail(t.module, ctx.Err().Error())
}
return fail(t.module, fmt.Sprintf("the check of %s#%d: %v", p.Repository, p.Number, err))
}
d, declined := delivery.DeclinedOf(step)
if !declined {
break
}
// A node that lacks the need dropped the ask (issue 890): it is put again, for a node that has it to take; the
// node that declined looks at the stream again only after its next pull's wait.
now := st.j.now()
if firstDecline.IsZero() {
firstDecline = now
}
bound := st.j.DeclineBound
if bound <= 0 {
bound = DefaultDeclineBound
}
if now.Sub(firstDecline) > bound {
return fail(step.Node, fmt.Sprintf("the check of %s#%d needs a node with %s, and none took it in %s: every node that pulled it lacks it (last %s)",
p.Repository, p.Number, strings.Join(needs, ", "), bound, step.Node))
}
st.j.logf("%s#%d: the ask %s was declined on %s, which lacks %s; asked again", p.Repository, p.Number, a.ID, step.Node, strings.Join(d.Lacks, ", "))
a.ID, a.At = buildstep.NewID(), now.UTC()
} }
// The fetch and the digests are what of the ask's time its levels' running did not take (Took.Clone). // The fetch and the digests are what of the ask's time its levels' running did not take (Took.Clone).
took := time.Duration(0) took := time.Duration(0)
@@ -478,28 +414,18 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
if st.c.Try == nil { if st.c.Try == nil {
st.j.writePreview(ctx, p, append(*merged, t.fetched)) st.j.writePreview(ctx, p, append(*merged, t.fetched))
} }
return fail(step.Node, fmt.Sprintf("%s#%d %s%s", p.Repository, p.Number, ConflictWords, strings.Join(t.fetched.Conflicts, ", "))) return fail(step.Node, fmt.Sprintf("%s#%d does not merge into the trunk: files in conflict: %s", p.Repository, p.Number, strings.Join(t.fetched.Conflicts, ", ")))
} }
t.changed = t.fetched.Changed t.changed = t.fetched.Changed
if !changesBuildGo(p.Repository, t.changed) { if !changesBuildGo(p.Repository, t.changed) {
t.runner = "" // the build module used no runner: the merge does not change it t.runner = "" // the build module used no runner: the merge does not change it
} }
if !placed { // the placed ask fetched the same merge: its preview and pins were read from the first *merged = append(*merged, t.fetched)
*merged = append(*merged, t.fetched) if v := st.checkPinsOf(ctx, t); v != nil {
if v := st.checkPinsOf(ctx, t); v != nil { return v
return v
}
} }
ran := 0.0 ran := 0.0
for _, la := range answer.Levels { for _, la := range answer.Levels {
if placed {
if len(la.Unmet) > 0 {
st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
return fail(step.Node, fmt.Sprintf("the %s level of %s#%d left %s untested on %s, which was asked for a node with %s",
la.Level, p.Repository, p.Number, unmetNames(la.Unmet), step.Node, strings.Join(needs, ", ")))
}
la.Kept = nil // it names its packages and takes no other's kept verdict; this is only a guard against an older module
}
v, elapsed := st.level(ctx, t, la) v, elapsed := st.level(ctx, t, la)
ran += elapsed ran += elapsed
if v != nil { if v != nil {
@@ -508,52 +434,12 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
} }
} }
st.clone += max(0, took-time.Duration(ran*float64(time.Second))) st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
if unmet := delivery.UnmetNeeds(answer.Levels); len(unmet) > 0 && !placed {
// Every level passed, its passes kept, and some packages were left to a node with what their tests need: the levels
// from the first that left one are asked of such a node, which takes the kept passes and tests those (issue 890).
var again, packages []string
seen := map[string]bool{}
for _, la := range answer.Levels {
if len(again) > 0 || len(la.Unmet) > 0 {
again = append(again, la.Level)
}
for _, u := range la.Unmet {
if !seen[u.Package] {
seen[u.Package] = true
packages = append(packages, u.Package)
}
}
}
st.j.logf("%s#%d: %s on %s left untested; asked of a node with %s", p.Repository, p.Number, unmetOf(answer.Levels), step.Node, strings.Join(unmet, ", "))
return st.askNeeding(ctx, t, merged, again, unmet, packages)
}
return nil return nil
} }
// unmetNames names the unmet packages and their needs.
func unmetNames(unmet []delivery.PackageNeeds) string {
var out []string
for _, u := range unmet {
out = append(out, u.Package+" (needs "+strings.Join(u.Needs, ", ")+")")
}
return strings.Join(out, ", ")
}
// unmetOf names the unmet packages of every level.
func unmetOf(levels []delivery.LevelAnswer) string {
var out []string
for _, la := range levels {
if len(la.Unmet) > 0 {
out = append(out, "at "+la.Level+" "+unmetNames(la.Unmet))
}
}
return strings.Join(out, "; ")
}
// level judges one level of a check ask's answer as the tools' answers were judged: the digest keys the kept verdicts // level judges one level of a check ask's answer as the tools' answers were judged: the digest keys the kept verdicts
// of the packages that passed, the kept ones taken are counted, and every test result is read and counted, also after // of the packages that passed, the kept ones taken are counted, and the first test result that did not pass is the
// one that did not pass, so the verdict names every failed test of the level (issue 1921). It answers the level's // verdict. It answers the level's running time (its tests' and setup's).
// running time (its tests' and setup's).
func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer) (*delivery.Verdict, float64) { func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer) (*delivery.Verdict, float64) {
level := la.Level level := la.Level
a := st.levels[level] a := st.levels[level]
@@ -575,7 +461,6 @@ func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer)
a.kept += k.Tests a.kept += k.Tests
} }
elapsed := 0.0 elapsed := 0.0
var failedResults []Tested
for _, raw := range la.Tests { for _, raw := range la.Tests {
var res Tested var res Tested
if err := json.Unmarshal(raw, &res); err != nil { if err := json.Unmarshal(raw, &res); err != nil {
@@ -604,12 +489,9 @@ func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer)
} }
} }
if !res.Passed { if !res.Passed {
failedResults = append(failedResults, res) return levelFailure(t, level, res), elapsed
} }
} }
if len(failedResults) > 0 {
return levelFailure(t, level, failedResults...), elapsed
}
return nil, elapsed return nil, elapsed
} }
@@ -751,60 +633,23 @@ func firstLine(s string) string {
return "" return ""
} }
// MaxFailureLine is the most bytes of a failure's line in a verdict's Failures: the first line of its words, which is // failure turns a level that did not pass into a verdict naming what failed first.
// its cause where the build module told one apart (issue 1901). Failed keeps the first failure's words whole. func levelFailure(t *target, level string, res Tested) *delivery.Verdict {
const MaxFailureLine = 300
// levelFailure turns a level that did not pass into a verdict: Failed names what failed first with its words whole,
// and Failures every failed test of the level's results with its first line, at most delivery.MaxFailures and an entry
// saying how many more (issue 1921).
func levelFailure(t *target, level string, results ...Tested) *delivery.Verdict {
v := delivery.Verdict{State: delivery.VerdictFail} v := delivery.Verdict{State: delivery.VerdictFail}
var all []delivery.FailedFirst switch {
for _, res := range results { case len(res.Failed) > 0:
if len(res.Failed) == 0 { f := res.Failed[0]
why := res.Why name := failedName(f)
if why == "" { if t.module == buildrun.NodeBuildModule && f.Package == "" {
why = "the " + level + " level did not pass" name = res.Module
}
all = append(all, delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: why})
continue
} }
for _, f := range res.Failed { v.Failed = delivery.FailedFirst{Test: level + ": " + name, Words: failureWords(f)}
name := failedName(f) default:
if t.module == buildrun.NodeBuildModule && f.Package == "" { why := res.Why
name = res.Module if why == "" {
} why = "the " + level + " level did not pass"
all = append(all, delivery.FailedFirst{Test: level + ": " + name, Words: failureWords(f)})
} }
v.Failed = delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: why}
} }
if len(all) == 0 { // no result: a level said failed with nothing to name
all = append(all, delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: "the " + level + " level did not pass"})
}
v.Failed = all[0]
v.Failures = failureLines(all)
return &v return &v
} }
// failureLines is every failure with the first line of its words, cut to MaxFailureLine, at most delivery.MaxFailures
// and then one entry saying how many more failed.
func failureLines(all []delivery.FailedFirst) []delivery.FailedFirst {
n := min(len(all), delivery.MaxFailures)
out := make([]delivery.FailedFirst, 0, n+1)
for _, f := range all[:n] {
out = append(out, delivery.FailedFirst{Test: f.Test, Machine: f.Machine, Words: clipLine(firstLine(f.Words), MaxFailureLine)})
}
if more := len(all) - n; more > 0 {
out = append(out, delivery.FailedFirst{Test: fmt.Sprintf("and %d more", more)})
}
return out
}
// clipLine cuts a line to at most n bytes on a whole character, marking the cut.
func clipLine(s string, n int) string {
if len(s) <= n {
return s
}
s = strings.ToValidUTF8(s[:n-3], "")
return s + "..."
}
@@ -56,22 +56,12 @@ func (k Kept) Keep(ctx context.Context, level, digest string, v delivery.Kept) (
// KeepPasses keeps, for the packages of a test result that passed with tests, a verdict under the digest the build module // KeepPasses keeps, for the packages of a test result that passed with tests, a verdict under the digest the build module
// named for the package at the level. A package that failed, has no tests or has no digest is never kept (ADR 0444 clause // named for the package at the level. A package that failed, has no tests or has no digest is never kept (ADR 0444 clause
// 1): it runs every check. A package with skipped tests is not kept either: a pass that skipped (a test needing a // 1): it runs every check. It answers the packages it kept now.
// toolchain the node lacks) would be reused for its digest and never run anywhere (issue 890). A build module that
// counts skips only per result, not per package, keeps nothing of a result in which a test skipped. It answers the
// packages it kept now.
func (k Kept) KeepPasses(ctx context.Context, repository, level, toolchain string, res Tested, digests map[string]string, at time.Time) ([]string, error) { func (k Kept) KeepPasses(ctx context.Context, repository, level, toolchain string, res Tested, digests map[string]string, at time.Time) ([]string, error) {
var kept []string var kept []string
perPackage := false
for _, p := range res.Packages {
perPackage = perPackage || p.Skipped > 0
}
if res.Skipped > 0 && !perPackage {
return kept, nil // an older build module: the skip cannot be placed in a package
}
for _, p := range res.Packages { for _, p := range res.Packages {
digest, ok := digests[p.Package] digest, ok := digests[p.Package]
if !ok || !p.Passed || p.NoTests || p.Tests == 0 || p.Skipped > 0 { if !ok || !p.Passed || p.NoTests || p.Tests == 0 {
continue continue
} }
if failedIn(res, p.Package) { if failedIn(res, p.Package) {
@@ -74,32 +74,6 @@ func TestOnlyPassingPackagesWithADigestAreKept(t *testing.T) { // 1287/AC3, AC4
} }
} }
func TestAResultWithSkippedTestsKeepsNothing(t *testing.T) { // issue 890: a pass that skipped proved nothing
ctx, st := context.Background(), newMemKV()
k := Kept{Store: st}
res := Tested{Level: "integration", Passed: true, Skipped: 2, Packages: []GoPackage{{Package: "m/a", Passed: true, Tests: 4}}}
kept, err := k.KeepPasses(ctx, "novox/mesh-catalog", "integration", "go1.26.0", res, map[string]string{"m/a": digA}, at0)
if err != nil || len(kept) != 0 {
t.Fatalf("kept %v, %v, want nothing", kept, err)
}
if keys, _ := st.Keys(ctx); len(keys) != 0 {
t.Errorf("the bucket holds %v, want none", keys)
}
}
func TestOnlyTheSkippingPackageOfAResultIsWithheld(t *testing.T) { // issue 890
ctx, st := context.Background(), newMemKV()
k := Kept{Store: st}
res := Tested{Level: "integration", Passed: true, Skipped: 2, Packages: []GoPackage{
{Package: "m/a", Passed: true, Tests: 4, Skipped: 2},
{Package: "m/b", Passed: true, Tests: 3},
}}
kept, err := k.KeepPasses(ctx, "novox/mesh-catalog", "integration", "go1.26.0", res, map[string]string{"m/a": digA, "m/b": digB}, at0)
if err != nil || strings.Join(kept, ",") != "m/b" {
t.Fatalf("kept %v, %v, want only m/b", kept, err)
}
}
func TestAKeptValueOfAnotherDigestIsNotRead(t *testing.T) { // 1287/AC7 func TestAKeptValueOfAnotherDigestIsNotRead(t *testing.T) { // 1287/AC7
ctx, st := context.Background(), newMemKV() ctx, st := context.Background(), newMemKV()
k := Kept{Store: st} k := Kept{Store: st}
@@ -49,34 +49,6 @@ func firstFailed(v delivery.Verdict) string {
return firstWordsLine(f.Words) return firstWordsLine(f.Words)
} }
// failedLines is everything a verdict names as failed, one line each: every failed test with its first line (issue
// 1921), or what failed first when the verdict names no list (a conflict, a refused step, an older verdict).
func failedLines(v delivery.Verdict) []string {
if len(v.Failures) == 0 {
if f := firstFailed(v); f != "" {
return []string{f}
}
return nil
}
lines := make([]string, 0, len(v.Failures))
for _, f := range v.Failures {
lines = append(lines, firstFailed(delivery.Verdict{Failed: f}))
}
return lines
}
// failedCount is how many tests a verdict names as failed, the ones after delivery.MaxFailures counted too.
func failedCount(v delivery.Verdict) int {
n := len(v.Failures)
if n > delivery.MaxFailures {
var more int
if _, err := fmt.Sscanf(v.Failures[n-1].Test, "and %d more", &more); err == nil {
return delivery.MaxFailures + more
}
}
return n
}
// firstWordsLine is the first line of a failure's words: the words are several lines (issue 1901) and the status line one. // firstWordsLine is the first line of a failure's words: the words are several lines (issue 1901) and the status line one.
func firstWordsLine(w string) string { return firstLine(w) } func firstWordsLine(w string) string { return firstLine(w) }
@@ -85,20 +57,18 @@ type pieces struct {
head, commits, counts, shortCounts, times, shortTimes, over, builds, composition, shortComposition, group string head, commits, counts, shortCounts, times, shortTimes, over, builds, composition, shortComposition, group string
// judged names the program that judged a change to a build module (ADR 0535 §5); never dropped. // judged names the program that judged a change to a build module (ADR 0535 §5); never dropped.
judged string judged string
// carried says, of a carried verdict, the commits it was carried from (ADR 0548 §8); never dropped.
carried string
} }
// pullKey names a pull request of a case: `<owner/repo>#<number>`. // pullKey names a pull request of a case: `<owner/repo>#<number>`.
func pullKey(p Pull) string { return fmt.Sprintf("%s#%d", p.Repository, p.Number) } func pullKey(p Pull) string { return fmt.Sprintf("%s#%d", p.Repository, p.Number) }
func (p pieces) join(drop int) string { func (p pieces) join(drop int) string {
parts := []string{p.head, p.commits, p.carried, p.counts, p.judged} parts := []string{p.head, p.commits, p.counts, p.judged}
switch { switch {
case drop < 3 && p.times != "": case drop < 3 && p.times != "":
parts = append(parts, p.times) parts = append(parts, p.times)
case drop == 3 && p.shortTimes != "": case drop == 3 && p.shortTimes != "":
parts = []string{p.head, p.commits, p.carried, p.shortCounts, p.judged, p.shortTimes} parts = []string{p.head, p.commits, p.shortCounts, p.judged, p.shortTimes}
} }
if p.over != "" { if p.over != "" {
parts = append(parts, p.over) parts = append(parts, p.over)
@@ -173,39 +143,6 @@ func buildWords(builds []BuildNote) string {
return "builds: " + strings.Join(parts, ", ") return "builds: " + strings.Join(parts, ", ")
} }
// carriedWords say the commits a verdict was carried from, for the line of pull request p (ADR 0548 §8): its own trunk's
// commit as twelve characters, a sibling's with its name before it; only those that moved, sorted. Empty when none did.
func carriedWords(from map[string]string, v delivery.Verdict, cs *Case, p Pull) string {
var own string
var others []string
for r, old := range from {
now, ok := v.Trunk[r]
if cs != nil && !ok {
now, ok = cs.Trunk[r]
}
if !ok {
now = v.Beside[r]
}
if now == "" || now == old {
continue
}
if strings.EqualFold(r, p.Repository) {
own = short12(old)
continue
}
name := r
if i := strings.LastIndex(r, "/"); i >= 0 {
name = r[i+1:]
}
others = append(others, name+" "+short12(old))
}
slices.Sort(others)
if own != "" {
others = append([]string{own}, others...)
}
return strings.Join(others, ", ")
}
// parts makes the pieces of the line of a pull request of a case. // parts makes the pieces of the line of a pull request of a case.
func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string, judged map[string]string) pieces { func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string, judged map[string]string) pieces {
var pc pieces var pc pieces
@@ -214,23 +151,10 @@ func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []s
switch v.State { switch v.State {
case delivery.VerdictRunning: case delivery.VerdictRunning:
pc.head = "checking " + commits pc.head = "checking " + commits
if cs != nil && cs.CarriedFrom != nil {
// Out of date and being carried to the new commit (ADR 0548 §6): pending, and saying so.
pc.head = "being carried to " + commits
if w := carriedWords(cs.CarriedFrom, v, cs, p); w != "" {
pc.carried = "from " + w
}
}
case delivery.VerdictPass: case delivery.VerdictPass:
pc.head = "pass " + commits pc.head = "pass " + commits
if w := carriedWords(v.CarriedFrom, v, cs, p); w != "" {
pc.carried = "carried from " + w // the counts that follow say what the carry ran and kept
}
default: default:
pc.head = "failed: " + firstFailed(v) pc.head = "failed: " + firstFailed(v)
if n := failedCount(v); n > 1 {
pc.head = fmt.Sprintf("failed (%d tests): %s", n, firstFailed(v))
}
pc.commits = commits pc.commits = commits
} }
var counts []string var counts []string
@@ -1,34 +0,0 @@
package check
import (
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// **An ask the build seat gave up on ends its step** (novox/hq issue 734): mesh-sdk buildrun.LostAskWaiter hands the
// waiting asker a failed step on the node buildrun.LostNode when DEAD_LETTERS keeps the ask's copy (buildrun.HearDeadLetters,
// granted by buscompose DeadLetterReads), and, under buildrun.NotTakenBound, when no build node took the ask within the
// bound. What follows is the checker's: a try ends as error (Tryer.run), with the not-taken bound at the step bound
// (Tryer.NotTaken); a check is asked again under its claim, at most LostAskTries times, then fails as not heard and is
// judged again when the nodes change (Checker.try). A check's ask has no not-taken bound: it waits in the stream as long as
// the backlog is (ADR 0525 §9), and a check ended for a backlog would owe its pull request a failure.
// LostAskTries is how often a check asks again whose ask was given up, before it fails as not heard (UnheardWords) and is
// judged again when the nodes change.
const LostAskTries = 3
// IsLostAsk says whether a verdict failed because the build seat gave its ask up (buildrun.LostWords).
func IsLostAsk(v delivery.Verdict) bool {
return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, buildrun.LostWords)
}
// IsNotTaken says whether a verdict failed because no build node took its ask within the bound (buildrun.NotTakenWords).
func IsNotTaken(v delivery.Verdict) bool {
return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, buildrun.NotTakenWords)
}
// HearingBucket is the checker's state hearing: whether it hears DEAD_LETTERS (buildrun.Hearing under
// buildrun.HearingKey), which checker_status says (dead_letters).
const HearingBucket = delivery.CheckerModule + "_" + buildrun.HearingState
@@ -1,170 +0,0 @@
package check
import (
"context"
"encoding/json"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// losingAsks is the build seat over a bus that gives up the first `lose` asks put (DEAD_LETTERS keeps them, as on
// 2026-10-26 during a bus reload: try-1793017340003993788) and answers the rest; with never set it neither answers nor
// gives up (an ask no node takes).
type losingAsks struct {
*buildrun.FakeAsks
answer func(delivery.StepAsk) delivery.Step
dead *buildrun.DeadLetters
never bool
mu sync.Mutex
lose int
}
func (l *losingAsks) PublishAsk(ctx context.Context, subject, id string, data []byte) error {
if err := l.FakeAsks.PublishAsk(ctx, subject, id, data); err != nil {
return err
}
l.mu.Lock()
lost := l.lose > 0
if lost {
l.lose--
}
l.mu.Unlock()
switch {
case l.never:
case lost:
h := map[string][]string{buildrun.DeadHeaderMsgID: {id}, buildrun.DeadHeaderDeliveries: {"5"}, buildrun.DeadHeaderStream: {"SEAT_NODE_BUILD_GO"},
buildrun.DeadHeaderGaveUp: {"2026-10-26T12:26:40Z"}}
g, ok := buildrun.GivenUpOf(h)
if !ok {
return nil
}
go l.dead.Heard(g)
default:
a, err := delivery.DecodeStepAsk(data)
if err != nil {
return err
}
l.Write(l.answer(a))
}
return nil
}
func losing(b *bench, lose int) *losingAsks {
l := &losingAsks{FakeAsks: &buildrun.FakeAsks{}, answer: b.answer, dead: &buildrun.DeadLetters{}, lose: lose}
b.ex.Asks = buildrun.Asker{Publisher: l, Waiter: buildrun.LostAskWaiter{Inner: l.FakeAsks, Dead: l.dead}, Bound: 5 * time.Second}
return l
}
// endable is the tries state as the end tool sees it.
type endable struct{ kv *memKV }
func (e endable) Entry(key string) (json.RawMessage, uint64, error) {
v, rev, _, err := e.kv.Get(context.Background(), key)
return v, rev, err
}
func (e endable) Update(key string, value []byte, revision uint64) error {
_, err := e.kv.Update(context.Background(), key, value, revision)
return err
}
// Issue 734: a try whose ask the build seat gave up on ends as error saying so, and the next ask of its head is a new try;
// one no node takes ends at the step bound; one ended by hand frees its place and stops being waited for.
func TestAnAskGivenUpOrNeverTakenEndsItsTryAsErrorAndItsHeadIsAskedAnew(t *testing.T) {
ctx := context.Background()
for _, row := range []struct {
name string
lose int
never bool
want string
}{
{"given up by the build seat", 1, false, buildrun.LostWords + " after 5 deliveries at 2026-10-26T12:26:40Z"},
{"taken by no build node within the bound", 0, true, buildrun.NotTakenWords + " within 200ms"},
} {
t.Run(row.name, func(t *testing.T) {
kv := newMemKV()
at := time.Date(2026, 10, 26, 12, 22, 20, 0, time.UTC)
id, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, at)
if err != nil {
t.Fatal(err)
}
b := newBench(t)
l := losing(b, row.lose)
l.never = row.never
tr := &Tryer{Tries: kv, Judge: b.ex, NotTaken: 200 * time.Millisecond}
raw, _, _, _ := kv.Get(ctx, id)
tr.Put(ctx, id, raw)
tr.Wait()
got := readTry(t, kv, id)
if got.State != TryError || len(got.Failed) != 1 || !strings.Contains(got.Failed[0], row.want) || !strings.Contains(got.Said, row.want) {
t.Fatalf("try = %+v, want error saying %q", got, row.want)
}
again, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, at.Add(time.Minute))
if err != nil || again == id {
t.Errorf("the head asked again answered %q, %v, want a new try", again, err)
}
})
}
t.Run("ended by hand", func(t *testing.T) {
kv := newMemKV()
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now())
b := newBench(t)
losing(b, 0).never = true
tr := &Tryer{Tries: kv, Judge: b.ex} // no bound: only the hand ends it
raw, _, _, _ := kv.Get(ctx, id)
tr.Put(ctx, id, raw)
deadline := time.Now().Add(2 * time.Second)
for readTry(t, kv, id).State != TryRunning && time.Now().Before(deadline) {
time.Sleep(5 * time.Millisecond)
}
if _, err := EndTry(endable{kv}, id, "", time.Now()); err == nil {
t.Error("an end without a why was taken")
}
ended, err := EndTry(endable{kv}, id, "its ask was lost in the bus reload", time.Now())
if err != nil || ended.State != TryError || !strings.HasPrefix(ended.Said, "ended by hand ("+TryEndTool+"): its ask was lost") {
t.Fatalf("ended = %+v, %v", ended, err)
}
raw, _, _, _ = kv.Get(ctx, id)
tr.Put(ctx, id, raw) // the watch of the tries hands the end over
tr.Wait() // returns: the try's judging stopped
if got := readTry(t, kv, id); got.State != TryError || got.Said != ended.Said {
t.Errorf("try = %+v, want the hand's end kept", got)
}
if _, err := EndTry(endable{kv}, id, "again", time.Now()); err == nil || !strings.Contains(err.Error(), "ended already") {
t.Errorf("a second end answered %v", err)
}
})
}
// Issue 734: a pull request's check whose ask the build seat gave up on is asked again under its claim, its status
// pending, and judged; one given up every time fails as not heard and is judged again when the nodes change.
func TestACheckWhoseAskWasGivenUpIsAskedAgain(t *testing.T) {
ctx := context.Background()
for _, row := range []struct {
lose int
state string
asks int
}{
{1, delivery.VerdictPass, 2},
{LostAskTries, delivery.VerdictFail, LostAskTries},
} {
b := newBench(t)
l := losing(b, row.lose)
r := newRig(t, b.ex)
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
v := r.verdict(t, keyOf(t, "feat/x", cand))
if v.State != row.state || len(l.Asks()) != row.asks {
t.Errorf("lose %d: verdict %s (%s) after %d asks, want %s after %d", row.lose, v.State, v.Failed.Words, len(l.Asks()), row.state, row.asks)
}
if row.state == delivery.VerdictFail && (!IsLostAsk(v) || !IsNodesFailure(v)) {
t.Errorf("a check given up every time = %q, want lost and judged again when the nodes change", v.Failed.Words)
}
}
}
@@ -240,12 +240,7 @@ func TestTheSiblingsAreWhereTheTestsStateThem(t *testing.T) { // review D2, ADR
catalogue + " go.mod " + head2: "module git.novox.be/novox/mesh-catalog\n"}, onTrunk: map[string]bool{"cb54756c3865": true}} catalogue + " go.mod " + head2: "module git.novox.be/novox/mesh-catalog\n"}, onTrunk: map[string]bool{"cb54756c3865": true}}
b.ex.Forge = forge b.ex.Forge = forge
// A group of the controller and the catalogue: the controller's tests read the catalogue's trunk, not the group's head. // A group of the controller and the catalogue: the controller's tests read the catalogue's trunk, not the group's head.
out := b.judge(t, caseOf(t, pullOf(controller, 7, head1), pullOf(catalogue, 12, head2))) b.judge(t, caseOf(t, pullOf(controller, 7, head1), pullOf(catalogue, 12, head2)))
// ADR 0548 §6, issue 568's own example: the verdict names the node-engine at the commit the mesh runs, and mesh-sdk at the
// trunk the catalogue's tests read it at; the catalogue, a member here, is not named.
if out.Beside[host] != hostRuns || out.Beside[catalogue] != "" || out.Beside[SDKRepository] != "9999999999999999999999999999999999999999" {
t.Errorf("the siblings named for the verdict = %v, want the node-engine at %s and mesh-sdk at its trunk", out.Beside, hostRuns[:12])
}
got := besideOf(t, b, controller) got := besideOf(t, b, controller)
if c := got[catalogue]; c.Commit != trunkS || c.Ref != "" { if c := got[catalogue]; c.Commit != trunkS || c.Ref != "" {
t.Errorf("the catalogue beside the controller = %+v, want its trunk %s", c, trunkS) t.Errorf("the catalogue beside the controller = %+v, want its trunk %s", c, trunkS)
@@ -329,11 +324,7 @@ func TestTheChangesOwnRepositoryIsBesideItAtItsTrunk(t *testing.T) {
func TestACatalogueChangeHasTheCatalogueBesideItAtItsTrunk(t *testing.T) { func TestACatalogueChangeHasTheCatalogueBesideItAtItsTrunk(t *testing.T) {
b := newBench(t) b := newBench(t)
b.ex.Trunks = trunksOf(t, map[string]string{catalogueRepository: trunkA, SDKRepository: trunkS, controllerRepo: trunkA2}) b.ex.Trunks = trunksOf(t, map[string]string{catalogueRepository: trunkA, SDKRepository: trunkS, controllerRepo: trunkA2})
out := b.judge(t, caseOf(t, pullOf(catalogueRepository, 12, head1))) b.judge(t, caseOf(t, pullOf(catalogueRepository, 12, head1)))
// ADR 0548 §6: the outcome names the siblings read at their trunk's head, never the repository it merges.
if out.Beside[SDKRepository] != trunkS || out.Beside[controllerRepo] != trunkA2 || out.Beside[catalogueRepository] != "" {
t.Errorf("the siblings named for the verdict = %v, want mesh-sdk and mesh-controller at their trunks and not the catalogue", out.Beside)
}
got := besideOf(t, b, catalogueRepository) got := besideOf(t, b, catalogueRepository)
if c := got[catalogueRepository]; c.Commit != trunkA || c.Ref != "" { if c := got[catalogueRepository]; c.Commit != trunkA || c.Ref != "" {
t.Fatalf("the catalogue beside its own change = %+v, want its trunk %s", c, trunkA) t.Fatalf("the catalogue beside its own change = %+v, want its trunk %s", c, trunkA)
@@ -75,40 +75,3 @@ func TestCheckerStatusNamesTheWaitingChecksByKind(t *testing.T) { // 1897/AC4
t.Errorf("waiting = %v, want %v", st.Waiting, want) t.Errorf("waiting = %v, want %v", st.Waiting, want)
} }
} }
// 568 note 21: a verdict that read the node-engine at its running commit is not waiting for a trunk move while that commit
// is still the one the mesh runs, though the node-engine's trunk has moved on (ADR 0548 §6).
func TestCheckerStatusReadsTheNodeEngineAtItsRunningCommit(t *testing.T) { // 568/21
cand := candidateOf(t, group(1), pullOf(repoA, 1, head1))
ckey, _ := delivery.CandidatesKey(group(1))
vkey := keyOf(t, group(1), cand)
dc, _ := delivery.DecodeCandidate(cand)
judged, err := delivery.EncodeVerdict(delivery.Verdict{Group: group(1), Heads: dc.HeadsHash, Trunk: map[string]string{repoA: trunkA}, Beside: map[string]string{nodeEngineRepo: "run111"}, State: delivery.VerdictPass, Levels: passing().Levels})
if err != nil {
t.Fatal(err)
}
heads := func() (map[string]string, error) {
return map[string]string{repoA: trunkA, nodeEngineRepo: "trunk222"}, nil
}
for _, row := range []struct {
name string
runs runsAt
want int
}{
{"the running commit the verdict read", runsAt{nodeEngineModule: "run111"}, 0},
{"the running commit moved", runsAt{nodeEngineModule: "run999"}, 1},
{"no running commit known: the trunk's", nil, 1},
} {
rd := Reader{CandidateState: fakeState{ckey: cand}.read(), VerdictState: fakeState{vkey: judged}.read(), Trunks: heads}
if row.runs != nil {
rd.Runs = row.runs
}
st, err := rd.Status()
if err != nil {
t.Fatal(err)
}
if got := len(st.Waiting[KindTrunk]); got != row.want {
t.Errorf("%s: waiting for a trunk move = %v, want %d", row.name, st.Waiting[KindTrunk], row.want)
}
}
}
@@ -8,7 +8,6 @@ import (
"time" "time"
"git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
) )
// The read tools (novox/hq ADR 0484 clauses 8 and 13, issue 1686): what the checker stored, as stored, with the line of the // The read tools (novox/hq ADR 0484 clauses 8 and 13, issue 1686): what the checker stored, as stored, with the line of the
@@ -40,14 +39,7 @@ type Reader struct {
StepBound time.Duration // how long the checker waits for one step; DefaultStepBound when 0 StepBound time.Duration // how long the checker waits for one step; DefaultStepBound when 0
// Trunks are the trunk heads of every repository (the gate's source); nil names no re-judge after a trunk move. // Trunks are the trunk heads of every repository (the gate's source); nil names no re-judge after a trunk move.
Trunks func() (map[string]string, error) Trunks func() (map[string]string, error)
// TurnState is the gate's merge turns (ADR 0548 §3); without it no check is named as held by a turn. Now func() time.Time
TurnState StateRead
// Runs says which commit the mesh runs of a module (the indexer's modules), as the checker's decision reads it: the
// node-engine a verdict names is compared at its running commit, not at the trunk's (ADR 0548 §6); nil knows none.
Runs interface{ CommitOf(module string) string }
// HearingState is the checker's state hearing: whether it hears DEAD_LETTERS (issue 734).
HearingState StateRead
Now func() time.Time
} }
// VerdictRead is a verdict as stored with the full report of the status of each pull request it judged (Report), when the // VerdictRead is a verdict as stored with the full report of the status of each pull request it judged (Report), when the
@@ -97,46 +89,37 @@ func (r Reader) read(key string, raw json.RawMessage) (VerdictRead, error) {
return out, nil return out, nil
} }
// Verdicts are the verdicts the checker holds, by key; with a group only that group's. The keys are narrowed by the group // Verdicts are the verdicts the checker holds, by key; with a group only that group's.
// before any value is read, at most MaxReads are read (sixteen at a time), and cut says when it stopped there. func (r Reader) Verdicts(group string) ([]VerdictRead, error) {
func (r Reader) Verdicts(group string) (out []VerdictRead, cut string, err error) {
if r.VerdictState.Keys == nil || r.VerdictState.Get == nil { if r.VerdictState.Keys == nil || r.VerdictState.Get == nil {
return nil, "", fmt.Errorf("the verdicts state cannot be read here") return nil, fmt.Errorf("the verdicts state cannot be read here")
} }
keys, err := r.VerdictState.Keys() keys, err := r.VerdictState.Keys()
if err != nil { if err != nil {
return nil, "", err return nil, err
} }
sort.Strings(keys) sort.Strings(keys)
var matching []string var out []VerdictRead
for _, k := range keys { for _, k := range keys {
if group != "" { if group != "" {
if g, _, err := delivery.ParseCheckerVerdictKey(k); err != nil || g != group { if g, _, err := delivery.ParseCheckerVerdictKey(k); err != nil || g != group {
continue continue
} }
} }
matching = append(matching, k) raw, err := r.VerdictState.Get(k)
} if err != nil {
matching, cut = capKeys(matching, "give a group to narrow the answer") return nil, err
rows := make([]VerdictRead, len(matching))
found := make([]bool, len(matching))
if err := parallel(len(matching), func(i int) error {
raw, err := r.VerdictState.Get(matching[i])
if err != nil || len(raw) == 0 {
return err
} }
rows[i], err = r.read(matching[i], raw) if len(raw) == 0 {
found[i] = err == nil continue
return err
}); err != nil {
return nil, "", err
}
for i, row := range rows {
if found[i] {
out = append(out, row)
} }
vr, err := r.read(k, raw)
if err != nil {
return nil, err
}
out = append(out, vr)
} }
return out, cut, nil return out, nil
} }
// Pull answers for one pull request: the verdict of the candidate that holds it, with its report; for one the indexer holds // Pull answers for one pull request: the verdict of the candidate that holds it, with its report; for one the indexer holds
@@ -150,95 +133,74 @@ func (r Reader) Pull(repository string, number int) (VerdictRead, error) {
return VerdictRead{}, err return VerdictRead{}, err
} }
sort.Strings(keys) sort.Strings(keys)
keys, cut := capKeys(keys, "") for _, k := range keys {
// The candidates are read in batches, each in parallel and taken in key order, so the read stops at the batch that holds it. raw, err := r.CandidateState.Get(k)
for from := 0; from < len(keys); from += ReadWorkers * 4 { if err != nil || len(raw) == 0 {
batch := keys[from:min(from+ReadWorkers*4, len(keys))] continue
cands := make([]delivery.Candidate, len(batch)) }
ok := make([]bool, len(batch)) c, err := delivery.DecodeCandidate(raw)
_ = parallel(len(batch), func(i int) error { if err != nil {
raw, err := r.CandidateState.Get(batch[i]) continue
if err != nil || len(raw) == 0 { }
return nil for _, p := range c.Pulls {
} if p.Repository != repository || p.Number != number {
c, err := delivery.DecodeCandidate(raw)
cands[i], ok[i] = c, err == nil
return nil
})
for i, c := range cands {
if !ok[i] {
continue continue
} }
for _, p := range c.Pulls { vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
if p.Repository != repository || p.Number != number { if err != nil {
continue return VerdictRead{}, err
}
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
if err != nil {
return VerdictRead{}, err
}
vraw, err := r.VerdictState.Get(vkey)
if err != nil {
return VerdictRead{}, err
}
if len(vraw) == 0 {
return VerdictRead{Key: vkey, Note: "no verdict yet for the heads of its candidate"}, nil
}
return r.read(vkey, vraw)
} }
vraw, err := r.VerdictState.Get(vkey)
if err != nil {
return VerdictRead{}, err
}
if len(vraw) == 0 {
return VerdictRead{Key: vkey, Note: "no verdict yet for the heads of its candidate"}, nil
}
return r.read(vkey, vraw)
} }
} }
if cut != "" {
return VerdictRead{}, fmt.Errorf("no candidate among the first %d holds %s#%d", MaxReads, repository, number)
}
return VerdictRead{}, fmt.Errorf("no candidate holds %s#%d", repository, number) return VerdictRead{}, fmt.Errorf("no candidate holds %s#%d", repository, number)
} }
// Kept is the kept verdicts: one key's value as stored, or with an empty key the keys of a level (or every level) and their values. // Kept is the kept verdicts: one key's value as stored, or with an empty key the keys of a level (or every level) and their values.
func (r Reader) Kept(level, digest string) (map[string]json.RawMessage, string, error) { func (r Reader) Kept(level, digest string) (map[string]json.RawMessage, error) {
if r.KeptState.Keys == nil || r.KeptState.Get == nil { if r.KeptState.Keys == nil || r.KeptState.Get == nil {
return nil, "", fmt.Errorf("the kept state cannot be read here") return nil, fmt.Errorf("the kept state cannot be read here")
} }
if digest != "" { if digest != "" {
key, err := delivery.KeptKey(level, digest) key, err := delivery.KeptKey(level, digest)
if err != nil { if err != nil {
return nil, "", err return nil, err
} }
raw, err := r.KeptState.Get(key) raw, err := r.KeptState.Get(key)
if err != nil { if err != nil {
return nil, "", err return nil, err
} }
if len(raw) == 0 { if len(raw) == 0 {
return nil, "", fmt.Errorf("no kept verdict %s", key) return nil, fmt.Errorf("no kept verdict %s", key)
} }
return map[string]json.RawMessage{key: raw}, "", nil return map[string]json.RawMessage{key: raw}, nil
} }
keys, err := r.KeptState.Keys() keys, err := r.KeptState.Keys()
if err != nil { if err != nil {
return nil, "", err return nil, err
} }
sort.Strings(keys) sort.Strings(keys)
var matching []string
for _, k := range keys {
if l, _, err := delivery.ParseKeptKey(k); err == nil && (level == "" || l == level) {
matching = append(matching, k)
}
}
matching, cut := capKeys(matching, "give a level and a digest to read one")
values := make([]json.RawMessage, len(matching))
if err := parallel(len(matching), func(i int) (err error) {
values[i], err = r.KeptState.Get(matching[i])
return err
}); err != nil {
return nil, "", err
}
out := map[string]json.RawMessage{} out := map[string]json.RawMessage{}
for i, k := range matching { for _, k := range keys {
if len(values[i]) > 0 { if l, _, err := delivery.ParseKeptKey(k); err != nil || (level != "" && l != level) {
out[k] = values[i] continue
}
raw, err := r.KeptState.Get(k)
if err != nil {
return nil, err
}
if len(raw) > 0 {
out[k] = raw
} }
} }
return out, cut, nil return out, nil
} }
// PreviewRead is one preview as stored, with the parts of its key. // PreviewRead is one preview as stored, with the parts of its key.
@@ -252,18 +214,17 @@ type PreviewRead struct {
// Previews are the previews the checker holds, each as stored. With a repository only that repository's; with a head (a // Previews are the previews the checker holds, each as stored. With a repository only that repository's; with a head (a
// prefix of at least 7 characters is enough) only previews of that head. A key that is no preview key is listed as unreadable. // prefix of at least 7 characters is enough) only previews of that head. A key that is no preview key is listed as unreadable.
func (r Reader) Previews(repository, head string) ([]PreviewRead, []string, string, error) { func (r Reader) Previews(repository, head string) ([]PreviewRead, []string, error) {
if r.PreviewState.Keys == nil || r.PreviewState.Get == nil { if r.PreviewState.Keys == nil || r.PreviewState.Get == nil {
return nil, nil, "", fmt.Errorf("the previews state cannot be read here") return nil, nil, fmt.Errorf("the previews state cannot be read here")
} }
keys, err := r.PreviewState.Keys() keys, err := r.PreviewState.Keys()
if err != nil { if err != nil {
return nil, nil, "", err return nil, nil, err
} }
sort.Strings(keys) sort.Strings(keys)
type part struct{ repo, head, trunk string } out := []PreviewRead{}
parts := map[string]part{} var bad []string
var matching, bad []string
for _, k := range keys { for _, k := range keys {
repo, h, trunk, err := delivery.ParsePreviewKey(k) repo, h, trunk, err := delivery.ParsePreviewKey(k)
if err != nil { if err != nil {
@@ -273,25 +234,15 @@ func (r Reader) Previews(repository, head string) ([]PreviewRead, []string, stri
if (repository != "" && repo != repository) || (head != "" && !strings.HasPrefix(h, head)) { if (repository != "" && repo != repository) || (head != "" && !strings.HasPrefix(h, head)) {
continue continue
} }
parts[k] = part{repo, h, trunk} raw, err := r.PreviewState.Get(k)
matching = append(matching, k) if err != nil {
} return nil, nil, err
matching, cut := capKeys(matching, "give a repository and a head to narrow the answer") }
values := make([]json.RawMessage, len(matching)) if len(raw) > 0 {
if err := parallel(len(matching), func(i int) (err error) { out = append(out, PreviewRead{Key: k, Repository: repo, Head: h, Trunk: trunk, Preview: raw})
values[i], err = r.PreviewState.Get(matching[i])
return err
}); err != nil {
return nil, nil, "", err
}
out := []PreviewRead{}
for i, k := range matching {
if len(values[i]) > 0 {
p := parts[k]
out = append(out, PreviewRead{Key: k, Repository: p.repo, Head: p.head, Trunk: p.trunk, Preview: values[i]})
} }
} }
return out, bad, cut, nil return out, bad, nil
} }
// RunningStep is a step in flight on a build module. // RunningStep is a step in flight on a build module.
@@ -320,68 +271,7 @@ type Status struct {
// Waiting are the candidates whose check waits, by kind (ADR 0536 §4): new (no verdict for its heads) and nodes (failed // Waiting are the candidates whose check waits, by kind (ADR 0536 §4): new (no verdict for its heads) and nodes (failed
// for the nodes, asked again when they change) start before trunk (judged on trunk heads that moved since). // for the nodes, asked again when they change) start before trunk (judged on trunk heads that moved since).
Waiting map[string][]string `json:"waiting_by_kind"` Waiting map[string][]string `json:"waiting_by_kind"`
// HeldByTurn are the candidates whose check an approved merge's turn holds off (ADR 0548 §3), by group: the repository and Notes []string `json:"notes,omitempty"`
// the group of the merge that waits. Their check is claimed when that turn goes.
HeldByTurn map[string]string `json:"held_by_turn"`
// DeadLetters says whether the checker hears the asks the build seats give up on (issue 734): deaf, a try or check
// whose ask is given up waits until its bound.
DeadLetters DeadLettersHeard `json:"dead_letters"`
Notes []string `json:"notes,omitempty"`
}
// DeadLettersHeard is the checker's hearing of DEAD_LETTERS as the status says it: healthy or not, and why.
type DeadLettersHeard struct {
Heard bool `json:"heard"`
Why string `json:"why"`
}
// deadLetters reads the checker's kept hearing (buildrun.HearingHealth).
func (r Reader) deadLetters() DeadLettersHeard {
if r.HearingState.Get == nil {
return DeadLettersHeard{Why: "the checker's state hearing is not readable here"}
}
raw, err := r.HearingState.Get(buildrun.HearingKey)
if err != nil {
return DeadLettersHeard{Why: "the checker's state hearing cannot be read: " + err.Error()}
}
a := buildrun.HearingHealth(raw)
return DeadLettersHeard{Heard: a.Healthy, Why: a.Why}
}
// turns reads the gate's merge turns, by repository; nil when they cannot be read, said in a note.
func (r Reader) turns(st *Status) *Turns {
if r.TurnState.Keys == nil || r.TurnState.Get == nil {
st.Notes = append(st.Notes, "the gate's merge turns are not readable here, so no check is named as held by one")
return nil
}
keys, err := r.TurnState.Keys()
if err != nil {
st.Notes = append(st.Notes, "the gate's merge turns cannot be listed, so no check is named as held by one: "+err.Error())
return nil
}
t := &Turns{}
for _, k := range keys {
if raw, err := r.TurnState.Get(k); err == nil && len(raw) > 0 {
t.Put(k, raw, false)
}
}
return t
}
// heldBy says the turn that holds a candidate's check off, or "".
func heldBy(t *Turns, c delivery.Candidate, heads map[string]string) string {
cs := CaseOf(c, heads)
if t == nil || cs == nil {
return ""
}
repos := make([]string, 0, len(cs.Trunk))
for r := range cs.Trunk {
repos = append(repos, r)
}
if repo, w, ok := t.Holding(repos, cs.Heads); ok {
return repo + ": the approved merge of " + w.Group + " waits on it"
}
return ""
} }
// Status reads the build modules' steps and the candidates against their verdicts. // Status reads the build modules' steps and the candidates against their verdicts.
@@ -410,28 +300,16 @@ func (r Reader) Status() (Status, error) {
st.Notes = append(st.Notes, "the steps of "+m+" cannot be listed: "+err.Error()) st.Notes = append(st.Notes, "the steps of "+m+" cannot be listed: "+err.Error())
continue continue
} }
// A step that runs started lately: the newest ids are read first, at most MaxReads of them. sort.Strings(keys)
newestFirst(keys) for _, k := range keys {
keys, cut := capKeys(keys, "") raw, err := sr.Get(k)
if cut != "" {
st.Notes = append(st.Notes, "the steps of "+m+": read the newest "+fmt.Sprint(MaxReads)+" of its keys; a step older than those is not shown")
}
steps := make([]delivery.Step, len(keys))
isRunning := make([]bool, len(keys))
_ = parallel(len(keys), func(i int) error {
raw, err := sr.Get(keys[i])
if err != nil || len(raw) == 0 { if err != nil || len(raw) == 0 {
return nil
}
step, err := delivery.DecodeStep(raw)
steps[i], isRunning[i] = step, err == nil && step.State == delivery.StepRunning
return nil
})
for i := range keys {
if !isRunning[i] {
continue continue
} }
step := steps[i] step, err := delivery.DecodeStep(raw)
if err != nil || step.State != delivery.StepRunning {
continue
}
rs := RunningStep{Module: m, Node: step.Node, ID: step.ID, Tool: step.Tool} rs := RunningStep{Module: m, Node: step.Node, ID: step.ID, Tool: step.Tool}
if !step.Started.IsZero() { if !step.Started.IsZero() {
rs.Since = step.Started.UTC().Format("2006-01-02T15:04:05Z") rs.Since = step.Started.UTC().Format("2006-01-02T15:04:05Z")
@@ -452,8 +330,6 @@ func (r Reader) Status() (Status, error) {
} }
st.Unjudged = []string{} st.Unjudged = []string{}
st.Waiting = map[string][]string{KindNew: {}, KindNodes: {}, KindTrunk: {}} st.Waiting = map[string][]string{KindNew: {}, KindNodes: {}, KindTrunk: {}}
st.HeldByTurn = map[string]string{}
turns := r.turns(&st)
var heads map[string]string var heads map[string]string
if r.Trunks == nil { if r.Trunks == nil {
st.Notes = append(st.Notes, "the trunk heads are not readable here, so no check is named as waiting for a trunk move") st.Notes = append(st.Notes, "the trunk heads are not readable here, so no check is named as waiting for a trunk move")
@@ -468,47 +344,26 @@ func (r Reader) Status() (Status, error) {
return st, err return st, err
} }
sort.Strings(keys) sort.Strings(keys)
total := len(keys) for _, k := range keys {
keys, cut := capKeys(keys, "") raw, err := r.CandidateState.Get(k)
if cut != "" {
st.Notes = append(st.Notes, fmt.Sprintf("read the first %d candidates of %d", MaxReads, total))
}
type judged struct {
c delivery.Candidate
raw json.RawMessage
ok bool
}
rows := make([]judged, len(keys))
_ = parallel(len(keys), func(i int) error {
raw, err := r.CandidateState.Get(keys[i])
if err != nil || len(raw) == 0 { if err != nil || len(raw) == 0 {
return nil continue
} }
c, err := delivery.DecodeCandidate(raw) c, err := delivery.DecodeCandidate(raw)
if err != nil { if err != nil {
return nil continue
} }
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash) vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
if err != nil { if err != nil {
return nil
}
rows[i].c = c
if vraw, err := r.VerdictState.Get(vkey); err == nil {
rows[i].raw, rows[i].ok = vraw, true
}
return nil
})
for _, row := range rows {
if !row.ok {
continue continue
} }
c, raw := row.c, row.raw raw, err = r.VerdictState.Get(vkey)
if err != nil {
continue
}
if len(raw) == 0 { if len(raw) == 0 {
st.Unjudged = append(st.Unjudged, c.Group) st.Unjudged = append(st.Unjudged, c.Group)
st.Waiting[KindNew] = append(st.Waiting[KindNew], c.Group) st.Waiting[KindNew] = append(st.Waiting[KindNew], c.Group)
if why := heldBy(turns, c, heads); why != "" {
st.HeldByTurn[c.Group] = why
}
continue continue
} }
v, err := delivery.DecodeVerdict(raw) v, err := delivery.DecodeVerdict(raw)
@@ -517,16 +372,12 @@ func (r Reader) Status() (Status, error) {
case IsNodesFailure(v): case IsNodesFailure(v):
st.Waiting[KindNodes] = append(st.Waiting[KindNodes], c.Group) st.Waiting[KindNodes] = append(st.Waiting[KindNodes], c.Group)
case heads != nil: case heads != nil:
if cs := CaseOf(c, heads); cs != nil && (!sameTrunk(v.Trunk, cs.Trunk) || besideMoved(v.Beside, besideAt(heads, r.Runs))) { if cs := CaseOf(c, heads); cs != nil && !sameTrunk(v.Trunk, cs.Trunk) {
st.Waiting[KindTrunk] = append(st.Waiting[KindTrunk], c.Group) st.Waiting[KindTrunk] = append(st.Waiting[KindTrunk], c.Group)
if why := heldBy(turns, c, heads); why != "" {
st.HeldByTurn[c.Group] = why
}
} }
} }
} }
} }
st.DeadLetters = r.deadLetters()
st.Notes = append(st.Notes, "nothing holds the checker and it holds no place: running_steps are the build modules' keys that can still be running (started within the step bound, their build module running on the node); a candidate without a verdict has its check ask in a build seat's stream, waiting for a node with a free slot, whose backlog the delivery view shows (ADR 0525 §4, §9)") st.Notes = append(st.Notes, "nothing holds the checker and it holds no place: running_steps are the build modules' keys that can still be running (started within the step bound, their build module running on the node); a candidate without a verdict has its check ask in a build seat's stream, waiting for a node with a free slot, whose backlog the delivery view shows (ADR 0525 §4, §9)")
return st, nil return st, nil
} }
@@ -3,9 +3,7 @@ package check
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"fmt"
"strings" "strings"
"sync/atomic"
"testing" "testing"
"time" "time"
@@ -53,7 +51,7 @@ func readerOf(t *testing.T) (Reader, string) {
func TestTheVerdictToolsAnswerTheStoredVerdictWithTheFullReportOfEachPullRequest(t *testing.T) { // N9/AC1, 1686 func TestTheVerdictToolsAnswerTheStoredVerdictWithTheFullReportOfEachPullRequest(t *testing.T) { // N9/AC1, 1686
rd, key := readerOf(t) rd, key := readerOf(t)
vs, _, err := rd.Verdicts("") vs, err := rd.Verdicts("")
if err != nil || len(vs) != 1 || vs[0].Key != key { if err != nil || len(vs) != 1 || vs[0].Key != key {
t.Fatalf("verdicts = %+v, %v", vs, err) t.Fatalf("verdicts = %+v, %v", vs, err)
} }
@@ -63,7 +61,7 @@ func TestTheVerdictToolsAnswerTheStoredVerdictWithTheFullReportOfEachPullRequest
if got := vs[0].Reports["novox/mesh-delivery#7"]; !strings.HasPrefix(got, "pass 111111111111 on aaaaaaaaaaaa") || !strings.Contains(got, "composition not yet run") { if got := vs[0].Reports["novox/mesh-delivery#7"]; !strings.HasPrefix(got, "pass 111111111111 on aaaaaaaaaaaa") || !strings.Contains(got, "composition not yet run") {
t.Errorf("reports = %v", vs[0].Reports) t.Errorf("reports = %v", vs[0].Reports)
} }
if only, _, _ := rd.Verdicts("feat/other"); len(only) != 0 { if only, _ := rd.Verdicts("feat/other"); len(only) != 0 {
t.Errorf("another group's verdicts = %+v", only) t.Errorf("another group's verdicts = %+v", only)
} }
} }
@@ -87,11 +85,11 @@ func TestAnHqPullRequestIsAnsweredAsAnyOther(t *testing.T) {
func TestTheKeptToolAnswersTheKeysAsStoredByLevelAndByDigest(t *testing.T) { // N9/AC1 func TestTheKeptToolAnswersTheKeysAsStoredByLevelAndByDigest(t *testing.T) { // N9/AC1
rd, _ := readerOf(t) rd, _ := readerOf(t)
all, _, err := rd.Kept("", "") all, err := rd.Kept("", "")
if err != nil || len(all) != 1 { if err != nil || len(all) != 1 {
t.Fatalf("kept = %v, %v", all, err) t.Fatalf("kept = %v, %v", all, err)
} }
one, _, err := rd.Kept("unit", digA) one, err := rd.Kept("unit", digA)
if err != nil || len(one) != 1 { if err != nil || len(one) != 1 {
t.Fatalf("one kept = %v, %v", one, err) t.Fatalf("one kept = %v, %v", one, err)
} }
@@ -100,10 +98,10 @@ func TestTheKeptToolAnswersTheKeysAsStoredByLevelAndByDigest(t *testing.T) { //
t.Errorf("kept value = %+v, %v", k, err) t.Errorf("kept value = %+v, %v", k, err)
} }
} }
if none, _, _ := rd.Kept("integration", ""); len(none) != 0 { if none, _ := rd.Kept("integration", ""); len(none) != 0 {
t.Errorf("integration = %v", none) t.Errorf("integration = %v", none)
} }
if _, _, err := rd.Kept("unit", digB); err == nil { if _, err := rd.Kept("unit", digB); err == nil {
t.Error("a digest nothing kept was answered") t.Error("a digest nothing kept was answered")
} }
} }
@@ -117,17 +115,17 @@ func TestThePreviewsToolShowsTheStoredPreviewOfAPullRequest(t *testing.T) {
} }
key, _ := delivery.PreviewKey(repoA, head1, head2) key, _ := delivery.PreviewKey(repoA, head1, head2)
rd := Reader{PreviewState: fakeState{key: data, "junk": []byte("{}")}.read()} rd := Reader{PreviewState: fakeState{key: data, "junk": []byte("{}")}.read()}
got, bad, _, err := rd.Previews(repoA, head1[:8]) got, bad, err := rd.Previews(repoA, head1[:8])
if err != nil || len(got) != 1 || got[0].Key != key || got[0].Head != head1 || got[0].Trunk != head2 || string(got[0].Preview) != string(data) { if err != nil || len(got) != 1 || got[0].Key != key || got[0].Head != head1 || got[0].Trunk != head2 || string(got[0].Preview) != string(data) {
t.Fatalf("previews = %+v, %v", got, err) t.Fatalf("previews = %+v, %v", got, err)
} }
if len(bad) != 1 || bad[0] != "junk" { if len(bad) != 1 || bad[0] != "junk" {
t.Errorf("unreadable = %v", bad) t.Errorf("unreadable = %v", bad)
} }
if got, _, _, _ := rd.Previews("novox/other", ""); len(got) != 0 { if got, _, _ := rd.Previews("novox/other", ""); len(got) != 0 {
t.Errorf("another repository's = %+v", got) t.Errorf("another repository's = %+v", got)
} }
if _, _, _, err := (Reader{}).Previews("", ""); err == nil { if _, _, err := (Reader{}).Previews("", ""); err == nil {
t.Error("answered with no state to read") t.Error("answered with no state to read")
} }
} }
@@ -144,10 +142,7 @@ func TestTheStatusToolReadsRunningSteps(t *testing.T) {
CandidateState: fakeState{ckey: cand}.read(), CandidateState: fakeState{ckey: cand}.read(),
VerdictState: fakeState{}.read(), VerdictState: fakeState{}.read(),
HaveState: fakeState{"shanks.mesh-build-go": haveOf(t, delivery.PhaseRunning)}.read(), HaveState: fakeState{"shanks.mesh-build-go": haveOf(t, delivery.PhaseRunning)}.read(),
HearingState: fakeState{"dead-letters": []byte(`{"heard":false,"streams":["SEAT_NODE_BUILD_GO"],` + Now: func() time.Time { return time.Date(2026, 10, 22, 15, 10, 0, 0, time.UTC) },
`"why":"DEAD_LETTERS is not heard, so an ask a build seat gives up on is waited for until its bound: SEAT_NODE_BUILD_GO: permissions violation",` +
`"at":"2026-10-26T13:00:00Z"}`)}.read(),
Now: func() time.Time { return time.Date(2026, 10, 22, 15, 10, 0, 0, time.UTC) },
} }
st, err := rd.Status() st, err := rd.Status()
if err != nil { if err != nil {
@@ -159,13 +154,9 @@ func TestTheStatusToolReadsRunningSteps(t *testing.T) {
if len(st.Unjudged) != 1 || st.Unjudged[0] != "feat/x" { if len(st.Unjudged) != 1 || st.Unjudged[0] != "feat/x" {
t.Errorf("unjudged = %v", st.Unjudged) t.Errorf("unjudged = %v", st.Unjudged)
} }
// Issue 734, review round 1: a checker that does not hear DEAD_LETTERS says so in its status, with why.
if st.DeadLetters.Heard || !strings.Contains(st.DeadLetters.Why, "permissions violation") {
t.Errorf("dead letters = %+v, want not heard, saying why", st.DeadLetters)
}
// What cannot be read is said, not guessed. // What cannot be read is said, not guessed.
st, err = Reader{}.Status() st, err = Reader{}.Status()
if err != nil || len(st.Notes) == 0 || st.DeadLetters.Heard { if err != nil || len(st.Notes) == 0 {
t.Errorf("empty status = %+v, %v", st, err) t.Errorf("empty status = %+v, %v", st, err)
} }
} }
@@ -241,134 +232,4 @@ func TestTheStatusCountsAnHQOnlyGroupAsUnjudged(t *testing.T) {
if len(st.Unjudged) != 1 || st.Unjudged[0] != "pb-x" { if len(st.Unjudged) != 1 || st.Unjudged[0] != "pb-x" {
t.Errorf("unjudged = %v, want pb-x", st.Unjudged) t.Errorf("unjudged = %v, want pb-x", st.Unjudged)
} }
// ADR 0548 §3: a candidate whose check an approved merge's turn holds off is named with the turn.
held := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
hkey, _ := delivery.CandidatesKey("feat/x")
st, err = Reader{CandidateState: fakeState{hkey: held}.read(), VerdictState: fakeState{}.read(),
Trunks: func() (map[string]string, error) { return map[string]string{repoA: trunkA}, nil },
TurnState: fakeState{turnKey(t, repoA): turnOf(t, repoA, "fix/other", "0123456789ab")}.read()}.Status()
if err != nil {
t.Fatal(err)
}
if why := st.HeldByTurn["feat/x"]; !strings.Contains(why, repoA) || !strings.Contains(why, "fix/other") {
t.Errorf("held_by_turn = %v, want feat/x held by the merge of fix/other on %s", st.HeldByTurn, repoA)
}
}
// slowBucket wraps a state with a Get that takes delay and counts the values read, the lists and the most Gets in flight.
type slowBucket struct {
vals fakeState
reads, lists atomic.Int64
inFlight, maxFlight atomic.Int64
}
func (b *slowBucket) get(key string) (json.RawMessage, error) {
b.reads.Add(1)
now := b.inFlight.Add(1)
for m := b.maxFlight.Load(); now > m && !b.maxFlight.CompareAndSwap(m, now); m = b.maxFlight.Load() {
}
time.Sleep(time.Millisecond)
b.inFlight.Add(-1)
return b.vals.read().Get(key)
}
func (b *slowBucket) read() StateRead {
return StateRead{Keys: func() ([]string, error) { b.lists.Add(1); return b.vals.read().Keys() }, Get: b.get}
}
func (b *slowBucket) Keys() ([]string, error) { return b.read().Keys() }
func (b *slowBucket) Get(key string) (json.RawMessage, error) { return b.get(key) }
func (b *slowBucket) Create(string, []byte) error { return nil }
// 1827: a read tool lists the keys of a bucket of thousands and reads only the values its answer needs, sixteen at a time, at
// most MaxReads, saying when it cut (on the live shape of 2026-10-26: kept 7990 keys, steps 10034, tries 1000s).
func TestReadToolsReadOnlyWhatTheyNeedInParallelAndSayWhenCut(t *testing.T) { // 1827
kept := fakeState{}
for i := 0; i < 1800; i++ {
level := map[bool]string{true: "unit", false: "integration"}[i%9 != 0]
k, _ := delivery.KeptKey(level, fmt.Sprintf("%064x", i))
kept[k] = []byte(`{"v":1}`)
}
previews := fakeState{}
for i := 0; i < 1500; i++ {
repo := map[bool]string{true: repoA, false: repoSDK}[i < 300]
k, _ := delivery.PreviewKey(repo, fmt.Sprintf("%08x%032x", i, 0), trunkA)
previews[k] = []byte(`{"v":1}`)
}
for _, row := range []struct {
name string
call func(*slowBucket) (rows int, cut string, err error)
rows int
reads, lists int64
cut bool
}{
{"kept of one key", func(b *slowBucket) (int, string, error) {
m, c, err := Reader{KeptState: b.read()}.Kept("unit", fmt.Sprintf("%064x", 1))
return len(m), c, err
}, 1, 1, 0, false},
{"kept of a level", func(b *slowBucket) (int, string, error) {
m, c, err := Reader{KeptState: b.read()}.Kept("integration", "")
return len(m), c, err
}, 200, 200, 1, false},
{"kept of everything", func(b *slowBucket) (int, string, error) {
m, c, err := Reader{KeptState: b.read()}.Kept("", "")
return len(m), c, err
}, MaxReads, MaxReads, 1, true},
{"previews of a repository", func(b *slowBucket) (int, string, error) {
p, _, c, err := Reader{PreviewState: b.read()}.Previews(repoSDK, "")
return len(p), c, err
}, 1200, 1000, 1, true},
{"previews of a repository and a head", func(b *slowBucket) (int, string, error) {
p, _, c, err := Reader{PreviewState: b.read()}.Previews(repoA, fmt.Sprintf("%08x", 7))
return len(p), c, err
}, 1, 1, 1, false},
} {
b := &slowBucket{}
b.vals = kept
if strings.HasPrefix(row.name, "previews") {
b.vals = previews
}
start := time.Now()
rows, cut, err := row.call(b)
if err != nil {
t.Fatalf("%s: %v", row.name, err)
}
if rows != min(row.rows, MaxReads) || b.reads.Load() != row.reads || b.lists.Load() != row.lists || (cut != "") != row.cut {
t.Errorf("%s: %d rows, %d reads, %d lists, cut %q; want %d, %d, %d, cut %v", row.name, rows, b.reads.Load(), b.lists.Load(), cut, min(row.rows, MaxReads), row.reads, row.lists, row.cut)
}
if row.reads > 20 && b.maxFlight.Load() < 2 {
t.Errorf("%s: values were read one at a time", row.name)
}
t.Logf("%s: %d values in %v (one after the other: %v)", row.name, b.reads.Load(), time.Since(start), time.Duration(b.reads.Load())*time.Millisecond)
}
// The status reads the newest steps of a bucket of 1500 first, at most MaxReads, and finds the one that runs.
now := time.Date(2026, 10, 26, 7, 0, 0, 0, time.UTC)
steps := fakeState{}
for i := 0; i < 1500; i++ {
id := fmt.Sprintf("mvo5%04dabcdef", i)
step := delivery.Step{ID: id, Tool: "t", Node: "g14", State: delivery.StepPassed}
if i == 1499 {
step.State, step.Started = delivery.StepRunning, now.Add(-time.Minute)
}
raw, _ := delivery.EncodeStep(step)
steps["g14."+id] = raw
}
sb := &slowBucket{vals: steps}
st, err := Reader{StepStates: map[string]StateRead{"mesh-build-go": sb.read()}, HaveState: fakeState{"g14.mesh-build-go": haveOf(t, delivery.PhaseRunning)}.read(),
Now: func() time.Time { return now }}.Status()
if err != nil || sb.reads.Load() != MaxReads || len(st.Running) != 1 || sb.maxFlight.Load() < 2 {
t.Errorf("status: %d reads (want %d), running %+v, in flight %d, %v", sb.reads.Load(), MaxReads, st.Running, sb.maxFlight.Load(), err)
}
// A new try reads the newest TryScan of 1500 ended tries, not all, and still finds the same head asked twice.
tries := fakeState{}
for i := 0; i < 1500; i++ {
raw, _ := encodeTry(Try{ID: fmt.Sprintf("try-%d", 1700000000000000000+i), Repository: repoA, Head: head1, Trunk: trunkA, State: TryPass})
tries[fmt.Sprintf("try-%d", 1700000000000000000+i)] = raw
}
tb := &slowBucket{vals: tries}
if _, err := AskTry(tb, tryTrunks, TryAsk{Repository: repoA, Head: head2, Trunk: trunkA}, now); err != nil || tb.reads.Load() != TryScan {
t.Errorf("AskTry read %d of 1500 tries (want %d), %v", tb.reads.Load(), TryScan, err)
}
} }
@@ -3,8 +3,6 @@ package check
import ( import (
"encoding/json" "encoding/json"
"strings" "strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
) )
// The shapes of what the build modules' steps answer, as their code writes them (mesh-catalog modules/mesh-build-go // The shapes of what the build modules' steps answer, as their code writes them (mesh-catalog modules/mesh-build-go
@@ -54,8 +52,6 @@ type GoPackage struct {
Elapsed float64 `json:"elapsed"` Elapsed float64 `json:"elapsed"`
Tests int `json:"tests"` Tests int `json:"tests"`
NoTests bool `json:"no_tests,omitempty"` NoTests bool `json:"no_tests,omitempty"`
// Skipped is how many of the package's tests skipped; an older build module names none.
Skipped int `json:"skipped,omitempty"`
} }
// Failure is a test that failed, or a package that failed with no test to name. // Failure is a test that failed, or a package that failed with no test to name.
@@ -111,6 +107,4 @@ type Digested struct {
WithoutTests []string `json:"without_tests,omitempty"` WithoutTests []string `json:"without_tests,omitempty"`
// JudgedBy is the program that ran the digest, as Tested.JudgedBy says. // JudgedBy is the program that ran the digest, as Tested.JudgedBy says.
JudgedBy string `json:"judged_by,omitempty"` JudgedBy string `json:"judged_by,omitempty"`
// Needs are the packages whose tests need something of the node at this level (issue 890).
Needs []delivery.PackageNeeds `json:"needs,omitempty"`
} }
@@ -13,7 +13,6 @@ import (
"time" "time"
"git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
) )
// A try is a pushed head judged by the checker as a pull request's head is, before its pull request is opened (ADR 0501 // A try is a pushed head judged by the checker as a pull request's head is, before its pull request is opened (ADR 0501
@@ -25,14 +24,11 @@ import (
const ( const (
TryTool = "checker_try" TryTool = "checker_try"
TryReadTool = "checker_try_read" TryReadTool = "checker_try_read"
TryEndTool = "checker_try_end"
// TriesState is the checker's state of tries: the bucket mesh-delivery-checker_tries, keys try-<unix nanoseconds>. // TriesState is the checker's state of tries: the bucket mesh-delivery-checker_tries, keys try-<unix nanoseconds>.
TriesState = "tries" TriesState = "tries"
TriesBucket = delivery.CheckerModule + "_" + TriesState TriesBucket = delivery.CheckerModule + "_" + TriesState
// MaxTries is how many tries may wait or run at once, mesh-wide: each holds a build node for minutes. // MaxTries is how many tries may wait or run at once, mesh-wide: each holds a build node for minutes.
MaxTries = 4 MaxTries = 4
// TryScan is how many of the newest tries a new ask reads to find the ones that wait or run (issue 1827).
TryScan = 64
// TryKeep is how long an ended try's key is kept before the checker removes it. // TryKeep is how long an ended try's key is kept before the checker removes it.
TryKeep = 7 * 24 * time.Hour TryKeep = 7 * 24 * time.Hour
) )
@@ -71,18 +67,13 @@ type Try struct {
// Said is the try's report in words: what failed first, the tests run, kept and skipped per level, the time of each part. // Said is the try's report in words: what failed first, the tests run, kept and skipped per level, the time of each part.
Said string `json:"said,omitempty"` Said string `json:"said,omitempty"`
// Took is the time from its start to its end, as Go writes a duration. // Took is the time from its start to its end, as Go writes a duration.
Took string `json:"took,omitempty"` Took string `json:"took,omitempty"`
// Failed is every test that failed, each with its first line, at most delivery.MaxFailures and then how many more
// (issue 1921); or what else failed first (a conflict, a refused step).
Failed []string `json:"failed,omitempty"` Failed []string `json:"failed,omitempty"`
Levels map[string]delivery.LevelResult `json:"levels,omitempty"` Levels map[string]delivery.LevelResult `json:"levels,omitempty"`
Machines []string `json:"machines,omitempty"` Machines []string `json:"machines,omitempty"`
// Asks are the ids of the asks the try put on the build seats' streams, each ending on the step key <node>.<id> of its Asked time.Time `json:"asked"`
// build module: the delivery view counts those steps as a try's, not a candidate's check (build_usage). Started time.Time `json:"started,omitzero"`
Asks []string `json:"asks,omitempty"` Ended time.Time `json:"ended,omitzero"`
Asked time.Time `json:"asked"`
Started time.Time `json:"started,omitzero"`
Ended time.Time `json:"ended,omitzero"`
} }
// Over says whether the try came to its end. // Over says whether the try came to its end.
@@ -164,33 +155,15 @@ func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (
if err != nil { if err != nil {
return "", fmt.Errorf("the tries cannot be listed: %w", err) return "", fmt.Errorf("the tries cannot be listed: %w", err)
} }
// An ended try's key stays TryKeep, so the bucket holds thousands; one that waits or runs was asked lately. Only the sort.Strings(keys)
// newest TryScan keys are read, in parallel (a try id is the time in nanoseconds, so the newest sorts last). var open []string
sort.Slice(keys, func(i, j int) bool { for _, k := range keys {
if len(keys[i]) != len(keys[j]) { raw, err := store.Get(k)
return len(keys[i]) > len(keys[j])
}
return keys[i] > keys[j]
})
if len(keys) > TryScan {
keys = keys[:TryScan]
}
tries := make([]Try, len(keys))
live := make([]bool, len(keys))
if err := parallel(len(keys), func(i int) error {
raw, err := store.Get(keys[i])
if err != nil || len(raw) == 0 { if err != nil || len(raw) == 0 {
return nil continue
} }
cur, err := DecodeTry(raw) cur, err := DecodeTry(raw)
tries[i], live[i] = cur, err == nil && !cur.Over() if err != nil || cur.Over() {
return nil
}); err != nil {
return "", err
}
var open []string
for i, cur := range tries {
if !live[i] {
continue continue
} }
if cur.Repository == t.Repository && cur.Head == t.Head && cur.Trunk == t.Trunk && cur.Tests == t.Tests && cur.Level == t.Level { if cur.Repository == t.Repository && cur.Head == t.Head && cur.Trunk == t.Trunk && cur.Tests == t.Tests && cur.Level == t.Level {
@@ -227,52 +200,6 @@ func ReadTry(store TryStore, id string) (json.RawMessage, error) {
return raw, nil return raw, nil
} }
// TryEnder is the tries state as the end tool sees it: one key with its revision, and a write at that revision.
type TryEnder interface {
Entry(key string) (json.RawMessage, uint64, error)
Update(key string, value []byte, revision uint64) error
}
// EndTry ends a try that waits or runs as error, saying why and that it was ended by hand (issue 734): its place is
// free at once, and the next checker_try of its head asks anew. A try that ended already is refused, naming its end.
func EndTry(store TryEnder, id, why string, now time.Time) (Try, error) {
if !tryKeyOK.MatchString(id) {
return Try{}, fmt.Errorf("%q is not a try's id, try-<digits>", id)
}
why = strings.TrimSpace(why)
if why == "" {
return Try{}, errors.New("say why the try is ended: the why is kept in what it says")
}
raw, rev, err := store.Entry(id)
if err != nil {
return Try{}, err
}
if len(raw) == 0 {
return Try{}, fmt.Errorf("no try %s: it was never asked, or it ended more than %s ago and was removed", id, TryKeep)
}
t, err := DecodeTry(raw)
if err != nil {
return Try{}, err
}
if t.Over() {
return Try{}, fmt.Errorf("%s ended already, %s at %s", id, t.State, t.Ended.Format(time.RFC3339))
}
t.State, t.Ended = TryError, now.UTC()
if !t.Started.IsZero() {
t.Took = t.Ended.Sub(t.Started).Round(time.Second).String()
}
t.Said = strings.TrimSpace("ended by hand (" + TryEndTool + "): " + why + " | " + t.Said)
t.Said = strings.TrimSuffix(t.Said, " |")
data, err := encodeTry(t)
if err != nil {
return Try{}, err
}
if err := store.Update(id, data, rev); err != nil {
return Try{}, fmt.Errorf("%s changed while it was ended, read it and end it again: %w", id, err)
}
return t, nil
}
// TryOptions narrow what a try runs; a pull request's check has none. // TryOptions narrow what a try runs; a pull request's check has none.
type TryOptions struct { type TryOptions struct {
Level string Level string
@@ -285,14 +212,12 @@ type Tryer struct {
Tries KV Tries KV
Judge Judge Judge Judge
Bound time.Duration Bound time.Duration
// NotTaken ends a try whose ask no build node took within it as error (issue 734); none when 0. Now func() time.Time
NotTaken time.Duration Logf func(format string, args ...any)
Now func() time.Time
Logf func(format string, args ...any)
mu sync.Mutex mu sync.Mutex
wg sync.WaitGroup wg sync.WaitGroup
inflight map[string]context.CancelFunc inflight map[string]bool
seen map[string]bool // looked at since this process started: a try left running is taken over the first time seen map[string]bool // looked at since this process started: a try left running is taken over the first time
} }
@@ -320,31 +245,25 @@ func (tr *Tryer) Put(ctx context.Context, key string, value []byte) {
tr.logf("tries.%s holds what is not a try, left: %v", key, err) tr.logf("tries.%s holds what is not a try, left: %v", key, err)
return return
} }
if t.Over() {
return
}
tr.mu.Lock() tr.mu.Lock()
if tr.inflight == nil { if tr.inflight == nil {
tr.inflight, tr.seen = map[string]context.CancelFunc{}, map[string]bool{} tr.inflight, tr.seen = map[string]bool{}, map[string]bool{}
} }
if t.Over() { if tr.inflight[key] || (t.State == TryRunning && tr.seen[key]) {
// Ended while it ran (checker_try_end): its judging stops, and its asks are no longer waited for.
if stop := tr.inflight[key]; stop != nil {
stop()
}
tr.mu.Unlock() tr.mu.Unlock()
return return
} }
if tr.inflight[key] != nil || (t.State == TryRunning && tr.seen[key]) {
tr.mu.Unlock()
return
}
run, stop := context.WithCancel(ctx)
tr.seen[key] = true tr.seen[key] = true
tr.inflight[key] = stop tr.inflight[key] = true
tr.mu.Unlock() tr.mu.Unlock()
tr.wg.Add(1) tr.wg.Add(1)
go func() { go func() {
defer tr.wg.Done() defer tr.wg.Done()
defer func() { tr.mu.Lock(); delete(tr.inflight, key); tr.mu.Unlock(); stop() }() defer func() { tr.mu.Lock(); delete(tr.inflight, key); tr.mu.Unlock() }()
tr.run(run, key) tr.run(ctx, key)
}() }()
} }
@@ -401,34 +320,28 @@ func (tr *Tryer) run(ctx context.Context, key string) {
Queued: t.Asked, Queued: t.Asked,
Progress: &Progress{}, Progress: &Progress{},
Try: &TryOptions{Level: t.Level, Run: t.Tests}} Try: &TryOptions{Level: t.Level, Run: t.Tests}}
judging := ctx out, judged := judgeWithin(ctx, tr.Judge, cs, tr.Bound)
if tr.NotTaken > 0 {
judging = buildrun.NotTakenBound(ctx, tr.NotTaken)
}
out, judged := judgeWithin(judging, tr.Judge, cs, tr.Bound)
if ctx.Err() != nil { if ctx.Err() != nil {
return // left running: the checker takes it over when it starts; or ended by hand, which wrote its end return // left running: the checker takes it over when it starts
} }
v := out.Verdict v := out.Verdict
t.Ended = tr.now().UTC() t.Ended = tr.now().UTC()
t.Took = t.Ended.Sub(t.Started).Round(time.Second).String() t.Took = t.Ended.Sub(t.Started).Round(time.Second).String()
t.Levels, t.Machines, t.Asks = v.Levels, v.Machines, cs.Asks() t.Levels, t.Machines = v.Levels, v.Machines
t.Said = Report(v, cs, cs.Pulls[0], out.Builds, out.Slowest, out.Judged) t.Said = Report(v, cs, cs.Pulls[0], out.Builds, out.Slowest, out.Judged)
for _, n := range cs.Notes() { for _, n := range cs.Notes() {
t.Said += " | " + n t.Said += " | " + n
} }
switch { switch {
case !judged || IsNodesFailure(v) || IsLostAsk(v) || IsNotTaken(v): case !judged || IsNodesFailure(v):
// Not the change's: its ask was given up or never taken (issue 734), or a part of the mesh was not heard.
t.State = TryError t.State = TryError
if IsLostAsk(v) || IsNotTaken(v) {
t.Failed = []string{v.Failed.Words}
}
case v.State == delivery.VerdictPass: case v.State == delivery.VerdictPass:
t.State = TryPass t.State = TryPass
default: default:
t.State = TryFail t.State = TryFail
t.Failed = failedLines(v) if f := firstFailed(v); f != "" {
t.Failed = []string{f}
}
} }
if t.Tests != "" { if t.Tests != "" {
t.Said += " | only the tests " + t.Tests + " (Go); nothing kept" t.Said += " | only the tests " + t.Tests + " (Go); nothing kept"
@@ -97,12 +97,6 @@ func TestATryIsJudgedAsAPullRequestsHeadAndOwesTheForgeNothing(t *testing.T) { /
if got.State != TryPass || got.Took == "" || got.Head != head1 || got.Trunk != trunkA || got.Said == "" || got.Levels[delivery.LevelUnit].Run != 2 { if got.State != TryPass || got.Took == "" || got.Head != head1 || got.Trunk != trunkA || got.Said == "" || got.Levels[delivery.LevelUnit].Run != 2 {
t.Fatalf("try = %+v", got) t.Fatalf("try = %+v", got)
} }
// The try names the asks it put, so the delivery view counts their steps as a try's (build_usage).
if len(got.Asks) != 1 {
t.Errorf("asks = %v, want the one check ask the try put", got.Asks)
} else if _, err := delivery.AskStepKey("shanks", got.Asks[0]); err != nil {
t.Errorf("the try names %q, which is no ask id: %v", got.Asks[0], err)
}
fetch := b.steps.tool("fetch") fetch := b.steps.tool("fetch")
if len(fetch) != 1 || fetch[0].args["commit"] != head1 || fetch[0].args["base"] != trunkA || fetch[0].args["ref"] != "" { if len(fetch) != 1 || fetch[0].args["commit"] != head1 || fetch[0].args["base"] != trunkA || fetch[0].args["ref"] != "" {
t.Errorf("fetch = %+v, want the head merged onto the trunk, with no pull request's ref", fetch) t.Errorf("fetch = %+v, want the head merged onto the trunk, with no pull request's ref", fetch)
@@ -1,78 +0,0 @@
package check
import (
"sort"
"strings"
"sync"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// Turns holds the gate's merge turns (novox/hq ADR 0548 §3), by repository, as a watch of the gate's turns bucket puts them
// in: the checker claims no check of a repository a turn names, except the check of a waiting delivery's own heads, so an
// approved merge goes before a check not yet started. A bucket not there is read as empty.
type Turns struct {
mu sync.Mutex
byRepo map[string]delivery.Turn
}
// Put takes a value of the turns bucket in, or its delete, and answers whether what the checker holds changed. A value that
// is not a turn is dropped as a delete would be: a turn the checker cannot read holds nothing.
func (t *Turns) Put(key string, value []byte, deleted bool) bool {
repository, err := delivery.ParseTurnsKey(key)
if err != nil {
return false
}
t.mu.Lock()
defer t.mu.Unlock()
if t.byRepo == nil {
t.byRepo = map[string]delivery.Turn{}
}
cur, had := t.byRepo[repository]
if !deleted {
if turn, err := delivery.DecodeTurn(value); err == nil {
t.byRepo[repository] = turn
return !had || !delivery.SameTurn(cur, turn)
}
}
delete(t.byRepo, repository)
return had
}
// Holding names the first of the repositories (by name) a turn holds off the check of the heads, and the delivery that
// waits first in it; false when none does.
func (t *Turns) Holding(repositories []string, heads string) (string, delivery.TurnWait, bool) {
if t == nil {
return "", delivery.TurnWait{}, false
}
sorted := make([]string, 0, len(repositories))
for _, r := range repositories {
sorted = append(sorted, strings.ToLower(r))
}
sort.Strings(sorted)
t.mu.Lock()
defer t.mu.Unlock()
for _, r := range sorted {
turn, ok := t.byRepo[r]
if !ok || !delivery.TurnHolds(turn, heads) {
continue
}
for _, w := range turn.Waiting {
if w.Heads != heads {
return r, w, true
}
}
}
return "", delivery.TurnWait{}, false
}
// All is every turn held now, by repository (a copy).
func (t *Turns) All() map[string]delivery.Turn {
t.mu.Lock()
defer t.mu.Unlock()
out := make(map[string]delivery.Turn, len(t.byRepo))
for r, turn := range t.byRepo {
out[r] = turn
}
return out
}
@@ -93,11 +93,11 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
t.Errorf("module = %q", m.Module) t.Errorf("module = %q", m.Module)
} }
// delivery.CheckerKeptHistory, CheckerOwedHistory, PreviewsHistory, CheckerVerdictsHistory; the names follow the SDK's buckets. // delivery.CheckerKeptHistory, CheckerOwedHistory, PreviewsHistory, CheckerVerdictsHistory; the names follow the SDK's buckets.
if want := []state{{Name: "hearing", History: 1}, {Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "tries", History: 1}, {Name: "verdicts", History: 5}}; !reflect.DeepEqual(m.State, want) { if want := []state{{Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "tries", History: 1}, {Name: "verdicts", History: 5}}; !reflect.DeepEqual(m.State, want) {
t.Errorf("state = %+v, want %+v", m.State, want) t.Errorf("state = %+v, want %+v", m.State, want)
} }
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-controller.assignments", if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-controller.assignments",
"mesh-delivery-assigner.compositions", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-gate.turns", "mesh-delivery-indexer.candidates", "mesh-delivery-assigner.compositions", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates",
"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) { "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) {
t.Errorf("reads = %v, want %v", m.Reads, want) t.Errorf("reads = %v, want %v", m.Reads, want)
} }
@@ -124,8 +124,8 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
t.Errorf("streams = %+v, want %+v", m.Streams, want) t.Errorf("streams = %+v, want %+v", m.Streams, want)
} }
// The read tools (issue 1686) are served by the tools bundle, not by the checker's process. // The read tools (issue 1686) are served by the tools bundle, not by the checker's process.
// The try's tools (ADR 0501 §5; the end by hand, issue 734) beside them: mesh-controller's try calls them. // The try's two tools (ADR 0501 §5) beside them: mesh-controller's try calls them.
if want := []string{"checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) { if want := []string{"checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_try", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
t.Errorf("tools = %v, want %v", m.Tools, want) t.Errorf("tools = %v, want %v", m.Tools, want)
} }
for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} { for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} {
@@ -10,7 +10,6 @@
{ "name": "checks", "events": ["preview.**", "owed.**"] } { "name": "checks", "events": ["preview.**", "owed.**"] }
], ],
"state": [ "state": [
{ "name": "hearing", "history": 1 },
{ "name": "kept", "history": 1 }, { "name": "kept", "history": 1 },
{ "name": "owed", "history": 5 }, { "name": "owed", "history": 5 },
{ "name": "previews", "history": 5 }, { "name": "previews", "history": 5 },
@@ -26,7 +25,6 @@
"mesh-delivery-assigner.compositions", "mesh-delivery-assigner.compositions",
"mesh-delivery-builder.builds", "mesh-delivery-builder.builds",
"mesh-delivery-gate.source", "mesh-delivery-gate.source",
"mesh-delivery-gate.turns",
"mesh-delivery-indexer.candidates", "mesh-delivery-indexer.candidates",
"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.manifests",
"mesh-delivery-indexer.modules", "mesh-delivery-indexer.modules",
@@ -43,7 +41,6 @@
"checker_pull", "checker_pull",
"checker_status", "checker_status",
"checker_try", "checker_try",
"checker_try_end",
"checker_try_read", "checker_try_read",
"checker_verdicts" "checker_verdicts"
], ],
@@ -22,8 +22,6 @@ import (
"syscall" "syscall"
"time" "time"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go" "github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream" "github.com/nats-io/nats.go/jetstream"
@@ -190,18 +188,18 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
call := callerOn(conn, cred.User) call := callerOn(conn, cred.User)
// No forge event is heard while the open pull requests are read and said (ADR 0443 §5): events and syncs take turns. // No forge event is heard while the open pull requests are read and said (ADR 0443 §5): events and syncs take turns.
var turn sync.Mutex var turn sync.Mutex
kv, err := process.EssentialBucket(ctx, js, "bridge", headsBucket, "keep the heads it wrote on the forge stream") kv, err := js.KeyValue(ctx, headsBucket)
if err != nil { if err != nil {
return err return fmt.Errorf("the bucket %s: %w", headsBucket, err)
} }
heads := kvHeads{kv} heads := kvHeads{kv}
// Everything written on FORGE, an event's and a sync's, keeps the head in the bucket. // Everything written on FORGE, an event's and a sync's, keeps the head in the bucket.
rec := &recorder{pub: js, heads: heads, logger: logger} rec := &recorder{pub: js, heads: heads, logger: logger}
said := func(ctx context.Context) error { return syncOpen(ctx, call, rec, heads, &turn, logger) } said := func(ctx context.Context) error { return syncOpen(ctx, call, rec, heads, &turn, logger) }
// What the gate asks of the forge (ADR 0437 §8): its owed acts and its merges, each performed once per revision. // What the gate asks of the forge (ADR 0437 §8): its owed acts and its merges, each performed once per revision.
performedKV, err := process.EssentialBucket(ctx, js, "bridge", performedBucket, "perform an act of the gate only once") performedKV, err := js.KeyValue(ctx, performedBucket)
if err != nil { if err != nil {
return err return fmt.Errorf("the bucket %s: %w", performedBucket, err)
} }
perf := &performer{call: call, pub: rec, done: kvRevisions{performedKV}, logger: logger} perf := &performer{call: call, pub: rec, done: kvRevisions{performedKV}, logger: logger}
watching := make(chan error, 3) watching := make(chan error, 3)
@@ -209,12 +207,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
bucket string bucket string
react func(ctx context.Context, key string, rev uint64, value []byte) react func(ctx context.Context, key string, rev uint64, value []byte)
}{{delivery.OwedBucket, perf.owed}, {delivery.MergesBucket, perf.merges}} { }{{delivery.OwedBucket, perf.owed}, {delivery.MergesBucket, perf.merges}} {
// The gate's buckets come with the gate and its grant; the bridge does not wait for them (no timer), and is started kv, err := js.KeyValue(ctx, w.bucket)
// again by its delivery when the gate arrives. A bucket refused or not answered is the same (issue 1945). if errors.Is(err, jetstream.ErrBucketNotFound) {
kv := process.OptionalBucket(ctx, js, w.bucket, "nothing of it is performed until the bridge starts again", logger.Printf) // The gate's buckets come with the gate; the bridge does not wait for them (no timer), and is started again
if kv == nil { // by its delivery when the gate arrives.
logger.Printf("the bucket %s does not exist yet: nothing of it is performed until the bridge starts again", w.bucket)
continue continue
} }
if err != nil {
return fmt.Errorf("the bucket %s: %w", w.bucket, err)
}
go func() { go func() {
watching <- watch(ctx, kv, func(key string, rev uint64, value []byte) { watching <- watch(ctx, kv, func(key string, rev uint64, value []byte) {
hctx, cancel := context.WithTimeout(ctx, syncDeadline) hctx, cancel := context.WithTimeout(ctx, syncDeadline)
@@ -229,8 +231,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
go onReconnect(ctx, again, said, logger) go onReconnect(ctx, again, said, logger)
cons, err := js.Consumer(ctx, eventsStream, cred.Node+"_"+cred.Module) cons, err := js.Consumer(ctx, eventsStream, cred.Node+"_"+cred.Module)
if err != nil { if err != nil {
// Essential: the forge's events are what the bridge carries. return fmt.Errorf("the consumer %s_%s on %s: %w", cred.Node, cred.Module, eventsStream, err)
return fmt.Errorf("the consumer %s_%s on %s cannot be read (%v): the bridge hears no forge event without it, so it stops; where the bus refused it or did not answer, this module's consumes (made by the controller with its grant) have not reached the bus yet", cred.Node, cred.Module, eventsStream, err)
} }
cc, err := cons.Consume(func(m jetstream.Msg) { cc, err := cons.Consume(func(m jetstream.Msg) {
hear(&turn, func() { hear(&turn, func() {
@@ -5,7 +5,6 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt"
"log" "log"
"slices" "slices"
"strings" "strings"
@@ -13,8 +12,6 @@ import (
"testing" "testing"
"time" "time"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go" "github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream" "github.com/nats-io/nats.go/jetstream"
@@ -69,16 +66,6 @@ func body(m map[string]any) []byte {
} }
func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit part; the bus is the integration test) func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit part; the bus is the integration test)
t.Run("1945: a refused bucket is said, or stops the start naming its grant", func(t *testing.T) {
refusal := errors.New("nats: permissions violation for publish to $JS.API.STREAM.INFO.KV_x")
var said []string
if kv := process.OptionalBucket(context.Background(), refusingBus{refusal}, "x", "x is not heard", func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }); kv != nil || len(said) != 1 || !strings.Contains(said[0], "permissions violation") {
t.Errorf("an optional bucket refused: %v, said %v", kv, said)
}
if _, err := process.EssentialBucket(context.Background(), refusingBus{refusal}, "bridge", "x", "work"); err == nil || !strings.Contains(err.Error(), "grant of x") {
t.Errorf("an essential bucket refused: %v, want its grant named", err)
}
})
var logs bytes.Buffer var logs bytes.Buffer
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{} m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{}
handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0)) handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0))
@@ -243,8 +230,3 @@ func someFiles(_, _, path, _ string) ([]byte, error) {
} }
return nil, bridge.ErrNotFound return nil, bridge.ErrNotFound
} }
// refusingBus refuses every bucket with err (issue 1945).
type refusingBus struct{ err error }
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { return nil, r.err }
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge
go 1.26.0 go 1.26.0
require ( require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9
github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0 github.com/nats-io/nats.go v1.54.0
) )
@@ -1,7 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -1,111 +0,0 @@
// Command mesh-delivery-indexer-tools serves the indexer's doing tool (novox/hq issue 1945): the runtime launches it for a
// call. indexer_take_in writes one value of the indexer's takeins state and waits for the indexer's answer there. The
// indexer is the other program: it serves no request subject (ADR 0437 §4). The read tools of the indexer's states are
// mesh-delivery-view's (indexer_module, indexer_modules, indexer_candidate, indexer_candidates).
package main
import (
"context"
"encoding/json"
"fmt"
"os"
"strings"
"sync"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"mesh-delivery-indexer/internal/index"
)
// asker writes a takeins value and waits for the indexer's answer to it.
type asker func(key string, value []byte) (index.TakeInAsk, error)
// Tools are the indexer's tools.
func Tools(ask asker, now func() time.Time) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
return []stdio.Tool{
{Name: index.TakeInTool,
Description: "Take a registered module in now: the indexer reads its manifest from the repository and commit the controller's " +
"build records register it from (at the head the gate's source names, when it names one), whatever it tried before, and " +
"writes it to its modules state as a merge would. For a module registered from outside the indexed repositories whose " +
"repository the forge could not read (a 404 before the forge's account could read it). Answers stored, from which " +
"repository, commit and path, or why not (held already, no registration, the forge's error, no manifest there). " +
"The indexer reads such a module again by itself on the next source revision and merged message; this asks now. " +
"indexer_module (mesh-delivery-view) reads the result.",
Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "the module's name, as the catalogue registers it"},
"why": map[string]any{"type": "string", "description": "why it is taken in now, in plain words"}}},
Run: func(args map[string]any) (any, error) {
key, value, err := index.NewTakeIn(str(args, "module"), str(args, "why"), stdio.CallerOf(args), now())
if err != nil {
return nil, err
}
a, err := ask(key, value)
if err != nil {
return nil, err
}
return a.Result, nil
}},
}
}
// askTakeIn writes the takeins value at key and waits, on a watch of the key, at most bound for the indexer's answer.
func askTakeIn(s stdio.KeptState, bound time.Duration) asker {
return func(key string, value []byte) (index.TakeInAsk, error) {
answered := make(chan index.TakeInAsk, 1)
var rev uint64
var mu sync.Mutex
take := func(revision uint64, v []byte) bool {
mu.Lock()
mine := rev
mu.Unlock()
if mine == 0 || revision <= mine {
return false
}
a, err := index.DecodeTakeIn(v)
if err != nil || a.Answered.IsZero() || a.Result == nil {
return false
}
select {
case answered <- a:
default:
}
return true
}
if err := s.Watch(key, func(c stdio.StateChange) error {
if c.Op == "put" {
take(c.Revision, c.Value)
}
return nil
}); err != nil {
return index.TakeInAsk{}, fmt.Errorf("watching %s.%s: %w", index.TakeInsState, key, err)
}
r, err := s.Put(key, json.RawMessage(value))
if err != nil {
return index.TakeInAsk{}, fmt.Errorf("the ask could not be written to %s: %w", index.TakeInsBucket, err)
}
mu.Lock()
rev = r
mu.Unlock()
// The watch may have heard the answer before the revision was known: read it once too.
if e, err := s.Get(key); err == nil && e != nil {
take(e.Revision, e.Value)
}
wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8)
defer cancel()
select {
case a := <-answered:
return a, nil
case <-wait.Done():
return index.TakeInAsk{}, fmt.Errorf("the indexer did not answer within %s: the take-in is still running (a forge read can take 30 s); its answer will be in %s.%s", bound, index.TakeInsBucket, key)
}
}
}
func main() {
if err := stdio.Serve("", Tools(askTakeIn(stdio.State(index.TakeInsState), 25*time.Second), time.Now)); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err)
os.Exit(1)
}
}
@@ -1,8 +1,7 @@
// Command mesh-delivery-indexer is delivery's indexer (novox/hq ADR 0437 §4, §5, issue 1238): it hears the forge // Command mesh-delivery-indexer is delivery's indexer (novox/hq ADR 0437 §4, §5, issue 1238): it hears the forge
// bridge's messages on the stream FORGE and keeps the candidate groups (candidates.<group>): the pull requests that share // bridge's messages on the stream FORGE and keeps the candidate groups (candidates.<group>): the pull requests that share
// a branch, their heads and the modules each touches. It reads the controller's builds to know which module a changed // a branch, their heads and the modules each touches. It reads the controller's builds to know which module a changed
// directory is. It serves no request subject and holds no timer: every reaction is a message or a watch (ADR 0437 §8). Its // directory is. It serves no request subject and holds no timer: every reaction is a message or a watch (ADR 0437 §8).
// tool indexer_take_in (the program mesh-delivery-indexer-tools) asks through its state takeins, which it watches.
// //
// Its build of 2026-10-24 00:37 named a program whose push never reached the artifact store, and a retry sends that // Its build of 2026-10-24 00:37 named a program whose push never reached the artifact store, and a retry sends that
// same build again; this paragraph was added only to give the indexer a new build (issue 1933). // same build again; this paragraph was added only to give the indexer a new build (issue 1933).
@@ -241,21 +240,29 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil { if err != nil {
return err return err
} }
candidates, err := process.EssentialBucket(ctx, js, "indexer", delivery.CandidatesBucket, "write a candidate's fingerprints") bucket := func(name string) (jetstream.KeyValue, error) {
kv, err := js.KeyValue(ctx, name)
if err != nil {
return nil, fmt.Errorf("the bucket %s: %w", name, err)
}
return kv, nil
}
candidates, err := bucket(delivery.CandidatesBucket)
if err != nil { if err != nil {
return err return err
} }
modules, err := process.EssentialBucket(ctx, js, "indexer", delivery.ModulesBucket, "write a module's fingerprint") modules, err := bucket(delivery.ModulesBucket)
if err != nil { if err != nil {
return err return err
} }
manifests, err := process.EssentialBucket(ctx, js, "indexer", delivery.ManifestsBucket, "keep a module's manifest") manifests, err := bucket(delivery.ManifestsBucket)
if err != nil {
return err
}
builds, err := bucket(delivery.BuildsBucket)
if err != nil { if err != nil {
return err return err
} }
// The controller's build records only name the directory of a module whose key names none (fillPaths): without them
// such a module waits for its path, every other is indexed (issue 1945).
builds := process.OptionalBucket(ctx, js, delivery.BuildsBucket, "a module whose key names no directory is not given one from the controller's build records until the indexer starts again", logger.Printf)
known := index.NewBuilds() known := index.NewBuilds()
// The controller's build records name the directory of a module whose key names none (fillPaths); no build of any // The controller's build records name the directory of a module whose key names none (fillPaths); no build of any
// module moves a fingerprint since no fingerprint takes the builder (ADR 0530 §1). // module moves a fingerprint since no fingerprint takes the builder (ADR 0530 §1).
@@ -266,15 +273,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
known.Delete(e.Key()) known.Delete(e.Key())
} }
} }
errs := make(chan error, 1) w, err := settle(ctx, builds, each)
var buildsWatch jetstream.KeyWatcher if err != nil {
if builds != nil { return err
if w, err := settle(ctx, builds, each); err != nil {
process.NotWatched(logger.Printf, delivery.BuildsBucket, err, "a module whose key names no directory is not given one until the indexer starts again")
} else {
buildsWatch = w
}
} }
errs := make(chan error, 1)
go func() { errs <- follow(ctx, w, each) }()
// The indexer asks the forge for the objects of a manifest's named paths as its own account (ADR 0499 clause 3). // The indexer asks the forge for the objects of a manifest's named paths as its own account (ADR 0499 clause 3).
account, err := process.ReadAccount(path) account, err := process.ReadAccount(path)
@@ -287,22 +291,27 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
} }
defer calls.Close() defer calls.Close()
ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Builds: known, Logf: logger.Printf, ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Builds: known, Logf: logger.Printf,
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call)} Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call)}
// The gate's source bucket (mesh-delivery-gate_source) comes with the gate (release 3e, novox/hq ADR 0450 §1): its key // The gate's source bucket (mesh-delivery-gate_source) comes with the gate (release 3e, novox/hq ADR 0450 §1): its key
// main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does // main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does
// not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate. // not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate.
sourceErrs := make(chan error, 1) sourceErrs := make(chan error, 1)
if source := process.OptionalBucket(ctx, js, delivery.SourceBucket, "no source revision is read until the indexer starts again", logger.Printf); source != nil { switch source, err := js.KeyValue(ctx, delivery.SourceBucket); {
case errors.Is(err, jetstream.ErrBucketNotFound):
logger.Printf("the bucket %s does not exist yet: no source revision is read until the indexer starts again", delivery.SourceBucket)
case err != nil:
return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err)
default:
each := func(e jetstream.KeyValueEntry) { each := func(e jetstream.KeyValueEntry) {
if opOf(e) { if opOf(e) {
applySource(ctx, ix, e.Key(), e.Revision(), e.Value(), logger) applySource(ctx, ix, e.Key(), e.Revision(), e.Value(), logger)
} }
} }
if sw, err := settle(ctx, source, each); err != nil { sw, err := settle(ctx, source, each)
process.NotWatched(logger.Printf, delivery.SourceBucket, err, "no source revision is read until the indexer starts again") if err != nil {
} else { return err
go func() { sourceErrs <- follow(ctx, sw, each) }()
} }
go func() { sourceErrs <- follow(ctx, sw, each) }()
} }
// Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's // Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's
// assignments are settled first, so the toolchains settle through the gate already knowing them. // assignments are settled first, so the toolchains settle through the gate already knowing them.
@@ -338,40 +347,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err := ix.RecomputeAtStart(ctx); err != nil { if err := ix.RecomputeAtStart(ctx); err != nil {
logger.Printf("computing the new-form fingerprints at start: %v", err) logger.Printf("computing the new-form fingerprints at start: %v", err)
} }
// A module registered in the catalogue from its own repository gets its manifest indexed when the controller's records
// name it and the bucket holds none (registered.go); the records that follow are heard below.
if err := ix.IndexRegisteredAll(ctx); err != nil {
logger.Printf("indexing the registered modules at start: %v", err)
}
if buildsWatch != nil {
go func() {
errs <- follow(ctx, buildsWatch, func(e jetstream.KeyValueEntry) {
each(e)
if !opOf(e) {
return
}
var r delivery.BuildRecord
if json.Unmarshal(e.Value(), &r) != nil || r.Module == "" {
return
}
if err := ix.IndexRegisteredModule(ctx, r.Module); err != nil {
logger.Printf("the registered module %s is not indexed: %v", r.Module, err)
}
})
}()
}
// A person takes a registered module in now with indexer_take_in, which writes the takeins state (issue 1945): watched,
// as every other input. A module the forge could not read is also read again by itself on the next source revision and
// merged message (index.RetryUnread).
// A state not on the bus yet at this start is watched once it appears (issue 1790).
process.AwaitBucket(ctx, js, conn, index.TakeInsBucket, "indexer_take_in is not answered; a module the forge could not read is still read again by itself", logger.Printf).Ready(func(kv jetstream.KeyValue) {
eachTakeIn := func(e jetstream.KeyValueEntry) { takenIn(ctx, ix, kv, e, logger.Printf) }
if tw, err := settle(ctx, kv, eachTakeIn); err != nil {
process.NotWatched(logger.Printf, index.TakeInsBucket, err, "indexer_take_in is not answered until the indexer starts again")
} else {
go func() { errs <- follow(ctx, tw, eachTakeIn) }()
}
})
// The modules the trunk no longer holds go from the bucket (issue 1900); a merge does the same from its own message. // The modules the trunk no longer holds go from the bucket (issue 1900); a merge does the same from its own message.
if err := ix.Reconcile(ctx); err != nil { if err := ix.Reconcile(ctx); err != nil {
logger.Printf("dropping the modules the trunk does not hold at start: %v", err) logger.Printf("dropping the modules the trunk does not hold at start: %v", err)
@@ -406,8 +381,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
} }
cons, err := js.Consumer(ctx, forgeStream, cred.Node+"_"+cred.Module) cons, err := js.Consumer(ctx, forgeStream, cred.Node+"_"+cred.Module)
if err != nil { if err != nil {
// Essential: the forge's merges are what the indexer indexes. return fmt.Errorf("the consumer %s_%s on %s: %w", cred.Node, cred.Module, forgeStream, err)
return fmt.Errorf("the consumer %s_%s on %s cannot be read (%v): the indexer hears no merge without it, so it stops; where the bus refused it or did not answer, this module's consumes (made by the controller with its grant) have not reached the bus yet", cred.Node, cred.Module, forgeStream, err)
} }
cc, err := cons.Consume(func(m jetstream.Msg) { handle(ctx, m, ix, logger) }, cc, err := cons.Consume(func(m jetstream.Msg) { handle(ctx, m, ix, logger) },
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) })) jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
@@ -2,16 +2,11 @@ package main
import ( import (
"context" "context"
"crypto/sha256"
"encoding/hex"
"encoding/json" "encoding/json"
"fmt" "fmt"
"path"
"strings" "strings"
"git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-indexer/internal/index"
) )
// commitTreesTool is the forge's tool for a commit's trees (mesh-catalog modules/gitea): with `paths`, by module directory // commitTreesTool is the forge's tool for a commit's trees (mesh-catalog modules/gitea): with `paths`, by module directory
@@ -98,75 +93,3 @@ func containsVia(call toolCaller) func(ctx context.Context, repository, commit,
return *a.Contains, nil return *a.Contains, nil
} }
} }
// getFileTool is the forge's tool for one file at a ref (mesh-catalog modules/gitea gitea_get_file): arguments owner,
// repo, path and ref, answer {file: {content}}; a path it does not have fails with a 404 in its words.
const getFileTool = "gitea.gitea_get_file"
// lockFiles are read beside a manifest, as the forge bridge reads them (bridge.LockFiles).
var lockFiles = []string{"go.sum", "package-lock.json"}
// readVia reads a repository at a commit as the forge bridge does for a merged message: the trees by gitea_commit_trees,
// then the module.json and lock files of the one directory dir by gitea_get_file (a repository can hold hundreds). A manifest the
// forge does not have is an error; a directory with no module.json in the trees is answered with no manifest; a lock file it does not have is left out.
func readVia(call toolCaller) func(ctx context.Context, repository, sha, dir string) (index.RepoRead, error) {
return func(_ context.Context, repository, sha, dir string) (index.RepoRead, error) {
owner, repo, ok := strings.Cut(delivery.RepositoryName(repository), "/")
if !ok {
return index.RepoRead{}, fmt.Errorf("%q is not <owner>/<name>", repository)
}
raw, err := call(commitTreesTool, map[string]any{"owner": owner, "repo": repo, "sha": sha})
if err != nil {
return index.RepoRead{}, err
}
var a struct {
Root string `json:"root"`
Trees map[string]string `json:"trees"`
}
if err := json.Unmarshal(raw, &a); err != nil {
return index.RepoRead{}, fmt.Errorf("the answer of %s is not {root, trees}: %w", commitTreesTool, err)
}
out := index.RepoRead{Root: a.Root, Trees: a.Trees, Manifests: map[string]string{}, Locks: map[string]map[string]string{}}
file := func(dir, name string) (string, error) {
p := name
if dir != "" {
p = dir + "/" + name
}
raw, err := call(getFileTool, map[string]string{"owner": owner, "repo": repo, "path": p, "ref": sha})
if err != nil {
return "", err
}
var f struct {
File *struct {
Content *string `json:"content"`
} `json:"file"`
}
if err := json.Unmarshal(raw, &f); err != nil || f.File == nil || f.File.Content == nil {
return "", fmt.Errorf("the answer of %s is not {file: {content}}: %v", getFileTool, err)
}
return *f.File.Content, nil
}
if _, held := a.Trees[dir]; held {
m, err := file(dir, "module.json")
if err != nil {
return index.RepoRead{}, fmt.Errorf("%s: %w", path.Join(dir, "module.json"), err)
}
out.Manifests[dir] = m
for _, name := range lockFiles {
c, err := file(dir, name)
if err != nil {
if strings.Contains(err.Error(), ": 404 ") {
continue
}
return index.RepoRead{}, fmt.Errorf("%s: %w", path.Join(dir, name), err)
}
sum := sha256.Sum256([]byte(c))
if out.Locks[dir] == nil {
out.Locks[dir] = map[string]string{}
}
out.Locks[dir][name] = hex.EncodeToString(sum[:])
}
}
return out, nil
}
}
@@ -3,7 +3,6 @@ package main
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"errors"
"slices" "slices"
"strings" "strings"
"testing" "testing"
@@ -51,33 +50,3 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3
t.Error("the trunk asked for paths") t.Error("the trunk asked for paths")
} }
} }
func TestReadViaReadsManifestsAndLocks(t *testing.T) {
var files []string
call := func(key string, body any) (json.RawMessage, error) {
switch key {
case commitTreesTool:
return json.RawMessage(`{"root":"r0","trees":{"":"t1","modules/other":"t2"}}`), nil
case getFileTool:
files = append(files, body.(map[string]string)["path"])
switch body.(map[string]string)["path"] {
case "module.json":
return json.RawMessage(`{"file":{"content":"{\"module\":\"snake\"}"}}`), nil
case "go.sum":
return json.RawMessage(`{"file":{"content":"x"}}`), nil
}
return nil, errors.New("Gitea API /x: 404 not found")
}
return nil, errors.New("unexpected " + key)
}
got, err := readVia(call)(context.Background(), "http://novox.internal:20000/jschoubben/snake.git", "abc", "")
if err != nil {
t.Fatal(err)
}
if len(got.Manifests) != 1 || len(files) != 3 { // the one directory asked: its manifest and two lock files, never the others
t.Fatalf("manifests = %v, files asked = %v", got.Manifests, files)
}
if got.Root != "r0" || got.Manifests[""] != `{"module":"snake"}` || len(got.Locks[""]) != 1 || got.Locks[""]["go.sum"] == "" {
t.Fatalf("read = %+v", got)
}
}
@@ -3,7 +3,7 @@ package main
import ( import (
"context" "context"
"errors" "errors"
"git.novox.be/novox/mesh-sdk/go/process" "fmt"
"log" "log"
"strings" "strings"
"sync" "sync"
@@ -55,14 +55,11 @@ func (w *previewWatch) ensure() error {
w.logger.Printf("the bucket %s does not exist yet: it is watched when the checker says it wrote a preview", delivery.PreviewsBucket) w.logger.Printf("the bucket %s does not exist yet: it is watched when the checker says it wrote a preview", delivery.PreviewsBucket)
return nil return nil
case err != nil: case err != nil:
// Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the checker's next preview. return fmt.Errorf("the bucket %s: %w", delivery.PreviewsBucket, err)
w.logger.Printf("the bucket %s cannot be read (%v): it is watched when the checker says it wrote a preview", delivery.PreviewsBucket, err)
return nil
} }
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.ix.PutPreview(e.Key(), e.Value(), !opOf(e)) }) watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.ix.PutPreview(e.Key(), e.Value(), !opOf(e)) })
if err != nil { if err != nil {
process.NotWatched(w.logger.Printf, delivery.PreviewsBucket, err, "it is watched when the checker says it wrote a preview") return err
return nil
} }
w.watched = true w.watched = true
go func() { go func() {
@@ -193,9 +193,7 @@ func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate,
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) } each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
w, err := settle(ctx, kv, each) w, err := settle(ctx, kv, each)
if err != nil { if err != nil {
// Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read. return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err)
logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
return errs, nil
} }
if gate.Settled() { if gate.Settled() {
changed(ctx) changed(ctx)
@@ -1,44 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"time"
"github.com/nats-io/nats.go/jetstream"
"mesh-delivery-indexer/internal/index"
)
// takerIn is the part of the indexer a takeins value asks through.
type takerIn interface {
TakeIn(ctx context.Context, module string) index.TakenIn
}
// takenIn answers one value of the takeins state (the tool indexer_take_in, issue 1945): a value not yet answered takes the
// module in now and is written back, by compare-and-set at its revision, with what the indexer stored or why not. A value
// answered already (this indexer's own write) or deleted is left.
func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) {
if e.Operation() != jetstream.KeyValuePut {
return
}
a, err := index.DecodeTakeIn(e.Value())
if err != nil {
logf("%s.%s holds what is not a take-in ask, left: %v", index.TakeInsState, e.Key(), err)
return
}
if !a.Answered.IsZero() {
return
}
r := ix.TakeIn(ctx, a.Module)
a.Result, a.Answered = &r, time.Now().UTC()
logf("%s: a take-in asked by %s (%s): %s", a.Module, a.By, a.Why, r.Answer)
data, err := json.Marshal(a)
if err != nil {
return
}
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
logf("%s.%s: the answer could not be written: %v", index.TakeInsState, e.Key(), err)
}
}
@@ -10,8 +10,6 @@ import (
"strings" "strings"
"sync" "sync"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream" "github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery"
@@ -75,14 +73,11 @@ func (w *toolchainWatch) ensure(module string) error {
w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module) w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module)
return nil return nil
case err != nil: case err != nil:
// Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the module's next report. return fmt.Errorf("the bucket %s: %w", name, err)
w.logger.Printf("the bucket %s cannot be read (%v): %s's toolchains are not heard until its next report", name, err, module)
return nil
} }
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.sink.put(module, e.Key(), e.Value(), !opOf(e)) }) watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.sink.put(module, e.Key(), e.Value(), !opOf(e)) })
if err != nil { if err != nil {
process.NotWatched(w.logger.Printf, name, err, fmt.Sprintf("%s's toolchains are not heard until its next report", module)) return err
return nil
} }
w.watched[module] = true w.watched[module] = true
go func() { go func() {
@@ -3,7 +3,6 @@ package main
import ( import (
"bytes" "bytes"
"context" "context"
"errors"
"log" "log"
"strings" "strings"
"sync" "sync"
@@ -22,7 +21,6 @@ type laterBuckets struct {
mu sync.Mutex mu sync.Mutex
appears map[string]jetstream.KeyValue // buckets that exist now appears map[string]jetstream.KeyValue // buckets that exist now
asked map[string]int asked map[string]int
refuse error // answered for a bucket that does not exist, when set: a bus that refuses it (issue 1945)
} }
func (l *laterBuckets) KeyValue(_ context.Context, name string) (jetstream.KeyValue, error) { func (l *laterBuckets) KeyValue(_ context.Context, name string) (jetstream.KeyValue, error) {
@@ -32,9 +30,6 @@ func (l *laterBuckets) KeyValue(_ context.Context, name string) (jetstream.KeyVa
if kv, ok := l.appears[name]; ok { if kv, ok := l.appears[name]; ok {
return kv, nil return kv, nil
} }
if l.refuse != nil {
return nil, l.refuse
}
return nil, jetstream.ErrBucketNotFound return nil, jetstream.ErrBucketNotFound
} }
@@ -105,12 +100,6 @@ func TestABuildModulesBucketThatAppearsLaterIsWatchedWhenItsEventArrives(t *test
recompute: func(context.Context) error { mu.Lock(); recomputes++; mu.Unlock(); return nil }, recompute: func(context.Context) error { mu.Lock(); recomputes++; mu.Unlock(); return nil },
} }
var logs bytes.Buffer var logs bytes.Buffer
// 1945: a bus that refuses both buckets, or does not answer, leaves them unwatched, said, and the indexer running.
refusing := &laterBuckets{appears: map[string]jetstream.KeyValue{}, asked: map[string]int{}, refuse: errors.New("context deadline exceeded")}
if err := newToolchainWatch(ctx, refusing, sink, log.New(&logs, "", 0)).start(); err != nil || !strings.Contains(logs.String(), "cannot be read") {
t.Fatalf("a refused toolchains bucket: start %v, log %q; want it said and the start going on", err, logs.String())
}
logs.Reset()
w := newToolchainWatch(ctx, buckets, sink, log.New(&logs, "", 0)) w := newToolchainWatch(ctx, buckets, sink, log.New(&logs, "", 0))
if err := w.start(); err != nil { // neither bucket exists: nothing is watched, nothing is written, nobody waits if err := w.start(); err != nil { // neither bucket exists: nothing is watched, nothing is written, nobody waits
t.Fatal(err) t.Fatal(err)
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
go 1.26.0 go 1.26.0
require ( require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142434-65ee43f07583 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468
github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0 github.com/nats-io/nats.go v1.54.0
) )
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142434-65ee43f07583 h1:ikD/ggf9/5k23sxJOX1PFcGzb4M3+QeC66AT0MyQ9Zs= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142434-65ee43f07583/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -2,8 +2,6 @@ package index
import ( import (
"encoding/json" "encoding/json"
"maps"
"slices"
"strings" "strings"
"sync" "sync"
@@ -90,102 +88,3 @@ func (b *Builds) PathOf(module, repository string) (string, bool) {
} }
return pathOf(strings.Trim(best.Path, "/")), true return pathOf(strings.Trim(best.Path, "/")), true
} }
// Registration is a module registered in the catalogue from its own repository, as the controller's newest outcome record
// of it names it: the repository, the directory of its manifest there ("" or "." for the root) and the commit it was
// built at.
type Registration struct {
Module, Repository, Path, Commit, Ref string
}
// registered is whether the record is an outcome that did not fail: what registers a module. Unlike IsTrunkBuild it
// asks for no recorded source inputs, which no build records since ADR 0530 clause 1, and for no particular branch: a
// repository's trunk is named main, master, or what its owner chose (the registered photos, de-spiegel and link2pay were
// built from nox-mesh, and every record of them names that ref).
func registered(r delivery.BuildRecord) bool {
return r.State == "" && r.Failed == ""
}
// RegistrationsOf are the places a module was registered from, by its outcome records: the newest record of each
// repository and directory (a module's name was once another repository's, as photos was a directory of the catalogue's).
// Records of a trunk's ref (main, master, trunk, none) come before those of another branch, then the newer first, so the
// first that holds the module's manifest is the one indexed.
func (b *Builds) RegistrationsOf(module string) []Registration {
type place struct{ repository, path string }
byPlace := map[place][]delivery.BuildRecord{}
b.mu.Lock()
for _, r := range b.records {
if r.Module == module && registered(r) {
k := place{delivery.RepositoryName(r.Repository), pathOf(strings.Trim(r.Path, "/"))}
byPlace[k] = append(byPlace[k], r)
}
}
b.mu.Unlock()
// Within one place the record is that of the ref a registration comes from, never a newer pull request's or try's: a
// trunk's ref (main, master, trunk, none), else the ref most of the place's records name (the registered photos, de-spiegel
// and link2pay were built from nox-mesh), the newer of two equal; then its newest record.
best := map[place]delivery.BuildRecord{}
for k, recs := range byPlace {
count, latest := map[string]int{}, map[string]delivery.BuildRecord{}
for _, r := range recs {
count[r.Ref]++
if cur, ok := latest[r.Ref]; !ok || newer(r, cur) {
latest[r.Ref] = r
}
}
var pick string
first := true
for ref, rec := range latest {
switch {
case first:
case trunkLike[ref] != trunkLike[pick]:
if !trunkLike[ref] {
continue
}
case count[ref] != count[pick]:
if count[ref] < count[pick] {
continue
}
case !newer(rec, latest[pick]):
continue
}
pick, first = ref, false
}
best[k] = latest[pick]
}
recs := slices.Collect(maps.Values(best))
slices.SortFunc(recs, func(x, y delivery.BuildRecord) int {
switch {
case trunkLike[x.Ref] != trunkLike[y.Ref]:
if trunkLike[x.Ref] {
return -1
}
return 1
case newer(x, y):
return -1
case newer(y, x):
return 1
}
return strings.Compare(x.Repository+x.Path, y.Repository+y.Path)
})
out := make([]Registration, 0, len(recs))
for _, r := range recs {
out = append(out, Registration{Module: r.Module, Repository: r.Repository, Path: r.Path, Commit: r.Commit, Ref: r.Ref})
}
return out
}
// Registered is the registrations of every module a record names, by module name.
func (b *Builds) Registered() []Registration {
b.mu.Lock()
names := map[string]bool{}
for _, r := range b.records {
names[r.Module] = true
}
b.mu.Unlock()
var out []Registration
for _, name := range slices.Sorted(maps.Keys(names)) {
out = append(out, b.RegistrationsOf(name)...)
}
return out
}
@@ -8,7 +8,6 @@ import (
"slices" "slices"
"strings" "strings"
"sync" "sync"
"sync/atomic"
"git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery"
) )
@@ -54,9 +53,6 @@ type Indexer struct {
// has none. // has none.
Named func(ctx context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error) Named func(ctx context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error)
Trunk func(ctx context.Context, repository, sha string) ([]string, error) Trunk func(ctx context.Context, repository, sha string) ([]string, error)
// Read asks the forge for a repository's trees at a commit and the manifest and lock-file hashes of the one module directory dir (the
// root spelled ""), as the forge bridge reads them for a merged message (registered.go). Nil indexes no registered module.
Read func(ctx context.Context, repository, sha, dir string) (RepoRead, error)
// Contains says whether a commit is the second or an ancestor of it (gitea.gitea_contains). // Contains says whether a commit is the second or an ancestor of it (gitea.gitea_contains).
Contains func(ctx context.Context, repository, commit, in string) (bool, error) Contains func(ctx context.Context, repository, commit, in string) (bool, error)
Logf func(format string, args ...any) Logf func(format string, args ...any)
@@ -65,11 +61,6 @@ type Indexer struct {
source sourceNow source sourceNow
previews map[string]delivery.Preview // by previews key (previews.go) previews map[string]delivery.Preview // by previews key (previews.go)
toolchains map[string]map[string]delivery.Toolchain // language, then node (toolchains.go) toolchains map[string]map[string]delivery.Toolchain // language, then node (toolchains.go)
tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go)
unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread
retrying atomic.Bool // a RetryUnread runs in the background (kickRetry)
retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one
retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on
recompute sync.Mutex // one Recompute at a time recompute sync.Mutex // one Recompute at a time
} }
@@ -94,7 +85,6 @@ func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error {
if err := ix.applyMerged(ctx, f); err != nil { if err := ix.applyMerged(ctx, f); err != nil {
return err return err
} }
ix.kickRetry(ctx) // a registered module the forge could not read is read again on a merge, in the background (issue 1945)
} }
var digests map[string]string var digests map[string]string
var held map[string]delivery.Module var held map[string]delivery.Module
@@ -25,13 +25,6 @@ import (
// A value the module already holds but for its commit is not written again: its commit stays the one its fingerprint first // A value the module already holds but for its commit is not written again: its commit stays the one its fingerprint first
// appeared at (writeOverlay), so a merge writes only the modules it changes. // appeared at (writeOverlay), so a merge writes only the modules it changes.
func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error { func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
return ix.applyManifests(ctx, f, true)
}
// applyManifests is applyMerged; drop says whether the modules of the repository the message holds no manifest for are
// deleted. A message that names only some of a repository's modules (a registered module read on its own, registered.go)
// drops none.
func (ix *Indexer) applyManifests(ctx context.Context, f delivery.Forge, drop bool) error {
digests, err := ix.keepManifests(ctx, f) digests, err := ix.keepManifests(ctx, f)
if err != nil { if err != nil {
return err return err
@@ -91,7 +84,7 @@ func (ix *Indexer) applyManifests(ctx context.Context, f delivery.Forge, drop bo
} }
// A module of this repository the merge commit holds no module.json for is gone from the trunk, and so is its key // A module of this repository the merge commit holds no module.json for is gone from the trunk, and so is its key
// (issue 1900). A message that carries no manifests (an older writer) or one that cannot be read whole says nothing. // (issue 1900). A message that carries no manifests (an older writer) or one that cannot be read whole says nothing.
if drop && len(f.Manifests) > 0 && every { if len(f.Manifests) > 0 && every {
if err := ix.dropRemoved(ctx, f.Repository, held, func(key string, _ delivery.Module) bool { return named[key] }); err != nil { if err := ix.dropRemoved(ctx, f.Repository, held, func(key string, _ delivery.Module) bool { return named[key] }); err != nil {
return err return err
} }
@@ -1,275 +0,0 @@
package index
import (
"context"
"fmt"
"maps"
"slices"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// A module registered in the catalogue from its own repository (the operator registers it with the repository and the
// path of its manifest) had no manifest here: the indexer learns a repository only from a merged message, and no merge
// followed the registration. The controller's outcome records name each registered module's repository, directory and
// commit (builds.go Registered), so the indexer reads that module's manifest at the repository's head (the gate's source names it) or, with none named, at the
// registered commit, and writes the same keys a merge writes. A later merge to the repository's trunk reaches the module through the forge bridge's merged
// message as for every other repository.
// RepoRead is a repository at a commit as the forge bridge reads it for a merged message: the root tree, the tree of each
// directory holding a module.json (the root spelled ""), their manifests and the hex sha256 of the lock files beside them.
type RepoRead struct {
Root string
Trees map[string]string
Manifests map[string]string
Locks map[string]map[string]string
}
// IndexRegistered writes the module of a registration as a merge of its repository would, when the modules bucket holds
// none of that name. Its commit is the head the gate's source names for the repository, else the registration's. The
// repository's other modules are neither written nor dropped: they are the merges'. A manifest that is not at the path or
// names another module is said and left. A forge that could not say (a timeout, or a 404 while the forge's account could
// not read the repository) is an error, and the module is kept as unread: it is read again at the next source revision
// and the next merged message (RetryUnread), not only when the indexer starts again (issue 1945).
func (ix *Indexer) IndexRegistered(ctx context.Context, r Registration) error {
_, err := ix.indexRegistered(ctx, r, false)
return err
}
// taken is what indexRegistered did with one registration: the module written, held already, or the reason it was not.
type taken struct {
stored, held bool
repository, commit, dir string
why string
}
func (ix *Indexer) indexRegistered(ctx context.Context, r Registration, force bool) (taken, error) {
if ix.Modules == nil || ix.Read == nil {
return taken{why: "the indexer reads no forge or writes no modules"}, nil
}
key, err := delivery.ModulesKey(r.Module)
if err != nil {
ix.logf("%s: the registered module has no usable key, skipped: %v", r.Module, err)
return taken{why: fmt.Sprintf("the module has no usable key: %v", err)}, nil
}
if _, _, found, err := ix.Modules.Get(ctx, key); err != nil {
return taken{}, fmt.Errorf("modules.%s: %w", key, err)
} else if found {
ix.markRead(r.Module)
return taken{held: true, why: "the modules state holds it already: a merge to its repository writes it from now on"}, nil
}
repository := delivery.RepositoryName(r.Repository)
commit := r.Commit
dir := strings.Trim(r.Path, "/")
if dir == delivery.RootPath {
dir = ""
}
// The head the gate's source names for the repository, when it names one, else the registration's commit. A module a
// merge removed is absent at the head and is said and left.
ix.mu.Lock()
for repo, head := range ix.source.heads {
if delivery.SameRepository(repo, r.Repository) && head != "" {
commit = head
}
}
ix.mu.Unlock()
t := taken{repository: repository, commit: commit, dir: dir}
if repository == "" || commit == "" {
ix.logf("%s: registered with no repository or commit, so its manifest is not read", r.Module)
t.why = "the controller's record names no repository or commit"
return t, nil
}
// One try for each repository, commit and directory: a repository with no manifest where it was registered is said once,
// not at every record of the controller's builds. A person's take-in (force) reads again.
tried := repository + "@" + commit + ":" + dir
ix.mu.Lock()
if ix.tried[tried] && !force {
ix.mu.Unlock()
t.why = "tried already at this commit and directory"
return t, nil
}
if ix.tried == nil {
ix.tried = map[string]bool{}
}
ix.tried[tried] = true
ix.mu.Unlock()
read, err := ix.Read(ctx, repository, commit, dir)
if err != nil {
ix.mu.Lock()
delete(ix.tried, tried) // a forge that could not say is asked again
if ix.unread == nil {
ix.unread = map[string]bool{}
}
ix.unread[r.Module] = true // and read again on the next source revision or merged message (RetryUnread)
ix.mu.Unlock()
return t, fmt.Errorf("%s: reading %s at %s: %w", r.Module, repository, commit, err)
}
ix.markRead(r.Module)
raw, ok := read.Manifests[dir]
if !ok || read.Trees[dir] == "" {
ix.logf("%s: %s at %s holds no module.json at %q, so the registered module is not indexed", r.Module, repository, commit, dir)
t.why = fmt.Sprintf("%s at %s holds no module.json at %q", repository, commit, dir)
return t, nil
}
if name := manifestModule(raw); name != r.Module {
ix.logf("%s: the manifest at %q of %s at %s names the module %q, so the registered module is not indexed", r.Module, dir, repository, commit, name)
t.why = fmt.Sprintf("the manifest at %q of %s at %s names the module %q", dir, repository, commit, name)
return t, nil
}
f := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: repository, Merge: commit, Root: read.Root,
Trees: map[string]string{dir: read.Trees[dir]},
Manifests: map[string]string{dir: raw}}
if l := read.Locks[dir]; len(l) > 0 {
f.Locks = map[string]map[string]string{dir: l}
}
if err := ix.applyManifests(ctx, f, false); err != nil {
return t, err
}
if _, _, found, err := ix.Modules.Get(ctx, key); err != nil {
return t, fmt.Errorf("modules.%s: %w", key, err)
} else if !found {
t.why = "its manifest was read but the modules state holds no value of it (the indexer's log says why)"
return t, nil
}
t.stored = true
ix.logf("%s: the registered module is indexed from %s at %s", r.Module, repository, commit)
return t, nil
}
// markRead takes a module out of the unread ones: the forge answered for it.
func (ix *Indexer) markRead(module string) {
ix.mu.Lock()
delete(ix.unread, module)
ix.mu.Unlock()
}
// RetryUnread reads again every registered module whose last read the forge could not answer (a timeout, or a 404 while
// the forge's account could not read the repository; issue 1945). It is called on the events that already reach the
// indexer, a new source revision and a merged message, so a repository that became readable is taken in without a hand
// act and without a timer of its own (ADR 0437 §8). What still cannot be read is said and kept for the next.
func (ix *Indexer) RetryUnread(ctx context.Context) {
ix.mu.Lock()
names := slices.Sorted(maps.Keys(ix.unread))
ix.mu.Unlock()
for _, module := range names {
if err := ix.IndexRegisteredModule(ctx, module); err != nil {
ix.logf("the registered module %s is still not indexed: %v", module, err)
}
}
}
// kickRetry runs RetryUnread in one background goroutine, so the message or revision that asked it is acknowledged without
// waiting on the forge (each unread module costs up to two forge calls of 30 s each). A kick while one runs makes it run
// once more when it ends; never two at once.
func (ix *Indexer) kickRetry(ctx context.Context) {
ix.mu.Lock()
none := len(ix.unread) == 0
ix.mu.Unlock()
if none {
return
}
ix.retryAgain.Store(true)
if !ix.retrying.CompareAndSwap(false, true) {
return
}
ix.retryWG.Add(1)
go func() {
defer ix.retryWG.Done()
for ix.retryAgain.Swap(false) {
ix.RetryUnread(ctx)
}
ix.retrying.Store(false)
// A kick between the last Swap and the Store found it running and returned: run for it.
if ix.retryAgain.Load() {
ix.kickRetry(ctx)
}
}()
}
// TakenIn is the answer of a take-in (the tool indexer_take_in): whether the module is now in the modules state, from
// where, and otherwise why not.
type TakenIn struct {
Module string `json:"module"`
Stored bool `json:"stored"`
Held bool `json:"held,omitempty"`
Repository string `json:"repository,omitempty"`
Commit string `json:"commit,omitempty"`
Path string `json:"path,omitempty"`
Answer string `json:"answer"`
}
// TakeIn reads a registered module's manifest now, whatever was tried before, and writes it as a merge would (the tool
// indexer_take_in, issue 1945). A module the modules state holds is answered so and left; one with no registration in the
// controller's records, or whose every registration could not be read or holds no manifest of it, is answered with why.
func (ix *Indexer) TakeIn(ctx context.Context, module string) TakenIn {
out := TakenIn{Module: module}
if ix.Builds == nil {
out.Answer = "the indexer reads no controller's build records, so it knows no registration"
return out
}
regs := ix.Builds.RegistrationsOf(module)
if len(regs) == 0 {
if key, err := delivery.ModulesKey(module); err == nil && ix.Modules != nil {
if _, _, found, err := ix.Modules.Get(ctx, key); err == nil && found {
out.Held, out.Answer = true, "the modules state holds it already: a merge to its repository writes it from now on"
return out
}
}
out.Answer = "the controller's build records name no registration of " + module + ": a module from outside the indexed repositories is known only from them, and one inside them from a merge"
return out
}
var whys []string
for _, r := range regs {
t, err := ix.indexRegistered(ctx, r, true)
out.Repository, out.Commit, out.Path = t.repository, t.commit, pathOf(t.dir)
switch {
case err != nil:
whys = append(whys, err.Error())
continue
case t.held:
out.Held, out.Answer = true, t.why
out.Repository, out.Commit, out.Path = "", "", ""
return out
case t.stored:
out.Stored = true
out.Answer = fmt.Sprintf("written to the modules state from %s at %s", t.repository, t.commit)
return out
}
whys = append(whys, t.why)
}
out.Answer = "not taken in: " + strings.Join(whys, "; ")
return out
}
// IndexRegisteredModule indexes the registration the controller's records name for a module, when there is one.
func (ix *Indexer) IndexRegisteredModule(ctx context.Context, module string) error {
if ix.Builds == nil {
return nil
}
var first error
for _, r := range ix.Builds.RegistrationsOf(module) {
if err := ix.IndexRegistered(ctx, r); err != nil && first == nil {
first = err
}
}
return first
}
// IndexRegisteredAll indexes every registered module the modules bucket does not hold. A module that cannot be read is
// said and the others go on; the first error is answered.
func (ix *Indexer) IndexRegisteredAll(ctx context.Context) error {
if ix.Builds == nil {
return nil
}
var first error
for _, r := range ix.Builds.Registered() {
if err := ix.IndexRegistered(ctx, r); err != nil {
ix.logf("the registered module %s is not indexed: %v", r.Module, err)
if first == nil {
first = err
}
}
}
return first
}
@@ -1,258 +0,0 @@
package index
import (
"context"
"errors"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
func TestARegisteredModuleIsIndexedFromItsRepository(t *testing.T) { // the grants part's gaps: modules registered from their own repositories had no manifest here
const forge = "http://novox.internal:20000/jschoubben/snake.git"
snake := `{"module":"snake","version":"1"}`
for _, tc := range []struct {
name string
path string
read RepoRead
err error
ref string // the build record's ref
heads map[string]string // the gate source's heads
wantRead bool // the forge is read
held bool // modules.snake exists already
other bool // modules.other of the same repository exists
want bool // modules.snake is written
}{
{wantRead: true, name: "a module at the root of its repository", path: "", want: true,
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
{wantRead: true, name: "a root spelled as a dot", path: ".", want: true,
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
{wantRead: true, name: "a module in a directory", path: "modules/snake", want: true,
read: RepoRead{Root: "r0", Trees: map[string]string{"modules/snake": "t1"}, Manifests: map[string]string{"modules/snake": snake}}},
{wantRead: true, name: "another module of the repository is neither written nor dropped", path: "", want: true, other: true,
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
{name: "a module the bucket holds is left", path: "", held: true,
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
{wantRead: true, name: "no manifest at the path", path: "modules/x",
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
{wantRead: true, name: "a manifest of another module", path: "",
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": `{"module":"other","version":"1"}`}}},
{wantRead: true, name: "a repository whose default branch is master", path: "", ref: "master", want: true,
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
{wantRead: true, name: "a repository built from a branch of its own (nox-mesh)", path: "", ref: "nox-mesh", want: true,
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
{wantRead: true, name: "a repository the source names, the module unheld and present at the head", path: "", want: true, heads: map[string]string{"jschoubben/snake": h2},
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
{wantRead: true, name: "a forge that cannot say", path: "", err: errors.New("down")},
} {
t.Run(tc.name, func(t *testing.T) {
ix, mods := modulesIndexer(t, rec(t, "build-1", "snake", forge, tc.path, tc.ref, "", "2026-10-19T09:00:00Z", ""))
if tc.held {
if err := mods.Create(context.Background(), "snake", mustModule(t, delivery.Module{Repository: "jschoubben/snake", Commit: h1, Path: ".", None2: "x"})); err != nil {
t.Fatal(err)
}
}
if tc.other {
if err := mods.Create(context.Background(), "other", mustModule(t, delivery.Module{Repository: "jschoubben/snake", Commit: h1, Path: "modules/other", None2: "x"})); err != nil {
t.Fatal(err)
}
}
ix.Manifests = newMem()
var asked struct {
repo, sha, dir string
n int
}
ix.Read = func(_ context.Context, repo, sha, dir string) (RepoRead, error) {
asked.repo, asked.sha, asked.dir = repo, sha, dir
asked.n++
return tc.read, tc.err
}
if tc.heads != nil {
ix.source = sourceNow{rev: 1, heads: tc.heads}
}
err := ix.IndexRegisteredAll(context.Background())
if (err != nil) != (tc.err != nil) {
t.Fatalf("error = %v", err)
}
wantCommit := "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" // the record's
if tc.heads != nil {
wantCommit = h2 // the source's head
}
if tc.wantRead && asked.sha != wantCommit {
t.Errorf("the forge was asked at %s, want %s", asked.sha, wantCommit)
}
wantReads := 1
if !tc.wantRead {
wantReads = 0
}
if tc.wantRead && asked.dir != dirOf(tc.path) {
t.Errorf("the forge was asked for directory %q", asked.dir)
}
if asked.n != wantReads {
t.Errorf("forge reads = %d, want %d", asked.n, wantReads)
}
m, ok := moduleOf(t, mods, "snake")
if tc.want {
if !ok || m.Manifest == "" || m.Repository != "jschoubben/snake" || m.Commit != wantCommit || m.Path != pathOf(dirOf(tc.path)) {
t.Fatalf("modules.snake = %+v, found %v", m, ok)
}
if len(ix.Manifests.(*memStore).vals) != 1 {
t.Errorf("manifests written = %d, want 1", len(ix.Manifests.(*memStore).vals))
}
if asked.repo != "jschoubben/snake" {
t.Errorf("the forge was asked for %q", asked.repo)
}
} else if !tc.held && ok {
t.Fatalf("modules.snake written: %+v", m)
}
if _, ok := moduleOf(t, mods, "other"); ok != tc.other {
t.Errorf("modules.other held = %v, want %v", ok, tc.other)
}
})
}
}
func mustModule(t *testing.T, m delivery.Module) []byte {
t.Helper()
b, err := delivery.EncodeModule(withV(m))
if err != nil {
t.Fatal(err)
}
return b
}
func dirOf(p string) string {
if p == "." {
return ""
}
return p
}
func TestARegistrationOfAnotherRepositoryDoesNotHideTheRealOne(t *testing.T) { // photos: its newest trunk record was a directory of the catalogue's, then novox/photos at nox-mesh
const catalogue, own = "http://novox.internal:20000/novox/mesh-catalog.git", "http://novox.internal:20000/novox/Link2Pay.git"
ix, mods := modulesIndexer(t,
rec(t, "build-1", "link2pay", own, "", "nox-mesh", "", "2026-10-06T23:02:00Z", ""),
rec(t, "build-2", "link2pay", catalogue, "modules/link2pay", "main", "", "2026-10-04T16:02:00Z", ""),
rec(t, "build-3", "link2pay", catalogue, "modules/link2pay", "main", "", "2026-10-03T16:02:00Z", ""))
ix.Manifests = newMem()
var asked []string
ix.Read = func(_ context.Context, repo, _, dir string) (RepoRead, error) {
asked = append(asked, repo+":"+dir)
if repo == "novox/Link2Pay" {
return RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": `{"module":"link2pay","version":"1"}`}}, nil
}
return RepoRead{Root: "r0"}, nil
}
if err := ix.IndexRegisteredAll(context.Background()); err != nil {
t.Fatal(err)
}
m, ok := moduleOf(t, mods, "link2pay")
if !ok || m.Repository != "novox/Link2Pay" {
t.Fatalf("modules.link2pay = %+v, found %v", m, ok)
}
if len(asked) != 2 || asked[0] != "novox/mesh-catalog:modules/link2pay" || asked[1] != "novox/Link2Pay:." && asked[1] != "novox/Link2Pay:" {
t.Errorf("the forge was asked %v: want the catalogue's trunk record first, once, then the repository's own", asked)
}
}
func TestAForgeThatCannotSayLeavesTheModuleUnwrittenAndAsksAgain(t *testing.T) { // a timeout or a 404 at the head: no read at an older commit
// Issue 1945: car-hunter and snake answered 404 until the forge's account could read them, and were read again only when
// the indexer started again. Each row's trigger must now take the module in once the forge reads it.
const snake = `{"module":"snake","version":"1"}`
ctx := context.Background()
for _, tc := range []struct{ failure, trigger string }{
{"timeout", "the indexer starts again"},
{"Gitea API /repos/x/git/trees/y: 404 not found", "the indexer starts again"},
{"Gitea API /repos/x/git/trees/y: 404 not found", "a new source revision"},
{"Gitea API /repos/x/git/trees/y: 404 not found", "a merged message of another repository"},
{"Gitea API /repos/x/git/trees/y: 404 not found", "a person's take-in"},
{"timeout", "a person's take-in"},
} {
t.Run(tc.failure+", then "+tc.trigger, func(t *testing.T) {
ix, mods := modulesIndexer(t, rec(t, "build-1", "snake", "http://novox.internal:20000/jschoubben/snake.git", "", "master", "", "2026-10-19T09:00:00Z", ""))
ix.Manifests = newMem()
ix.source = sourceNow{rev: 1, heads: map[string]string{"jschoubben/snake": h2}}
var asked []string
readable := false
ix.Read = func(_ context.Context, _, sha, _ string) (RepoRead, error) {
asked = append(asked, sha)
if readable {
return RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}, nil
}
return RepoRead{}, errors.New(tc.failure)
}
if err := ix.IndexRegisteredAll(ctx); err == nil {
t.Errorf("no error")
}
if _, ok := moduleOf(t, mods, "snake"); ok || len(asked) != 1 || asked[0] != h2 {
t.Errorf("written %v, asked %v", ok, asked)
}
if err := ix.IndexRegisteredAll(ctx); err == nil || len(asked) != 2 {
t.Errorf("not asked again (asked %v)", asked)
}
readable = true // the forge's account reads the repository now
var got TakenIn
switch tc.trigger {
case "the indexer starts again":
if err := ix.IndexRegisteredAll(ctx); err != nil {
t.Fatal(err)
}
case "a new source revision":
if err := ix.ApplySource(ctx, 2, delivery.Source{Heads: map[string]string{"jschoubben/snake": h2, "novox/other": h1}}); err != nil {
t.Fatal(err)
}
ix.retryWG.Wait() // read again in the background
case "a merged message of another repository":
if err := ix.Apply(ctx, mergedMsg("novox/other", map[string]string{"": "t9"})); err != nil {
t.Fatal(err)
}
ix.retryWG.Wait() // read again in the background
case "a person's take-in":
got = ix.TakeIn(ctx, "snake")
if !got.Stored || got.Repository != "jschoubben/snake" || got.Commit != h2 || got.Path != "." || got.Answer == "" {
t.Errorf("take-in answered %+v", got)
}
}
if m, ok := moduleOf(t, mods, "snake"); !ok || m.Manifest == "" || m.Commit != h2 {
t.Fatalf("after %s the readable module is not taken in: %+v, found %v (asked %v)", tc.trigger, m, ok, asked)
}
// Taken in, it is read no more: a further event asks the forge nothing.
n := len(asked)
if err := ix.ApplySource(ctx, 3, delivery.Source{Heads: map[string]string{"jschoubben/snake": h2}}); err != nil {
t.Fatal(err)
}
ix.retryWG.Wait()
if len(asked) != n {
t.Errorf("the forge was asked again after the module was taken in: %v", asked)
}
if again := ix.TakeIn(ctx, "snake"); again.Stored || !again.Held || len(asked) != n {
t.Errorf("a take-in of a held module answered %+v and asked %v", again, asked)
}
if none := ix.TakeIn(ctx, "nobody"); none.Stored || none.Held || !strings.Contains(none.Answer, "no registration") {
t.Errorf("a take-in of an unregistered module answered %+v", none)
}
})
}
}
func TestARecordOfAPullRequestDoesNotReplaceTheRegisteredRef(t *testing.T) { // a newer feat/x record of the same place
const repo = "http://novox.internal:20000/novox/photos.git"
b := NewBuilds()
for _, r := range [][2]any{
rec(t, "b1", "photos", repo, "", "nox-mesh", "", "2026-10-01T09:00:00Z", ""),
rec(t, "b2", "photos", repo, "", "nox-mesh", "", "2026-10-02T09:00:00Z", ""),
rec(t, "b3", "photos", repo, "", "feat/x", "", "2026-10-09T09:00:00Z", ""),
} {
b.Put(r[0].(string), r[1].([]byte))
}
got := b.RegistrationsOf("photos")
if len(got) != 1 || got[0].Ref != "nox-mesh" {
t.Fatalf("registrations = %+v, want the ref most records name, not the newer feat/x", got)
}
k, v := record(t, "b4", "photos", repo, "", "main", "", "2026-10-03T09:00:00Z", "")
b.Put(k, v) // a trunk's ref beats a branch of its own
if got := b.RegistrationsOf("photos"); len(got) != 1 || got[0].Ref != "main" {
t.Fatalf("registrations = %+v", got)
}
}
@@ -47,10 +47,6 @@ func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s delivery.Sourc
if ix.Modules == nil { if ix.Modules == nil {
return ix.RecomputeCandidates(ctx) return ix.RecomputeCandidates(ctx)
} }
// A registered module the forge could not read is read again at each new source revision (issue 1945), in the
// background: a forge that does not answer must not hold this revision. One taken in later is stamped when it is written
// (applyManifests reads srcOf).
ix.kickRetry(ctx)
held, err := ix.heldModules(ctx) held, err := ix.heldModules(ctx)
if err != nil { if err != nil {
return err return err
@@ -1,64 +0,0 @@
package index
import (
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The indexer's doing tool (novox/hq issue 1945; the instruction "tools are how a module is spoken to"): indexer_take_in
// reads a registered module's manifest now and writes it to the modules state. The tools bundle serves it through the tool
// runner; the indexer's process serves no request subject (ADR 0437 §4), so the tool writes an ask into the indexer's
// state takeins, which the indexer watches and answers there, as the builder answers builder_ask_again. The read tools
// of the indexer's states stay in mesh-delivery-view.
const (
TakeInTool = "indexer_take_in"
// TakeInsState is the indexer's state of take-in asks: the bucket mesh-delivery-indexer_takeins, keyed as the modules state.
TakeInsState = "takeins"
TakeInsBucket = delivery.IndexerModule + "_" + TakeInsState
)
// TakeInAsk is a value of the takeins state: which module a person asked to be taken in, why, who and when, and, once the
// indexer read it, its answer.
type TakeInAsk struct {
V int `json:"v"`
Module string `json:"module"`
Why string `json:"why,omitempty"`
By string `json:"by,omitempty"`
At time.Time `json:"at"`
Result *TakenIn `json:"result,omitempty"`
Answered time.Time `json:"answered,omitzero"`
}
// DecodeTakeIn reads a takeins value.
func DecodeTakeIn(data []byte) (TakeInAsk, error) {
var a TakeInAsk
if err := json.Unmarshal(data, &a); err != nil {
return a, err
}
if a.V != 1 {
return a, fmt.Errorf("a takeins value of version %d, not 1", a.V)
}
if a.Module == "" {
return a, errors.New("a takeins value naming no module")
}
return a, nil
}
// NewTakeIn is the value indexer_take_in writes for a module: its key and the value.
func NewTakeIn(module, why, by string, at time.Time) (string, []byte, error) {
module = strings.TrimSpace(module)
if module == "" {
return "", nil, errors.New("name the module to take in (module)")
}
key, err := delivery.ModulesKey(module)
if err != nil {
return "", nil, err
}
data, err := json.Marshal(TakeInAsk{V: 1, Module: module, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
return key, data, err
}
+13 -24
View File
@@ -24,20 +24,18 @@ type state struct {
} }
type artifact struct { type artifact struct {
Name string `json:"name"` Name string `json:"name"`
Kind string `json:"kind"` Kind string `json:"kind"`
Language string `json:"language"` Language string `json:"language"`
System string `json:"system"` System string `json:"system"`
From string `json:"from"` From string `json:"from"`
Binary string `json:"binary"` Binary string `json:"binary"`
Loads []string `json:"loads"`
} }
type manifest struct { type manifest struct {
Module string `json:"module"` Module string `json:"module"`
Version string `json:"version"` Version string `json:"version"`
Slug string `json:"slug"` Slug string `json:"slug"`
Tools []string `json:"tools"`
Consumes []string `json:"consumes"` Consumes []string `json:"consumes"`
State []state `json:"state"` State []state `json:"state"`
Reads []string `json:"reads"` Reads []string `json:"reads"`
@@ -80,7 +78,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
if m.Module != "mesh-delivery-indexer" { if m.Module != "mesh-delivery-indexer" {
t.Errorf("module = %q", m.Module) t.Errorf("module = %q", m.Module)
} }
if want := []state{{Name: "candidates", History: 10}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "takeins", History: 1}}; !reflect.DeepEqual(m.State, want) { if want := []state{{Name: "candidates", History: 10}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}}; !reflect.DeepEqual(m.State, want) {
t.Errorf("state = %+v, want %+v", m.State, want) t.Errorf("state = %+v, want %+v", m.State, want)
} }
if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-controller.assignments", "mesh-controller.builds", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) { if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-controller.assignments", "mesh-controller.builds", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) {
@@ -97,18 +95,14 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
if m.SecretsOwner != "mesh-delivery-indexer" { if m.SecretsOwner != "mesh-delivery-indexer" {
t.Errorf("secrets-owner = %q", m.SecretsOwner) t.Errorf("secrets-owner = %q", m.SecretsOwner)
} }
// Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths: // Two invokes: the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths:
// ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a // ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a
// commit is in another (issue 1900). // commit is in another (issue 1900).
if want := []string{"gitea.gitea_commit_trees", "gitea.gitea_contains", "gitea.gitea_get_file"}; !reflect.DeepEqual(m.Invokes, want) { if want := []string{"gitea.gitea_commit_trees", "gitea.gitea_contains"}; !reflect.DeepEqual(m.Invokes, want) {
t.Errorf("invokes = %v, want %v", m.Invokes, want) t.Errorf("invokes = %v, want %v", m.Invokes, want)
} }
// 1945: one tool, indexer_take_in, served by its tools bundle through the takeins state; the process serves none. // No emits, no streams, no bus, no seat, no tool (ADR 0437 §4).
if want := []string{"indexer_take_in"}; !reflect.DeepEqual(m.Tools, want) { for _, k := range []string{"streams", "logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats", "bus"} {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
// No emits, no streams, no bus, no seat (ADR 0437 §4).
for _, k := range []string{"streams", "logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "seat", "seats", "bus"} {
if _, ok := top[k]; ok { if _, ok := top[k]; ok {
t.Errorf("the manifest declares %q: the indexer has none", k) t.Errorf("the manifest declares %q: the indexer has none", k)
} }
@@ -140,13 +134,8 @@ func TestManifestProcess(t *testing.T) { // 1238/AC1
if _, bus := p["bus"]; bus { if _, bus := p["bus"]; bus {
t.Errorf("the process declares a bus: it runs on its own broker credential") t.Errorf("the process declares a bus: it runs on its own broker credential")
} }
if len(m.Build.Artifacts) != 2 { if len(m.Build.Artifacts) != 1 {
t.Fatalf("artifacts = %+v, want the program and the tools", m.Build.Artifacts) t.Fatalf("artifacts = %+v", m.Build.Artifacts)
}
if a := m.Build.Artifacts[1]; a.Name != "tools" || a.Kind != "bundle" || a.Binary != "mesh-delivery-indexer-tools" || !reflect.DeepEqual(a.Loads, []string{"mesh-delivery-indexer-tools"}) {
t.Errorf("tools artifact = %+v", a) // 1945
} else if st, err := os.Stat(a.From); err != nil || !st.IsDir() {
t.Errorf("the tools' source %s: %v", a.From, err)
} }
a := m.Build.Artifacts[0] a := m.Build.Artifacts[0]
if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-indexer" || !strings.HasSuffix(a.From, a.Binary) { if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-indexer" || !strings.HasSuffix(a.From, a.Binary) {
+2 -18
View File
@@ -2,9 +2,6 @@
"module": "mesh-delivery-indexer", "module": "mesh-delivery-indexer",
"version": "1", "version": "1",
"slug": "indexer", "slug": "indexer",
"tools": [
"indexer_take_in"
],
"consumes": [ "consumes": [
"mesh-build-go.toolchain.reported", "mesh-build-go.toolchain.reported",
"mesh-build-node.toolchain.reported", "mesh-build-node.toolchain.reported",
@@ -14,8 +11,7 @@
"state": [ "state": [
{ "name": "candidates", "history": 10 }, { "name": "candidates", "history": 10 },
{ "name": "manifests", "history": 1 }, { "name": "manifests", "history": 1 },
{ "name": "modules", "history": 20 }, { "name": "modules", "history": 20 }
{ "name": "takeins", "history": 1 }
], ],
"reads": [ "reads": [
"mesh-build-go.toolchains", "mesh-build-go.toolchains",
@@ -27,8 +23,7 @@
], ],
"invokes": [ "invokes": [
"gitea.gitea_commit_trees", "gitea.gitea_commit_trees",
"gitea.gitea_contains", "gitea.gitea_contains"
"gitea.gitea_get_file"
], ],
"own-secrets": { "own-secrets": {
"broker": "${dir:state}/broker" "broker": "${dir:state}/broker"
@@ -76,17 +71,6 @@
"system": "arch", "system": "arch",
"from": "cmd/mesh-delivery-indexer", "from": "cmd/mesh-delivery-indexer",
"binary": "mesh-delivery-indexer" "binary": "mesh-delivery-indexer"
},
{
"name": "tools",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/mesh-delivery-indexer-tools",
"binary": "mesh-delivery-indexer-tools",
"loads": [
"mesh-delivery-indexer-tools"
]
} }
] ]
} }
@@ -22,8 +22,6 @@ import (
"strings" "strings"
"syscall" "syscall"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go" "github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream" "github.com/nats-io/nats.go/jetstream"
@@ -121,14 +119,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return nil return nil
} }
if err != nil { if err != nil {
// Essential: the gate's asks are what the teller tells (issue 1945: the error names the grant). return fmt.Errorf("the bucket %s: %w", delivery.AsksBucket, err)
return process.EssentialErr("teller", delivery.AsksBucket, "tell any of the gate's asks", err)
} }
// The answers it writes and what it asked: its own work, so each stops the start, naming its grant.
var kvs [2]jetstream.KeyValue var kvs [2]jetstream.KeyValue
for i, name := range []string{delivery.AnswersBucket, askedBucket} { for i, name := range []string{delivery.AnswersBucket, askedBucket} {
if kvs[i], err = process.EssentialBucket(ctx, js, "teller", name, "write an answer or remember what it asked"); err != nil { if kvs[i], err = js.KeyValue(ctx, name); err != nil {
return err return fmt.Errorf("the bucket %s: %w", name, err)
} }
} }
state := kvStore{ctx: ctx, kv: kvs[1]} state := kvStore{ctx: ctx, kv: kvs[1]}
@@ -144,7 +140,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// The view's answers to the assigner's asks (issue 1297 B10): both buckets are the controller's to create, and a bucket not // The view's answers to the assigner's asks (issue 1297 B10): both buckets are the controller's to create, and a bucket not
// there is said and leaves the teller without that path rather than without the gate's asks (a wait for it needs a timer). // there is said and leaves the teller without that path rather than without the gate's asks (a wait for it needs a timer).
failed := make(chan error, 2) failed := make(chan error, 2)
assignerKV, viewKV := process.OptionalBucket(ctx, js, delivery.AssignerAsksBucket, teller, logger.Printf), process.OptionalBucket(ctx, js, delivery.ViewAnswersBucket, teller, logger.Printf) assignerKV, viewKV := optionalBucket(ctx, js, delivery.AssignerAsksBucket, logger), optionalBucket(ctx, js, delivery.ViewAnswersBucket, logger)
// The assigner's asks are raised on the operator channel whether or not the view's answers can be read (issue 1906). // The assigner's asks are raised on the operator channel whether or not the view's answers can be read (issue 1906).
if viewKV != nil { if viewKV != nil {
t.View = kvGate{ctx: ctx, kv: viewKV} t.View = kvGate{ctx: ctx, kv: viewKV}
@@ -176,8 +172,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
} }
} }
// teller is what an optional bucket of the teller leaves out. // optionalBucket is a bucket the teller reads when it exists, and nil, said once, when it does not.
const teller = "the asks and answers it holds are not carried until the teller is started again with it" func optionalBucket(ctx context.Context, js jetstream.JetStream, name string, logger *log.Logger) jetstream.KeyValue {
kv, err := js.KeyValue(ctx, name)
if err != nil {
logger.Printf("the bucket %s cannot be read (%v): the asks and answers it holds are not carried until the teller is started again with it", name, err)
return nil
}
return kv
}
// askedBucket is the teller's state bucket asked, `<module>_<name>`: what it asked of the router and which condition it // askedBucket is the teller's state bucket asked, `<module>_<name>`: what it asked of the router and which condition it
// raised (manifest state entry asked, novox/hq ADR 0437 §4). // raised (manifest state entry asked, novox/hq ADR 0437 §4).
@@ -1,17 +1,12 @@
package main package main
import ( import (
"context"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"strings" "strings"
"testing" "testing"
"time" "time"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
stdio "git.novox.be/novox/mesh-sdk/go" stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/asker" "git.novox.be/novox/mesh-sdk/go/asker"
"git.novox.be/novox/mesh-sdk/go/asks" "git.novox.be/novox/mesh-sdk/go/asks"
@@ -241,16 +236,6 @@ func (b *bed) askedOf(subject string) []published {
} }
func TestAnAsksKeyBecomesOneRouterAskAndDeletingItCancelsIt(t *testing.T) { // 1343/AC1 func TestAnAsksKeyBecomesOneRouterAskAndDeletingItCancelsIt(t *testing.T) { // 1343/AC1
t.Run("1945: a refused bucket is said, or stops the start naming its grant", func(t *testing.T) {
refusal := errors.New("nats: permissions violation for publish to $JS.API.STREAM.INFO.KV_x")
var said []string
if kv := process.OptionalBucket(context.Background(), refusingBus{refusal}, "x", "x is not heard", func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }); kv != nil || len(said) != 1 || !strings.Contains(said[0], "permissions violation") {
t.Errorf("an optional bucket refused: %v, said %v", kv, said)
}
if _, err := process.EssentialBucket(context.Background(), refusingBus{refusal}, "teller", "x", "work"); err == nil || !strings.Contains(err.Error(), "grant of x") {
t.Errorf("an essential bucket refused: %v, want its grant named", err)
}
})
b := newBed(t) b := newBed(t)
key, value, ga := gateAsk(t, twoHeads(), twoHeads()) key, value, ga := gateAsk(t, twoHeads(), twoHeads())
b.gate[key] = value b.gate[key] = value
@@ -533,8 +518,3 @@ func TestAnAnswerToAnEarlierAskingDoesNotBlockTheNextAsking(t *testing.T) { // R
t.Errorf("writes %d asks %d dropped %v", b.answers.writes, len(b.askedOf(asks.AskSubject(delivery.TellerModule))), b.dropped) t.Errorf("writes %d asks %d dropped %v", b.answers.writes, len(b.askedOf(asks.AskSubject(delivery.TellerModule))), b.dropped)
} }
} }
// refusingBus refuses every bucket with err (issue 1945).
type refusingBus struct{ err error }
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { return nil, r.err }
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-teller
go 1.26.0 go 1.26.0
require ( require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5
github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0 github.com/nats-io/nats.go v1.54.0
) )
-2
View File
@@ -6,8 +6,6 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8 h1:vhYk/iv4
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 h1:gE80bR0tPME7MTMBLcdP62v+F/9soVXmdcBYNMOye44= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 h1:gE80bR0tPME7MTMBLcdP62v+F/9soVXmdcBYNMOye44=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -9,27 +9,13 @@ import (
"os" "os"
"strings" "strings"
"mesh-delivery-view/internal/view"
"git.novox.be/novox/mesh-sdk/go/delivery"
stdio "git.novox.be/novox/mesh-sdk/go" stdio "git.novox.be/novox/mesh-sdk/go"
"mesh-delivery-view/internal/view"
) )
// Tools are the read tools over a modules state, and build_usage over the build modules' states. // Tools are the read tools over a modules state.
func Tools(r view.Reader, u view.UsageReader) []stdio.Tool { func Tools(r view.Reader) []stdio.Tool {
return []stdio.Tool{ return []stdio.Tool{
{Name: view.BuildUsageTool,
Description: "How the build nodes are used, and whether a node is eligible to build, per node (every node holding a build module or named by a capacity): each build seat it holds (node-build-go, -node, -image) with its capacity " +
"(mesh-config's resolved value, or the modules' default) and pause, and whether it is eligible to build and why not; the asks " +
"running now by kind (delivery, check, recheck, try), the keys left running (past the step bound, or their module not running) and the ones " +
"their module ended for that (a restart, or past the step bound), counted apart and in no median or occupancy; " +
"the steps started and finished in the last hour and the last day, failed ones, total and median time, by kind, and how long and " +
"what share of the window the node ran its capacity of asks. Mesh-wide: the backlog of asks waiting and held on each build seat's " +
"stream, and the tries running. Read from the bus's states (have, mesh-config, the build modules' steps and pauses, the checker's tries); " +
"the backlog from the view program's reading of the streams; CPU is not on the bus and the answer names the per-node tool. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) { return u.Usage(), nil }},
{Name: view.ModuleReadTool, {Name: view.ModuleReadTool,
Description: "What the indexer stored for one module under modules.<module>, as stored: its fingerprint (fp, none), the " + Description: "What the indexer stored for one module under modules.<module>, as stored: its fingerprint (fp, none), the " +
"new-form fingerprint (fp2) or why it has none (none2), the inputs it was computed from (inputs2), the module's " + "new-form fingerprint (fp2) or why it has none (none2), the inputs it was computed from (inputs2), the module's " +
@@ -99,8 +85,7 @@ func Tools(r view.Reader, u view.UsageReader) []stdio.Tool {
}}, }},
{Name: view.WantsReadTool, {Name: view.WantsReadTool,
Description: "The assigner's wants and holds as stored in mesh-delivery-assigner_wants: wants by key (want.<node>.<module>) and " + Description: "The assigner's wants and holds as stored in mesh-delivery-assigner_wants: wants by key (want.<node>.<module>) and " +
"held fingerprints by key (held.<module>.<hex>). Narrow by node and module (a hold has no node and is shown when no node is given); a node and a module read that one key. " + "held fingerprints by key (held.<module>.<hex>). Narrow by node and module (a hold has no node and is shown when no node is given). Read-only.",
"At most 1000 values a call; the answer says when it was cut (cut, note). Read-only.",
Input: map[string]any{"type": "object", "properties": nodeModule}, Input: map[string]any{"type": "object", "properties": nodeModule},
Run: func(args map[string]any) (any, error) { Run: func(args map[string]any) (any, error) {
node, module := nodeModuleOf(args) node, module := nodeModuleOf(args)
@@ -115,8 +100,7 @@ func Tools(r view.Reader, u view.UsageReader) []stdio.Tool {
}}, }},
{Name: view.DifferencesReadTool, {Name: view.DifferencesReadTool,
Description: "The assigner's differences between a want and what the controller sends, as stored in mesh-delivery-assigner_differences, " + Description: "The assigner's differences between a want and what the controller sends, as stored in mesh-delivery-assigner_differences, " +
"by key (<node>.<module>), with the rule that explains each or unexplained. Narrow by node and module (both: that one key). " + "by key (<node>.<module>), with the rule that explains each or unexplained. Narrow by node and module. Read-only.",
"At most 1000 values a call; the answer says when it was cut (cut, note). Read-only.",
Input: map[string]any{"type": "object", "properties": nodeModule}, Input: map[string]any{"type": "object", "properties": nodeModule},
Run: func(args map[string]any) (any, error) { Run: func(args map[string]any) (any, error) {
node, module := nodeModuleOf(args) node, module := nodeModuleOf(args)
@@ -124,8 +108,7 @@ func Tools(r view.Reader, u view.UsageReader) []stdio.Tool {
}}, }},
{Name: view.HavesReadTool, {Name: view.HavesReadTool,
Description: "What each node has, as the node-engine stored it in node-engine_have: by key <node>.<module> (the fingerprint it runs, its phase and health), " + Description: "What each node has, as the node-engine stored it in node-engine_have: by key <node>.<module> (the fingerprint it runs, its phase and health), " +
"and <node>.node (the node's own facts). Narrow by node and module (both: that one key). " + "and <node>.node (the node's own facts). Narrow by node and module. Read-only.",
"At most 1000 values a call; the answer says when it was cut (cut, note). Read-only.",
Input: map[string]any{"type": "object", "properties": nodeModule}, Input: map[string]any{"type": "object", "properties": nodeModule},
Run: func(args map[string]any) (any, error) { Run: func(args map[string]any) (any, error) {
node, module := nodeModuleOf(args) node, module := nodeModuleOf(args)
@@ -144,24 +127,6 @@ func nodeModuleOf(args map[string]any) (string, string) {
return strings.TrimSpace(node), strings.TrimSpace(module) return strings.TrimSpace(node), strings.TrimSpace(module)
} }
// socket is the view program's socket, as the program reads it (MESH_DELIVERY_SOCKET, else its default).
func socket() string {
if s := strings.TrimSpace(os.Getenv("MESH_DELIVERY_SOCKET")); s != "" {
return s
}
return "/var/lib/mesh-delivery-view-cli/cli.sock"
}
// backlogOf asks the view program on its socket for the backlog it reads from the seats' streams' workers: the tool
// runtime grants a bundle states, and the stream and worker information is granted to the view program's own account
// (mesh-sdk buscompose.BuildBacklogReads).
func backlogOf(path string) func() ([]view.BacklogRow, error) {
return func() ([]view.BacklogRow, error) {
s, err := view.Ask(path)
return s.Backlog, err
}
}
func read(state stdio.KeptState) view.StateRead { func read(state stdio.KeptState) view.StateRead {
return view.StateRead{Keys: state.Keys, GetEntry: func(key string) (json.RawMessage, uint64, error) { return view.StateRead{Keys: state.Keys, GetEntry: func(key string) (json.RawMessage, uint64, error) {
e, err := state.Get(key) e, err := state.Get(key)
@@ -181,14 +146,7 @@ func read(state stdio.KeptState) view.StateRead {
func main() { func main() {
r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds")), CandidatesState: read(stdio.State("mesh-delivery-indexer.candidates")), r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds")), CandidatesState: read(stdio.State("mesh-delivery-indexer.candidates")),
WantsState: read(stdio.State("mesh-delivery-assigner.wants")), DifferencesState: read(stdio.State("mesh-delivery-assigner.differences")), HavesState: read(stdio.State("node-engine.have"))} WantsState: read(stdio.State("mesh-delivery-assigner.wants")), DifferencesState: read(stdio.State("mesh-delivery-assigner.differences")), HavesState: read(stdio.State("node-engine.have"))}
u := view.UsageReader{Haves: r.HavesState, Resolved: read(stdio.State("mesh-config.resolved")), Tries: read(stdio.State("mesh-delivery-checker.tries")), if err := stdio.Serve("", Tools(r)); err != nil {
Steps: map[string]view.StateRead{}, Pauses: map[string]view.StateRead{}, Backlog: backlogOf(socket())}
for _, lang := range delivery.AskLanguages {
module, _ := delivery.BuildModuleOf(lang)
u.Steps[lang] = read(stdio.State(module + "." + delivery.StepsState))
u.Pauses[lang] = read(stdio.State(module + "." + delivery.PausesState))
}
if err := stdio.Serve("", Tools(r, u)); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-view-tools] %v\n", err) fmt.Fprintf(os.Stderr, "[mesh-delivery-view-tools] %v\n", err)
os.Exit(1) os.Exit(1)
} }
@@ -4,9 +4,7 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"strings" "strings"
"sync/atomic"
"testing" "testing"
"time"
"mesh-delivery-view/internal/view" "mesh-delivery-view/internal/view"
) )
@@ -33,7 +31,7 @@ func (f fake) reader() view.Reader {
func run(t *testing.T, r view.Reader, name string, args map[string]any) (any, error) { func run(t *testing.T, r view.Reader, name string, args map[string]any) (any, error) {
t.Helper() t.Helper()
for _, tl := range Tools(r, view.UsageReader{}) { for _, tl := range Tools(r) {
if tl.Name == name { if tl.Name == name {
return tl.Run(args) return tl.Run(args)
} }
@@ -64,7 +62,11 @@ func TestModuleToolAnswersTheStoredValue(t *testing.T) {
if err := json.Unmarshal(out.(json.RawMessage), &m); err != nil || m["fp"] != "a1" || m["fp2"] != "b2" || m["path"] != "." || m["inputs2"] == nil || m["toolchains"] == nil { if err := json.Unmarshal(out.(json.RawMessage), &m); err != nil || m["fp"] != "a1" || m["fp2"] != "b2" || m["path"] != "." || m["inputs2"] == nil || m["toolchains"] == nil {
t.Errorf("stored fields are not all there: %v %v", m, err) t.Errorf("stored fields are not all there: %v %v", m, err)
} }
// A module with no key, and a name that is no key, are said so. }
// A module with no key, and a name that is no key, are said so.
func TestModuleToolRefusesWhatIsNotThere(t *testing.T) {
r := fake{"a": withFP2}.reader()
if _, err := run(t, r, view.ModuleReadTool, map[string]any{"module": "gone"}); err == nil || !strings.Contains(err.Error(), "no module gone") { if _, err := run(t, r, view.ModuleReadTool, map[string]any{"module": "gone"}); err == nil || !strings.Contains(err.Error(), "no module gone") {
t.Errorf("a missing module: %v", err) t.Errorf("a missing module: %v", err)
} }
@@ -162,7 +164,7 @@ func TestBuilderBuildJoinsTheIndexersFingerprint(t *testing.T) {
// bigBuilds is a bucket of n keys over 100 modules that counts the values read. // bigBuilds is a bucket of n keys over 100 modules that counts the values read.
type bigBuilds struct { type bigBuilds struct {
fake fake fake fake
reads *atomic.Int64 reads *int
} }
func newBig(n int) bigBuilds { func newBig(n int) bigBuilds {
@@ -170,15 +172,15 @@ func newBig(n int) bigBuilds {
for i := 0; i < n; i++ { for i := 0; i < n; i++ {
f[fmt.Sprintf("mod%02d.%064x", i%100, i)] = doneBuild f[fmt.Sprintf("mod%02d.%064x", i%100, i)] = doneBuild
} }
return bigBuilds{fake: f, reads: new(atomic.Int64)} return bigBuilds{fake: f, reads: new(int)}
} }
func (b bigBuilds) reader() view.Reader { func (b bigBuilds) reader() view.Reader {
r := b.fake.reader() r := b.fake.reader()
get := r.State.Get get := r.State.Get
r.Builds = view.StateRead{Keys: r.State.Keys, Get: func(k string) (json.RawMessage, error) { b.reads.Add(1); return get(k) }, r.Builds = view.StateRead{Keys: r.State.Keys, Get: func(k string) (json.RawMessage, error) { *b.reads++; return get(k) },
GetEntry: func(k string) (json.RawMessage, uint64, error) { GetEntry: func(k string) (json.RawMessage, uint64, error) {
b.reads.Add(1) *b.reads++
raw, err := get(k) raw, err := get(k)
var rev uint64 var rev uint64
fmt.Sscanf(k[strings.Index(k, ".")+1:], "%x", &rev) // the fake's revision is the key's number fmt.Sscanf(k[strings.Index(k, ".")+1:], "%x", &rev) // the fake's revision is the key's number
@@ -196,8 +198,8 @@ func TestBuilderBuildsFiltersByPrefixBeforeAnyGet(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
a := out.(view.BuildsAnswer) a := out.(view.BuildsAnswer)
if b.reads.Load() != 20 { if *b.reads != 20 {
t.Errorf("read %d values for a module of 20 keys, want 20", b.reads.Load()) t.Errorf("read %d values for a module of 20 keys, want 20", *b.reads)
} }
if len(a.Builds) != 5 || !a.Cut || !strings.Contains(a.Note, "5") { if len(a.Builds) != 5 || !a.Cut || !strings.Contains(a.Note, "5") {
t.Errorf("answer = %d rows, cut %v, note %q", len(a.Builds), a.Cut, a.Note) t.Errorf("answer = %d rows, cut %v, note %q", len(a.Builds), a.Cut, a.Note)
@@ -220,13 +222,13 @@ func TestBuilderBuildsWithoutAModuleIsBounded(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
a := out.(view.BuildsAnswer) a := out.(view.BuildsAnswer)
if len(a.Builds) != view.DefaultBuildsLimit || !a.Cut || b.reads.Load() > view.DefaultBuildsLimit { if len(a.Builds) != view.DefaultBuildsLimit || !a.Cut || *b.reads > view.DefaultBuildsLimit {
t.Errorf("%d rows, cut %v, %d reads", len(a.Builds), a.Cut, b.reads.Load()) t.Errorf("%d rows, cut %v, %d reads", len(a.Builds), a.Cut, *b.reads)
} }
b = newBig(2000) b = newBig(2000)
out, _ = run(t, b.reader(), view.BuilderBuildsReadTool, map[string]any{"state": "failed"}) out, _ = run(t, b.reader(), view.BuilderBuildsReadTool, map[string]any{"state": "failed"})
if a := out.(view.BuildsAnswer); b.reads.Load() != view.MaxBuildReads || !a.Cut || len(a.Builds) != 0 { if a := out.(view.BuildsAnswer); *b.reads != view.MaxBuildReads || !a.Cut || len(a.Builds) != 0 {
t.Errorf("a state matching nothing: %d reads (want the bound %d), cut %v", b.reads.Load(), view.MaxBuildReads, a.Cut) t.Errorf("a state matching nothing: %d reads (want the bound %d), cut %v", *b.reads, view.MaxBuildReads, a.Cut)
} }
} }
@@ -278,41 +280,6 @@ func keptReader() view.Reader {
} }
} }
// slowBucket is a bucket of the live shape (2026-10-25: node_haves and assigner_wants timed out at 30 s): keys over 4 nodes
// and perNode modules each, written by key(node, module), whose Get takes delay and counts the values read, the lists of
// the bucket and the most Gets in flight at once.
type slowBucket struct {
fake fake
delay time.Duration
reads, lists atomic.Int64
inFlight, maxFlight atomic.Int64
}
func newSlow(perNode int, delay time.Duration, key func(node string, i int) string) *slowBucket {
b := &slowBucket{fake: fake{}, delay: delay}
for _, n := range []string{"ace", "g14", "novox", "shanks"} {
for i := 0; i < perNode; i++ {
b.fake[key(n, i)] = `{"fp":"x"}`
}
}
return b
}
func (b *slowBucket) state() view.StateRead {
f := b.fake.reader().State
return view.StateRead{
Keys: func() ([]string, error) { b.lists.Add(1); return f.Keys() },
Get: func(k string) (json.RawMessage, error) {
b.reads.Add(1)
now := b.inFlight.Add(1)
for m := b.maxFlight.Load(); now > m && !b.maxFlight.CompareAndSwap(m, now); m = b.maxFlight.Load() {
}
time.Sleep(b.delay)
b.inFlight.Add(-1)
return f.Get(k)
}}
}
func TestAssignerWantsToolAnswersAsStoredAndNarrows(t *testing.T) { func TestAssignerWantsToolAnswersAsStoredAndNarrows(t *testing.T) {
r := keptReader() r := keptReader()
out, err := run(t, r, view.WantsReadTool, map[string]any{}) out, err := run(t, r, view.WantsReadTool, map[string]any{})
@@ -331,39 +298,6 @@ func TestAssignerWantsToolAnswersAsStoredAndNarrows(t *testing.T) {
if got := out.(view.WantsAnswer).Wants; len(got) != 2 { if got := out.(view.WantsAnswer).Wants; len(got) != 2 {
t.Errorf("module zsh = %v", got) t.Errorf("module zsh = %v", got)
} }
// On the live shape (4 nodes, 150 wants each, 1 ms a Get): a node and a module read that one key and list nothing;
// a node reads only its own keys; the whole answer reads its values in parallel.
slow := newSlow(150, time.Millisecond, func(n string, i int) string { return fmt.Sprintf("want.%s.m%03d", n, i) })
r = view.Reader{WantsState: slow.state()}
for _, row := range []struct {
args map[string]any
wants int
reads, lists int64
}{
{map[string]any{"node": "ace", "module": "m007"}, 1, 1, 0},
{map[string]any{"node": "ace", "module": "gone"}, 0, 1, 0},
{map[string]any{"node": "g14"}, 150, 150, 1},
{map[string]any{"module": "m007"}, 4, 4, 1},
{map[string]any{}, 600, 600, 1},
} {
slow.reads.Store(0)
slow.lists.Store(0)
start := time.Now()
out, err := run(t, r, view.WantsReadTool, row.args)
took := time.Since(start)
if err != nil {
t.Fatalf("%v: %v", row.args, err)
}
a := out.(view.WantsAnswer)
if len(a.Wants) != row.wants || slow.reads.Load() != row.reads || slow.lists.Load() != row.lists || a.Cut {
t.Errorf("%v: %d wants, %d reads, %d lists, cut %v; want %d, %d, %d", row.args, len(a.Wants), slow.reads.Load(), slow.lists.Load(), a.Cut, row.wants, row.reads, row.lists)
}
t.Logf("assigner_wants %v: %d values in %v (one after the other: %v)", row.args, slow.reads.Load(), took, time.Duration(slow.reads.Load())*slow.delay)
}
if slow.maxFlight.Load() < 2 {
t.Errorf("the values were read one at a time (at most %d in flight)", slow.maxFlight.Load())
}
} }
func TestAssignerWantToolAnswersOneAndSaysWhenThereIsNone(t *testing.T) { func TestAssignerWantToolAnswersOneAndSaysWhenThereIsNone(t *testing.T) {
@@ -394,39 +328,4 @@ func TestAssignerDifferencesAndNodeHavesToolsAnswerAsStored(t *testing.T) {
if got := out.(view.KeptAnswer).Keys; len(got) != 2 || got["ace.node"] == nil { if got := out.(view.KeptAnswer).Keys; len(got) != 2 || got["ace.node"] == nil {
t.Errorf("haves of ace (the node's own key too) = %v", got) t.Errorf("haves of ace (the node's own key too) = %v", got)
} }
// More haves than one call reads (4 nodes, 300 each, 1 ms a Get): a node and a module read that one key and list
// nothing; the whole answer reads MaxStateReads values in parallel and says it was cut.
slow := newSlow(300, time.Millisecond, func(n string, i int) string { return fmt.Sprintf("%s.m%03d", n, i) })
r = view.Reader{HavesState: slow.state(), DifferencesState: slow.state()}
for _, row := range []struct {
tool string
args map[string]any
keys int
reads, lists int64
cut bool
}{
{view.HavesReadTool, map[string]any{"node": "shanks", "module": "m042"}, 1, 1, 0, false},
{view.DifferencesReadTool, map[string]any{"node": "shanks", "module": "m042"}, 1, 1, 0, false},
{view.HavesReadTool, map[string]any{"node": "ace"}, 300, 300, 1, false},
{view.HavesReadTool, map[string]any{"module": "m042"}, 4, 4, 1, false},
{view.HavesReadTool, map[string]any{}, view.MaxStateReads, view.MaxStateReads, 1, true},
} {
slow.reads.Store(0)
slow.lists.Store(0)
start := time.Now()
out, err := run(t, r, row.tool, row.args)
took := time.Since(start)
if err != nil {
t.Fatalf("%s %v: %v", row.tool, row.args, err)
}
a := out.(view.KeptAnswer)
if len(a.Keys) != row.keys || slow.reads.Load() != row.reads || slow.lists.Load() != row.lists || a.Cut != row.cut || (a.Cut && !strings.Contains(a.Note, "narrow")) {
t.Errorf("%s %v: %d keys, %d reads, %d lists, cut %v %q; want %d, %d, %d, %v", row.tool, row.args, len(a.Keys), slow.reads.Load(), slow.lists.Load(), a.Cut, a.Note, row.keys, row.reads, row.lists, row.cut)
}
t.Logf("%s %v: %d values in %v (one after the other: %v)", row.tool, row.args, slow.reads.Load(), took, time.Duration(slow.reads.Load())*slow.delay)
}
if slow.maxFlight.Load() < 2 {
t.Errorf("the values were read one at a time (at most %d in flight)", slow.maxFlight.Load())
}
} }
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-view
go 1.26.0 go 1.26.0
require ( require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026125731-2aae6f5580f2 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c
github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0 github.com/nats-io/nats.go v1.54.0
) )
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026125731-2aae6f5580f2 h1:s1WcG7Ll1OkEwwvMKEabdWvPvTiFCHghlXiY1LoIJDA= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026125731-2aae6f5580f2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
+53 -203
View File
@@ -10,8 +10,6 @@ import (
"slices" "slices"
"sort" "sort"
"strings" "strings"
"sync"
"sync/atomic"
"git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery"
) )
@@ -72,13 +70,12 @@ func (r Reader) Modules(none2 bool) (map[string]json.RawMessage, error) {
return nil, err return nil, err
} }
sort.Strings(keys) sort.Strings(keys)
values, err := r.State.values(keys)
if err != nil {
return nil, err
}
out := map[string]json.RawMessage{} out := map[string]json.RawMessage{}
for i, k := range keys { for _, k := range keys {
raw := values[i] raw, err := r.State.Get(k)
if err != nil {
return nil, err
}
if len(raw) == 0 { if len(raw) == 0 {
continue continue
} }
@@ -132,14 +129,13 @@ func (r Reader) Candidates(module string) (map[string]json.RawMessage, []string,
return nil, nil, err return nil, nil, err
} }
sort.Strings(keys) sort.Strings(keys)
values, err := r.CandidatesState.values(keys)
if err != nil {
return nil, nil, err
}
out := map[string]json.RawMessage{} out := map[string]json.RawMessage{}
var unreadable []string var unreadable []string
for i, k := range keys { for _, k := range keys {
raw := values[i] raw, err := r.CandidatesState.Get(k)
if err != nil {
return nil, nil, err
}
if len(raw) == 0 { if len(raw) == 0 {
continue continue
} }
@@ -246,51 +242,35 @@ func (r Reader) BuilderBuilds(module, state string, limit int) (BuildsAnswer, er
} }
candidates = append(candidates, k) candidates = append(candidates, k)
} }
// The values are read in batches, each batch in parallel and taken in key order: without a module or a state a batch
// is only the rows still missing, so no more is read than the answer needs; otherwise up to readWorkers*4.
reads := 0 reads := 0
walk: for _, k := range candidates {
for reads < len(candidates) {
if reads >= MaxBuildReads { if reads >= MaxBuildReads {
out.Cut, out.Note = true, fmt.Sprintf("read %d values and stopped; give a module (and a state) to narrow the answer", MaxBuildReads) out.Cut, out.Note = true, fmt.Sprintf("read %d values and stopped; give a module (and a state) to narrow the answer", MaxBuildReads)
break break
} }
size := readWorkers * 4 reads++
if prefix == "" && state == "" { row, ok, err := r.builderRow(k)
size = limit - len(out.Builds) if err != nil {
}
size = min(size, MaxBuildReads-reads, len(candidates)-reads)
batch := candidates[reads : reads+size]
rows := make([]BuilderRow, len(batch))
found := make([]bool, len(batch))
if err := parallel(len(batch), func(i int) error {
var err error
rows[i], found[i], err = r.builderRow(batch[i])
return err
}); err != nil {
return BuildsAnswer{}, err return BuildsAnswer{}, err
} }
reads += len(batch) if !ok {
for i, row := range rows { continue
if !found[i] { }
if state != "" {
var b struct {
State string `json:"state"`
}
if json.Unmarshal(row.Build, &b) != nil || b.State != state {
continue continue
} }
if state != "" { }
var b struct { out.Builds = append(out.Builds, row)
State string `json:"state"` // Without a module the walk is in key order and stops at the limit; with one, all its (few) keys are read to order them.
} if prefix == "" && len(out.Builds) >= limit {
if json.Unmarshal(row.Build, &b) != nil || b.State != state { if reads < len(candidates) {
continue out.Cut, out.Note = true, fmt.Sprintf("stopped at the limit of %d builds in key order, not newest first; give a module for its newest", limit)
}
}
out.Builds = append(out.Builds, row)
// Without a module the walk is in key order and stops at the limit; with one, all its (few) keys are read to order them.
if prefix == "" && len(out.Builds) >= limit {
if reads-len(batch)+i+1 < len(candidates) {
out.Cut, out.Note = true, fmt.Sprintf("stopped at the limit of %d builds in key order, not newest first; give a module for its newest", limit)
}
break walk
} }
break
} }
} }
sort.SliceStable(out.Builds, func(i, j int) bool { sort.SliceStable(out.Builds, func(i, j int) bool {
@@ -372,9 +352,6 @@ type WantsAnswer struct {
Wants map[string]json.RawMessage `json:"wants"` Wants map[string]json.RawMessage `json:"wants"`
Held map[string]json.RawMessage `json:"held"` Held map[string]json.RawMessage `json:"held"`
Unreadable []string `json:"unreadable,omitempty"` Unreadable []string `json:"unreadable,omitempty"`
// Cut says the answer stopped at MaxStateReads values, and Note how to narrow it.
Cut bool `json:"cut,omitempty"`
Note string `json:"note,omitempty"`
} }
// KeptAnswer is the answer of assigner_differences and node_haves: the keys of the bucket as stored, and the keys that // KeptAnswer is the answer of assigner_differences and node_haves: the keys of the bucket as stored, and the keys that
@@ -382,158 +359,60 @@ type WantsAnswer struct {
type KeptAnswer struct { type KeptAnswer struct {
Keys map[string]json.RawMessage `json:"keys"` Keys map[string]json.RawMessage `json:"keys"`
Unreadable []string `json:"unreadable,omitempty"` Unreadable []string `json:"unreadable,omitempty"`
Cut bool `json:"cut,omitempty"`
Note string `json:"note,omitempty"`
} }
// readWorkers is how many values a read tool asks the bus for at once. Each value is a round trip from the tool through the func (s StateRead) all(what string) (map[string]json.RawMessage, error) {
// runtime to the bus (the runtime opens the bucket and gets the key: two bus requests), so reading a bucket of a few if s.Keys == nil || s.Get == nil {
// hundred keys one after the other outlasted the 30 s a tool call is given on 2026-10-25 (node_haves, assigner_wants). return nil, fmt.Errorf("%s cannot be read here", what)
const readWorkers = 16
// MaxStateReads bounds the values one call of assigner_wants, assigner_differences or node_haves reads; past it the
// answer says it was cut and how to narrow it.
const MaxStateReads = 1000
// parallel runs read(i) for i in [0, n), at most readWorkers at a time, and answers the first error; once one failed no
// further read starts.
func parallel(n int, read func(i int) error) error {
var (
next atomic.Int64
failed atomic.Bool
once sync.Once
first error
wg sync.WaitGroup
)
workers := min(readWorkers, n)
for w := 0; w < workers; w++ {
wg.Add(1)
go func() {
defer wg.Done()
for !failed.Load() {
i := int(next.Add(1) - 1)
if i >= n {
return
}
if err := read(i); err != nil {
once.Do(func() { first = err })
failed.Store(true)
}
}
}()
} }
wg.Wait() keys, err := s.Keys()
return first
}
// values reads the stored value of each key, in parallel, in the order of keys (nil for a key with none).
func (s StateRead) values(keys []string) ([]json.RawMessage, error) {
out := make([]json.RawMessage, len(keys))
err := parallel(len(keys), func(i int) error {
raw, err := s.Get(keys[i])
out[i] = raw
return err
})
return out, err
}
// read reads the values of keys (sorted first) into a map, at most MaxStateReads of them; cut says it stopped there.
func (s StateRead) read(keys []string) (map[string]json.RawMessage, bool, error) {
sort.Strings(keys)
cut := len(keys) > MaxStateReads
if cut {
keys = keys[:MaxStateReads]
}
values, err := s.values(keys)
if err != nil { if err != nil {
return nil, false, err return nil, err
} }
sort.Strings(keys)
out := map[string]json.RawMessage{} out := map[string]json.RawMessage{}
for i, k := range keys { for _, k := range keys {
if len(values[i]) > 0 { raw, err := s.Get(k)
out[k] = values[i] if err != nil {
return nil, err
}
if len(raw) > 0 {
out[k] = raw
} }
} }
return out, cut, nil return out, nil
}
func cutNote(cut bool, n int) string {
if !cut {
return ""
}
return fmt.Sprintf("read the first %d of %d matching keys in key order and stopped; give a node and a module to narrow the answer", MaxStateReads, n)
} }
// Wants is every want and hold of the assigner, narrowed by node and module (a hold has no node and is kept when the module // Wants is every want and hold of the assigner, narrowed by node and module (a hold has no node and is kept when the module
// matches). // matches).
func (r Reader) Wants(node, module string) (WantsAnswer, error) { func (r Reader) Wants(node, module string) (WantsAnswer, error) {
s := r.WantsState all, err := r.WantsState.all("the assigner's wants")
if s.Get == nil {
return WantsAnswer{}, fmt.Errorf("the assigner's wants cannot be read here")
}
out := WantsAnswer{Wants: map[string]json.RawMessage{}, Held: map[string]json.RawMessage{}}
if node != "" && module != "" {
// One key, read directly: no list of the bucket (a hold has no node, so none is shown).
key, err := delivery.WantKey(node, module)
if module == "node" {
key, err = delivery.WantNodeKey(node)
}
if err != nil {
return WantsAnswer{}, err
}
raw, err := s.Get(key)
if err != nil {
return WantsAnswer{}, err
}
if len(raw) > 0 {
out.Wants[key] = raw
}
return out, nil
}
if s.Keys == nil {
return WantsAnswer{}, fmt.Errorf("the assigner's wants cannot be read here")
}
keys, err := s.Keys()
if err != nil { if err != nil {
return WantsAnswer{}, err return WantsAnswer{}, err
} }
// Narrowed by the key before any value is read. out := WantsAnswer{Wants: map[string]json.RawMessage{}, Held: map[string]json.RawMessage{}}
var matching []string for k, raw := range all {
for _, k := range keys {
if n, m, err := delivery.ParseWantKey(k); err == nil { if n, m, err := delivery.ParseWantKey(k); err == nil {
if (node == "" || n == node) && (module == "" || m == module) { if (node == "" || n == node) && (module == "" || m == module) {
matching = append(matching, k) out.Wants[k] = raw
} }
continue continue
} }
// A node's own want, want.<node>.node (ADR 0539 clause 6): listed with the wants, under the module name "node". // A node's own want, want.<node>.node (ADR 0539 clause 6): listed with the wants, under the module name "node".
if n, err := delivery.ParseWantNodeKey(k); err == nil { if n, err := delivery.ParseWantNodeKey(k); err == nil {
if (node == "" || n == node) && (module == "" || module == "node") { if (node == "" || n == node) && (module == "" || module == "node") {
matching = append(matching, k) out.Wants[k] = raw
} }
continue continue
} }
if m, _, err := delivery.ParseHeldKey(k); err == nil { if m, _, err := delivery.ParseHeldKey(k); err == nil {
if node == "" && (module == "" || m == module) { if node == "" && (module == "" || m == module) {
matching = append(matching, k) out.Held[k] = raw
} }
continue continue
} }
out.Unreadable = append(out.Unreadable, k) out.Unreadable = append(out.Unreadable, k)
} }
sort.Strings(out.Unreadable)
values, cut, err := s.read(matching)
if err != nil {
return WantsAnswer{}, err
}
out.Cut, out.Note = cut, cutNote(cut, len(matching))
for k, raw := range values {
if _, _, err := delivery.ParseHeldKey(k); err == nil {
out.Held[k] = raw
} else {
out.Wants[k] = raw
}
}
return out, nil return out, nil
} }
@@ -557,50 +436,21 @@ func (r Reader) Want(node, module string) (json.RawMessage, error) {
} }
func (r Reader) kept(s StateRead, what, node, module string) (KeptAnswer, error) { func (r Reader) kept(s StateRead, what, node, module string) (KeptAnswer, error) {
if s.Get == nil { all, err := s.all(what)
return KeptAnswer{}, fmt.Errorf("%s cannot be read here", what)
}
out := KeptAnswer{Keys: map[string]json.RawMessage{}}
if node != "" && module != "" {
// One key, read directly: no list of the bucket.
key, err := delivery.HaveKey(node, module)
if err != nil {
return KeptAnswer{}, err
}
raw, err := s.Get(key)
if err != nil {
return KeptAnswer{}, err
}
if len(raw) > 0 {
out.Keys[key] = raw
}
return out, nil
}
if s.Keys == nil {
return KeptAnswer{}, fmt.Errorf("%s cannot be read here", what)
}
keys, err := s.Keys()
if err != nil { if err != nil {
return KeptAnswer{}, err return KeptAnswer{}, err
} }
// Narrowed by the key before any value is read. out := KeptAnswer{Keys: map[string]json.RawMessage{}}
var matching []string for k, raw := range all {
for _, k := range keys {
n, m, isNode, err := delivery.ParseHaveKey(k) n, m, isNode, err := delivery.ParseHaveKey(k)
if err != nil { if err != nil {
out.Unreadable = append(out.Unreadable, k) out.Unreadable = append(out.Unreadable, k)
continue continue
} }
if (node == "" || n == node) && (module == "" || (!isNode && m == module)) { if (node == "" || n == node) && (module == "" || (!isNode && m == module)) {
matching = append(matching, k) out.Keys[k] = raw
} }
} }
sort.Strings(out.Unreadable)
values, cut, err := s.read(matching)
if err != nil {
return KeptAnswer{}, err
}
out.Keys, out.Cut, out.Note = values, cut, cutNote(cut, len(matching))
return out, nil return out, nil
} }

Some files were not shown because too many files have changed in this diff Show More