Compare commits
76
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8fee764c73 | ||
|
|
352066dde6 | ||
|
|
2e0013ec25 | ||
|
|
2493aa6529 | ||
|
|
85c89b16ef | ||
|
|
4f75925410 | ||
|
|
df8b9ac3b1 | ||
|
|
fcb1e40582 | ||
|
|
4973372612 | ||
|
|
3f6c085e8a | ||
|
|
040e3097b5 | ||
|
|
feef96ed16 | ||
|
|
4f722899df | ||
|
|
166a918b43 | ||
|
|
1b4c860f63 | ||
|
|
3b6945b2db | ||
|
|
4174e23c04 | ||
|
|
3993175e45 | ||
|
|
60c7ffc981 | ||
|
|
33de9dde4f | ||
|
|
6b85220fa4 | ||
|
|
1cbae7d41c | ||
|
|
b3a4b09f5b | ||
|
|
c8b26735c8 | ||
|
|
907e092abc | ||
|
|
3073495a06 | ||
|
|
16c411e436 | ||
|
|
2d841870a8 | ||
|
|
7f8ce7f5c0 | ||
|
|
07b5f13474 | ||
|
|
f4aac599ca | ||
|
|
2d63f0ba62 | ||
|
|
1d41682715 | ||
|
|
705765d113 | ||
|
|
16a97ed65d | ||
|
|
ecf532af0e | ||
|
|
ce600df98a | ||
|
|
b4a6a34705 | ||
|
|
a7e51b416b | ||
|
|
71f786f146 | ||
|
|
848c6fdc56 | ||
|
|
ead9ee2d13 | ||
|
|
4861d1f78d | ||
|
|
5c76c9d3ac | ||
|
|
699cbfb7c1 | ||
|
|
319d56be7e | ||
|
|
11d3678984 | ||
|
|
f55f150896 | ||
|
|
e1f1b5d18b | ||
|
|
f0fe454145 | ||
|
|
4026f6c188 | ||
|
|
29cfb8c177 | ||
|
|
a092ee9ff9 | ||
|
|
6c3cbee35c | ||
|
|
4b23db28b3 | ||
|
|
646f83642a | ||
|
|
d9bfba1e62 | ||
|
|
f1bb7747ae | ||
|
|
d2eb3e5ecf | ||
|
|
b0b1cf4f01 | ||
|
|
d91ea41f64 | ||
|
|
5fed429827 | ||
|
|
9a569bfb8d | ||
|
|
e2c41ffa5a | ||
|
|
13e4ed8515 | ||
|
|
c66fe11e16 | ||
|
|
75d0eb32be | ||
|
|
c44b9f91cd | ||
|
|
82da252cf0 | ||
|
|
ce72b67cef | ||
|
|
32bc4f3c48 | ||
|
|
2a94d16834 | ||
|
|
da7baa5e39 | ||
|
|
ed1125b66d | ||
|
|
366bc750da | ||
|
|
04a56b7aff |
@@ -1 +1,4 @@
|
||||
vendor/
|
||||
# A compiled program never enters a commit (issue 1577): `go build .` in a command directory leaves one there.
|
||||
modules/*/cmd/*/mesh-*
|
||||
!modules/*/cmd/*/mesh-*.go
|
||||
|
||||
@@ -172,6 +172,9 @@ func TestTheAssignerStartsAndSaysWhatItKeeps(t *testing.T) {
|
||||
// 1297/B2b: a bucket it reads that the bus does not hold stops the start, naming the bucket.
|
||||
func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
|
||||
for _, missing := range readBuckets {
|
||||
if missing == delivery.PlacementBucket {
|
||||
continue // it starts without it, below
|
||||
}
|
||||
b := startBed(t, missing)
|
||||
select {
|
||||
case err := <-b.start(t):
|
||||
@@ -218,6 +221,34 @@ func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
|
||||
t.Fatal("the put was not taken further once its bucket was made")
|
||||
}
|
||||
}
|
||||
|
||||
// Review of L2 (D1): without the indexer's placement state it starts all the same, and once the bucket is made (the
|
||||
// indexer and the assigner come in one delivery) it is read with no restart: a module only the placement names is
|
||||
// said as placed by one source only.
|
||||
pb := startBed(t, delivery.PlacementBucket)
|
||||
pdone := pb.start(t)
|
||||
select {
|
||||
case err := <-pdone:
|
||||
t.Fatalf("without %s the assigner stopped: %v", delivery.PlacementBucket, err)
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
placement, err := pb.js.CreateKeyValue(pb.ctx, jetstream.KeyValueConfig{Bucket: delivery.PlacementBucket, History: 5})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pl, _ := delivery.EncodePlacement(delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "web"}}})
|
||||
plkey, _ := delivery.PlacementKey("n1")
|
||||
if _, err := placement.Put(pb.ctx, plkey, pl); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for deadline := time.Now().Add(15 * time.Second); ; time.Sleep(200 * time.Millisecond) {
|
||||
if c, ok := pb.conditionsSaid(t)[delivery.AssignerModule+".placement.n1.web"]; ok && c.State == asks.StateRaised {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("the placement state made after the start was not read")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// putJSON writes a value into one of the bed's buckets.
|
||||
@@ -406,6 +437,32 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
|
||||
if !cleared {
|
||||
t.Errorf("the condition of web on neither list on g14 was not cleared once g14 got its want")
|
||||
}
|
||||
|
||||
// Issue 1126 (ADR 0547 §2): the catalogue's placement reaches the indexer's state naming web on every node the
|
||||
// assignments name it on. Each node then composes its own: its want names the build alone (Self, no composition), and
|
||||
// its own want says it composes its own, which the node agent reads (mesh-host's TestNats...ComposesAndAppliesItsOwn).
|
||||
for _, node := range []string{"ace", "g14", "shanks"} {
|
||||
pl, err := delivery.EncodePlacement(delivery.Placement{Node: node, Modules: []delivery.PlacedModule{{Module: "web"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pkey, _ := delivery.PlacementKey(node)
|
||||
b.putJSON(t, delivery.PlacementBucket, pkey, pl)
|
||||
}
|
||||
if w := b.wantNow(t, "ace", "web", "ace's want once placed: the build alone, composed by the node", func(w delivery.Want) bool { return w.Self }); w.Composed != "" || w.FP != fNew {
|
||||
t.Errorf("ace's want once placed = %+v", w)
|
||||
}
|
||||
nkey, _ := delivery.WantNodeKey("ace")
|
||||
var self bool
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && !self; time.Sleep(100 * time.Millisecond) {
|
||||
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, nkey); err == nil {
|
||||
nw, err := delivery.DecodeNodeWant(e.Value())
|
||||
self = err == nil && nw.Self
|
||||
}
|
||||
}
|
||||
if !self {
|
||||
t.Error("ace's own want does not say it composes its own once placed")
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B9 (the assigner's half, ADR 0528 clause 6): started whole, the assigner holds a fingerprint whose health check
|
||||
|
||||
@@ -59,6 +59,7 @@ var readBuckets = []string{
|
||||
delivery.CandidatesBucket,
|
||||
delivery.ManifestsBucket,
|
||||
delivery.ModulesBucket,
|
||||
delivery.PlacementBucket,
|
||||
delivery.PresenceBucket,
|
||||
delivery.HaveBucket,
|
||||
delivery.StepsBucket(goBuildModule),
|
||||
@@ -98,6 +99,12 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
|
||||
logf("the bucket %s cannot be opened (%v): no put is taken further until the assigner starts again", name, err)
|
||||
continue
|
||||
}
|
||||
if name == delivery.PlacementBucket && err != nil {
|
||||
// The indexer's placement state comes with the second milestone (ADR 0549 §2): without it the controller's
|
||||
// assignments alone name where modules run, as before any placement was read (ADR 0549 §5).
|
||||
logf("the bucket %s cannot be opened (%v): the controller's assignments alone are read until the assigner starts again", name, err)
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, process.EssentialErr("assigner", name, "write a want from the whole mesh", err)
|
||||
}
|
||||
@@ -151,7 +158,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
logger.Printf("what the assigner keeps could not be said on the bus, so its readers are not granted until it is: %v", err)
|
||||
}
|
||||
logger.Printf("started on the account %s, reading %d buckets", account.User, len(kvs))
|
||||
return follow(ctx, js, kvs, sayVia(ctx, js), emitting{said}, logger)
|
||||
return follow(ctx, js, conn, kvs, sayVia(ctx, js), emitting{said}, logger)
|
||||
}
|
||||
|
||||
// emitting is the process connection as the puts' caller: its calls, and its events with no options.
|
||||
@@ -169,6 +176,7 @@ var watched = []string{
|
||||
delivery.BuilderBuildsBucket,
|
||||
delivery.CandidatesBucket,
|
||||
delivery.ModulesBucket,
|
||||
delivery.PlacementBucket,
|
||||
delivery.PresenceBucket,
|
||||
delivery.HaveBucket,
|
||||
delivery.AssignerAsksBucket,
|
||||
@@ -179,7 +187,7 @@ var watched = []string{
|
||||
|
||||
// follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after
|
||||
// each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends.
|
||||
func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(asks.Condition) error, bus caller, logger *log.Logger) error {
|
||||
func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, kvs buckets, say func(asks.Condition) error, bus caller, logger *log.Logger) error {
|
||||
w := newWorld()
|
||||
dirty := make(chan struct{}, 1)
|
||||
wake := func() {
|
||||
@@ -263,7 +271,12 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
||||
}
|
||||
logger.Printf("the bucket %s is open: puts are taken further", delivery.AssignerPutsBucket)
|
||||
}
|
||||
started := 0
|
||||
for _, name := range watched {
|
||||
if kvs[name] == nil && name == delivery.PlacementBucket {
|
||||
continue // not on the bus at this start (open said so)
|
||||
}
|
||||
started++
|
||||
var opts []jetstream.WatchOpt
|
||||
if name == delivery.HaveBucket {
|
||||
opts = append(opts, jetstream.IncludeHistory()) // what each node ran healthy before, for a set-back
|
||||
@@ -275,7 +288,20 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
||||
if err := watch(delivery.ManifestsBucket, jetstream.MetaOnly()); err != nil {
|
||||
return err
|
||||
}
|
||||
for range len(watched) + 1 {
|
||||
// The indexer's placement state arrives with the same delivery as this assigner (ADR 0549 §2): when it is not on the
|
||||
// bus at this start it is bound once the bus says it was created, and watched from then on, with no restart (issue
|
||||
// 1790's kind). Until then the controller's assignments alone name where modules run (ADR 0549 §5).
|
||||
placementAppeared := make(chan jetstream.KeyValue, 1)
|
||||
if kvs[delivery.PlacementBucket] == nil && js != nil {
|
||||
process.AwaitBucket(ctx, js, nc, delivery.PlacementBucket, "the controller's assignments alone are read",
|
||||
logger.Printf).Ready(func(kv jetstream.KeyValue) { placementAppeared <- kv })
|
||||
}
|
||||
takePlacement := func(kv jetstream.KeyValue) error {
|
||||
kvs[delivery.PlacementBucket] = kv
|
||||
logger.Printf("the bucket %s is open: placement is read beside the controller's assignments", delivery.PlacementBucket)
|
||||
return watch(delivery.PlacementBucket)
|
||||
}
|
||||
for range started + 1 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
@@ -297,6 +323,10 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
||||
case err := <-errs:
|
||||
return err
|
||||
case <-settled:
|
||||
case kv := <-placementAppeared:
|
||||
if err := takePlacement(kv); err != nil {
|
||||
return err
|
||||
}
|
||||
case <-dirty:
|
||||
reopenPuts(p)
|
||||
p.pass(ctx)
|
||||
|
||||
@@ -94,7 +94,7 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
|
||||
kvs[delivery.ControllerComposedBucket].(*watchKV).entries = []kvEntry{{key: ckey, value: cd}}
|
||||
kvs[delivery.AssignerPutsBucket].(*watchKV).closed = true
|
||||
kvs[delivery.AssignerWantsBucket].(*watchKV).wrote = wrote
|
||||
go follow(ctx, nil, kvs, func(asks.Condition) error { return nil }, nil, log.New(io.Discard, "", 0))
|
||||
go follow(ctx, nil, nil, kvs, func(asks.Condition) error { return nil }, nil, log.New(io.Discard, "", 0))
|
||||
select {
|
||||
case key := <-wrote:
|
||||
if key != "want.ace.node" {
|
||||
|
||||
@@ -3,7 +3,6 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -61,7 +60,23 @@ func (p *passer) pass(ctx context.Context) {
|
||||
// controller has composed it and again whenever that digest changes (ADR 0539 §6).
|
||||
func (p *passer) nodeWants(ctx context.Context) {
|
||||
written := p.w.read(delivery.AssignerWantsBucket)
|
||||
placed := p.w.placedNodes()
|
||||
// A node the catalogue places composes its own composition (ADR 0547 §2): its own want names none.
|
||||
for node := range placed {
|
||||
key, err := delivery.WantNodeKey(node)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if before, err := delivery.DecodeNodeWant(written[key]); err == nil && before.Self {
|
||||
continue
|
||||
}
|
||||
want := delivery.NodeWant{Self: true, Words: "the node composes its own composition (ADR 0547 §2)"}
|
||||
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeNodeWant(want) })
|
||||
}
|
||||
for node, d := range p.w.nodeComposed() {
|
||||
if placed[node] {
|
||||
continue
|
||||
}
|
||||
key, err := delivery.WantNodeKey(node)
|
||||
if err != nil {
|
||||
continue
|
||||
@@ -98,6 +113,8 @@ func (p *passer) wants(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
wanted := p.w.wantsOf()
|
||||
frozen := p.sayPlacement(p.w.placementDiffers())
|
||||
placed := p.w.placedNodes()
|
||||
puts := p.putsByModule()
|
||||
p.staleAsks(ctx, trunk)
|
||||
names := map[string]bool{}
|
||||
@@ -114,12 +131,20 @@ func (p *passer) wants(ctx context.Context) {
|
||||
// quiet at the router.
|
||||
first := p.neither == nil
|
||||
stale := map[string]neitherOne{}
|
||||
defer func() { p.sayNeither(neither, stale) }()
|
||||
left := map[string]string{}
|
||||
defer func() { p.sayNeither(neither, stale, left) }()
|
||||
for name := range names {
|
||||
fp := trunk[name].FP2
|
||||
name := name
|
||||
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name],
|
||||
Composed: func(node, fp string) (delivery.ComposedDigest, bool) { return p.w.composed(node, name, fp) },
|
||||
Composed: func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
if placed[node] {
|
||||
// The node composes its own (ADR 0547 §2, §4): no composition of the controller's is waited for, the want
|
||||
// names the build alone, and the node judges the move itself.
|
||||
return delivery.ComposedDigest{Judgement: delivery.JudgedNeither, Words: "the node composes its own"}, true
|
||||
}
|
||||
return p.w.composed(node, name, fp)
|
||||
},
|
||||
GrantedAt: func(fp string) bool {
|
||||
key, err := delivery.ControllerGrantKey(name, fp)
|
||||
return err == nil && p.w.has(delivery.ControllerGrantsBucket, key)
|
||||
@@ -153,7 +178,9 @@ func (p *passer) wants(ctx context.Context) {
|
||||
last[node] = p.w.lastHealthy(node, name, fp)
|
||||
}
|
||||
out := assign.Wants(m, nodes, wanted[name], last)
|
||||
p.yeses(ctx, name, m, placed, wanted[name], &out)
|
||||
gatherNeither(neither, name, out)
|
||||
leftNeither(left, m, wanted[name], out)
|
||||
p.sayNoWant(name, m.Nodes, wanted[name], out)
|
||||
if first {
|
||||
startClears(stale, m, out)
|
||||
@@ -182,6 +209,9 @@ func (p *passer) wants(ctx context.Context) {
|
||||
p.recordAsk(ctx, name, fp, need)
|
||||
}
|
||||
for node, want := range out.Wants {
|
||||
if frozen[node+"."+name] {
|
||||
continue // one source alone names it (ADR 0549 §5): no want, and the one written before stays
|
||||
}
|
||||
key, err := delivery.WantKey(node, name)
|
||||
if err != nil {
|
||||
p.logf("no want of %s on %s is written: %v", name, node, err)
|
||||
@@ -210,7 +240,7 @@ func (p *passer) sayNoWant(module string, nodes []string, before map[string]deli
|
||||
}
|
||||
reason := ""
|
||||
if h, held := out.Rules[node]; held && (!written || w.Removed) {
|
||||
reason = fmt.Sprintf("rule %d: %s", h.Rule, h.Words)
|
||||
reason = h.Reason()
|
||||
}
|
||||
last, said := p.noWant[key]
|
||||
switch {
|
||||
@@ -242,6 +272,48 @@ func gatherNeither(into map[string]neitherOne, module string, out assign.Outcome
|
||||
}
|
||||
}
|
||||
|
||||
// leftNeither says, by node and module, for every node of m not on neither list after the pass, why: "" when the node has a live
|
||||
// want for it (written before and kept, or written in this pass), else why it has none. sayNeither clears a neither condition in
|
||||
// these words, so a clear never says a want the node does not hold (issue 1945: "nats on novox has a want again" while the bus bar
|
||||
// left it with none).
|
||||
func leftNeither(into map[string]string, m assign.Module, before map[string]delivery.Want, out assign.Outcome) {
|
||||
for _, node := range m.Nodes {
|
||||
if _, on := out.Neither[node]; on {
|
||||
continue
|
||||
}
|
||||
w, written := out.Wants[node]
|
||||
if !written {
|
||||
w, written = before[node]
|
||||
}
|
||||
why := ""
|
||||
h, ruled := out.Rules[node]
|
||||
switch {
|
||||
case written && !w.Removed:
|
||||
case (m.Held && !out.Lifted) || out.Held != nil:
|
||||
why = "it is held, and its held condition says so"
|
||||
case slices.ContainsFunc(out.Record, func(r assign.RecordNeed) bool { return r.Node == node }):
|
||||
why = "its move waits for the operator's yes on its record ask"
|
||||
case putOn(m, node):
|
||||
why = "its put waits on its own steps and the operator's answer"
|
||||
case m.FP == "":
|
||||
why = "the trunk has no build of it"
|
||||
case ruled:
|
||||
why = h.Reason()
|
||||
default:
|
||||
why = "the node does not run it"
|
||||
}
|
||||
into[node+"."+m.Name] = why
|
||||
}
|
||||
}
|
||||
|
||||
// putOn says whether an operator's put on node owns m's want there: one still on its steps (ADR 0546), so the rules write none
|
||||
// meanwhile. An ended put (done, refused, released) stays in the bucket and owns nothing.
|
||||
func putOn(m assign.Module, node string) bool {
|
||||
p, put := m.Puts[node]
|
||||
_, owns := assign.PutOwns(p)
|
||||
return put && owns
|
||||
}
|
||||
|
||||
// startClears adds, for the first pass after a start, every node of m that is not on neither list after the pass: a neither
|
||||
// condition a previous process raised for it is cleared. That covers a node with a want and one with none that is not on neither
|
||||
// list, such as a held module (issue 1945: the hold is said on its own, so a neither raised for it before the restart would stay up
|
||||
@@ -258,7 +330,9 @@ func startClears(into map[string]neitherOne, m assign.Module, out assign.Outcome
|
||||
// once, the condition of each one said before that has a want again (issue 1945). A say that fails is made again at the next pass.
|
||||
// stale is, on the first pass after a start, the modules whose condition a previous process may have raised and that have a want:
|
||||
// each is cleared once.
|
||||
func (p *passer) sayNeither(now, stale map[string]neitherOne) {
|
||||
// left is, by node and module, why each assigned node not on neither list after this pass has no want ("" when it has one); a key
|
||||
// it does not hold is no longer assigned.
|
||||
func (p *passer) sayNeither(now, stale map[string]neitherOne, left map[string]string) {
|
||||
if p.neither == nil {
|
||||
p.neither = map[string]neitherOne{}
|
||||
for key, one := range stale {
|
||||
@@ -278,21 +352,29 @@ func (p *passer) sayNeither(now, stale map[string]neitherOne) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
p.logf("%s is assigned to %s and on neither list: no want, and not in the node's own composition; rule %d holds its first want back: %s",
|
||||
one.module, one.node, one.holding.Rule, one.holding.Words)
|
||||
p.logf("%s is assigned to %s and on neither list: no want, and not in the node's own composition; what holds its first want back: %s",
|
||||
one.module, one.node, one.holding.Reason())
|
||||
p.neither[key] = one
|
||||
}
|
||||
for key, one := range p.neither {
|
||||
if _, still := now[key]; still {
|
||||
continue
|
||||
}
|
||||
why, assigned := left[key]
|
||||
if !assigned {
|
||||
why = "it is no longer assigned there"
|
||||
}
|
||||
if p.say != nil {
|
||||
if err := p.say(assign.NeitherCondition(one.node, one.module, assign.Holding{}, false)); err != nil {
|
||||
if err := p.say(assign.NeitherLeftCondition(one.node, one.module, why)); err != nil {
|
||||
p.logf("the condition of %s on neither list on %s could not be cleared, and is cleared at the next pass: %v", one.module, one.node, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
p.logf("%s on %s has a want again", one.module, one.node)
|
||||
if why == "" {
|
||||
p.logf("%s on %s has a want again", one.module, one.node)
|
||||
} else {
|
||||
p.logf("%s on %s is no longer on neither list and has no want: %s", one.module, one.node, why)
|
||||
}
|
||||
delete(p.neither, key)
|
||||
}
|
||||
}
|
||||
@@ -481,13 +563,19 @@ func (p *passer) staleAsks(ctx context.Context, trunk map[string]delivery.Module
|
||||
}
|
||||
for key := range p.w.read(delivery.AssignerAsksBucket) {
|
||||
if p.staleAsk(key, trunk) {
|
||||
// The condition is cleared first and the ask goes only once that was said: the ask is what brings this key back at the
|
||||
// next pass, so a clear that failed once is made again, with no state of its own (issue 1906, note 29).
|
||||
k, kerr := delivery.ParseAnyAskKey(key)
|
||||
held := kerr == nil && (k.Kind == delivery.AskKindHold || k.Kind == delivery.AskKindRestore)
|
||||
if held && !p.clearHeld(k.Module, k.FP) {
|
||||
continue
|
||||
}
|
||||
p.logf("the ask %s is about a fingerprint or composition the trunk has left behind and is taken away", key)
|
||||
p.remove(ctx, delivery.AssignerAsksBucket, key)
|
||||
// The held record stays: it is the memory that this fingerprint failed (ADR 0408 §4), and a revert returns the trunk to
|
||||
// exactly it. Only the condition, which would else ask for an answer that can no longer be given, is cleared, and it is
|
||||
// raised again if the trunk returns to the fingerprint.
|
||||
if k, err := delivery.ParseAnyAskKey(key); err == nil && (k.Kind == delivery.AskKindHold || k.Kind == delivery.AskKindRestore) {
|
||||
p.clearHeld(k.Module, k.FP)
|
||||
if held {
|
||||
delete(p.said, assign.HeldCondition(k.Module, k.FP, "", "", false, false).Key+".raised")
|
||||
}
|
||||
}
|
||||
@@ -566,16 +654,18 @@ func (p *passer) answeredWith(key, option string) bool {
|
||||
}
|
||||
|
||||
// clearHeld clears, once, the condition of a hold the operator released.
|
||||
func (p *passer) clearHeld(module, fp string) {
|
||||
// It says whether nothing is left to say: the clear was said now or before, or there is no router to say it to.
|
||||
func (p *passer) clearHeld(module, fp string) bool {
|
||||
c := assign.HeldCondition(module, fp, "", "", false, false)
|
||||
if p.say == nil || p.said[c.Key+".cleared"] {
|
||||
return
|
||||
return true
|
||||
}
|
||||
if err := p.say(c); err != nil {
|
||||
p.logf("the condition of the released %s could not be cleared, and is cleared at the next pass: %v", module, err)
|
||||
return
|
||||
return false
|
||||
}
|
||||
p.said[c.Key+".cleared"] = true
|
||||
return true
|
||||
}
|
||||
|
||||
// recordAsk writes, once, the ask about a move of a module of policy record that waits for the operator's yes.
|
||||
@@ -667,3 +757,83 @@ func (p *passer) backupAsk(ctx context.Context, key, node, module string, want d
|
||||
}
|
||||
p.said[c.Key] = true
|
||||
}
|
||||
|
||||
// sayPlacement raises, once, the condition of every pair the controller's assignments and the catalogue's placement do not
|
||||
// both name, clears it once they agree, and answers those pairs by node and module (ADR 0549 §5): the pass writes no want
|
||||
// for them and keeps the one written before. A say that fails is made again at the next pass.
|
||||
func (p *passer) sayPlacement(differ []assign.Pair) map[string]bool {
|
||||
frozen := map[string]bool{}
|
||||
now := map[string]bool{}
|
||||
for _, d := range differ {
|
||||
frozen[d.Node+"."+d.Module] = true
|
||||
c := assign.PlacementCondition(d, true)
|
||||
now[c.Key] = true
|
||||
if p.said[c.Key+".raised"] {
|
||||
continue
|
||||
}
|
||||
p.logf("%s on %s is named by %s alone: no want is written for it there until both sources name it (ADR 0549 §5)", d.Module, d.Node, d.Only)
|
||||
if err := p.say(c); err != nil {
|
||||
p.logf("the placement condition of %s on %s could not be said, and is said at the next pass: %v", d.Module, d.Node, err)
|
||||
continue
|
||||
}
|
||||
p.said[c.Key+".raised"] = true
|
||||
}
|
||||
prefix := delivery.AssignerModule + ".placement."
|
||||
for key := range p.said {
|
||||
c, raised := strings.CutSuffix(key, ".raised")
|
||||
if !raised || !strings.HasPrefix(c, prefix) || now[c] {
|
||||
continue
|
||||
}
|
||||
node, module, _ := strings.Cut(strings.TrimPrefix(c, prefix), ".")
|
||||
if err := p.say(assign.PlacementCondition(assign.Pair{Node: node, Module: module}, false)); err != nil {
|
||||
p.logf("the placement condition of %s on %s could not be cleared, and is cleared at the next pass: %v", module, node, err)
|
||||
continue
|
||||
}
|
||||
delete(p.said, key)
|
||||
}
|
||||
return frozen
|
||||
}
|
||||
|
||||
// yeses writes a placed node's wants naming the build alone, and sees to the moves a node that composes its own judged a
|
||||
// person must allow (ADR 0547 §3): for a have of phase
|
||||
// waiting-yes it raises the record ask once, naming the composition waiting and the node's judgement, and on a yes writes
|
||||
// the want again naming it; a yes once written is kept while the want stays at its build, so the node is not asked again.
|
||||
func (p *passer) yeses(ctx context.Context, module string, m assign.Module, placed map[string]bool, before map[string]delivery.Want, out *assign.Outcome) {
|
||||
for _, node := range m.Nodes {
|
||||
if !placed[node] {
|
||||
continue
|
||||
}
|
||||
was, had := before[node]
|
||||
w, written := out.Wants[node]
|
||||
if !written {
|
||||
if !had || was.Removed {
|
||||
continue
|
||||
}
|
||||
w = was
|
||||
}
|
||||
if w.Removed {
|
||||
continue
|
||||
}
|
||||
yes := ""
|
||||
if had && was.FP == w.FP {
|
||||
yes = was.Yes
|
||||
}
|
||||
key, err := delivery.HaveKey(node, module)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if h, err := delivery.DecodeHave(p.w.value(delivery.HaveBucket, key)); err == nil && h.Phase == delivery.PhaseWaitingYes && h.Waits != yes {
|
||||
if m.Allowed != nil && m.Allowed(node, h.Waits) {
|
||||
yes = h.Waits
|
||||
} else {
|
||||
p.recordAsk(ctx, module, w.FP, assign.RecordNeed{Node: node, Digest: h.Waits, Judgement: h.Judgement, Words: h.Detail})
|
||||
}
|
||||
}
|
||||
// A want carried over from before the node composed its own may still name the controller's composition; a placed
|
||||
// node's want names the build alone (ADR 0547 §2).
|
||||
if yes != w.Yes || w.Composed != "" || !w.Self {
|
||||
w.Yes, w.Composed, w.Self = yes, "", true
|
||||
out.Wants[node] = w
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -342,6 +342,9 @@ func putOfTheBus(t *testing.T) {
|
||||
// again after a say that failed, and clears it once when the module has a want again; a module on neither list is also written to
|
||||
// the journal, so it is loud with no operator channel.
|
||||
func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
|
||||
// A pair only one source names is a condition of the same kind (issue 1126): folded here to hold the count.
|
||||
aPairOneSourceNamesAloneGetsNoWantAndOneCondition(t)
|
||||
t.Run("gathers", testThePassGathersTheModulesOnNeitherList)
|
||||
p := newPasser(buckets{}, time.Date(2026, 10, 25, 15, 0, 0, 0, time.UTC))
|
||||
var said []asks.Condition
|
||||
failing := true
|
||||
@@ -357,30 +360,66 @@ func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
|
||||
p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
|
||||
one := map[string]neitherOne{"g14.screen-lock": {node: "g14", module: "screen-lock",
|
||||
holding: assign.Holding{Rule: delivery.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}}}
|
||||
p.sayNeither(one, nil) // the say fails
|
||||
p.sayNeither(one, nil)
|
||||
p.sayNeither(one, nil)
|
||||
p.sayNeither(one, nil, nil) // the say fails
|
||||
p.sayNeither(one, nil, nil)
|
||||
p.sayNeither(one, nil, nil)
|
||||
if len(said) != 1 || said[0].State != asks.StateRaised || said[0].Subject.Machine != "g14" || said[0].Subject.ID != "screen-lock" {
|
||||
t.Fatalf("said = %+v, want one raised condition of screen-lock on g14", said)
|
||||
}
|
||||
if !strings.Contains(strings.Join(logged, "\n"), "screen-lock") {
|
||||
t.Errorf("the journal does not name the module on neither list: %q", logged)
|
||||
}
|
||||
p.sayNeither(map[string]neitherOne{}, nil)
|
||||
p.sayNeither(map[string]neitherOne{}, nil)
|
||||
wants := map[string]string{"g14.screen-lock": ""}
|
||||
p.sayNeither(map[string]neitherOne{}, nil, wants)
|
||||
p.sayNeither(map[string]neitherOne{}, nil, wants)
|
||||
if len(said) != 2 || said[1].State != asks.StateCleared || said[1].Key != said[0].Key {
|
||||
t.Errorf("said = %+v, want the condition cleared once under its key", said)
|
||||
}
|
||||
if last := logged[len(logged)-1]; last != "screen-lock on g14 has a want again" {
|
||||
t.Errorf("the clear said %q", last)
|
||||
}
|
||||
// 1945 (the bus bar): a module that leaves neither list with no want is cleared in words saying so, never "has a want again".
|
||||
bus := map[string]neitherOne{"novox.nats": {node: "novox", module: "nats", holding: assign.Holding{Bus: true, Words: "the bus moves only by the operator's put"}}}
|
||||
p.sayNeither(bus, nil, nil)
|
||||
if c := said[len(said)-1]; c.State != asks.StateRaised || !strings.HasPrefix(c.Explanation, asks.NeedsYou) || !strings.Contains(c.Explanation, "the bus moves only by the operator's put") {
|
||||
t.Errorf("the bus's neither condition = %+v", c)
|
||||
}
|
||||
p.sayNeither(map[string]neitherOne{}, nil, map[string]string{"novox.nats": "it is held, and its held condition says so"})
|
||||
if c := said[len(said)-1]; c.State != asks.StateCleared || strings.Contains(c.Resolved, "has a want") {
|
||||
t.Errorf("a module that left neither list with no want was cleared as %+v", c)
|
||||
}
|
||||
if last := logged[len(logged)-1]; last != "nats on novox is no longer on neither list and has no want: it is held, and its held condition says so" {
|
||||
t.Errorf("the clear without a want said %q", last)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC21: the pass gathers the modules on neither list of every module's outcome, keyed by node and module.
|
||||
func TestThePassGathersTheModulesOnNeitherList(t *testing.T) {
|
||||
func testThePassGathersTheModulesOnNeitherList(t *testing.T) {
|
||||
got := map[string]neitherOne{}
|
||||
gatherNeither(got, "screen-lock", assign.Outcome{Neither: map[string]assign.Holding{"g14": {Rule: delivery.RuleGrant, Words: "w"}}})
|
||||
gatherNeither(got, "rofi", assign.Outcome{})
|
||||
if len(got) != 1 || got["g14.screen-lock"].node != "g14" || got["g14.screen-lock"].module != "screen-lock" || got["g14.screen-lock"].holding.Rule != delivery.RuleGrant {
|
||||
t.Errorf("gathered = %+v", got)
|
||||
}
|
||||
// 1945 (the bus bar): a node off neither list is cleared in words that say why it has no want; an operator's put running
|
||||
// through its steps owns the want, so it is named, never "the node does not run it".
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
puts map[string]delivery.Put
|
||||
before map[string]delivery.Want
|
||||
why string
|
||||
}{
|
||||
{"a live want", nil, map[string]delivery.Want{"novox": {FP: fpTest}}, ""},
|
||||
{"a put on its steps", map[string]delivery.Put{"novox": {FP: fpTest, Step: delivery.PutPerson}}, nil, "its put waits on its own steps and the operator's answer"},
|
||||
{"nothing", nil, nil, "the node does not run it"},
|
||||
{"an ended put", map[string]delivery.Put{"novox": {FP: fpTest, Step: delivery.PutRefused}}, nil, "the node does not run it"},
|
||||
} {
|
||||
left := map[string]string{}
|
||||
leftNeither(left, assign.Module{Name: "nats", FP: fpTest, Nodes: []string{"novox"}, Puts: c.puts}, c.before, assign.Outcome{})
|
||||
if why, there := left["novox.nats"]; !there || why != c.why {
|
||||
t.Errorf("%s: left %q (%v), want %q", c.name, why, there, c.why)
|
||||
}
|
||||
}
|
||||
// 1945 (a first want held back silently): the journal says why a node gets no want when the reason appears or changes, and
|
||||
// when it has a want again: one line per change, not per pass.
|
||||
p := newPasser(buckets{}, time.Date(2026, 10, 25, 21, 30, 0, 0, time.UTC))
|
||||
@@ -399,6 +438,7 @@ func TestThePassGathersTheModulesOnNeitherList(t *testing.T) {
|
||||
{nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: delivery.RuleGrant, Words: "no credential"}}}, "rofi on shanks gets no want: rule 6: no credential"},
|
||||
{nil, wanted, "rofi on shanks has a want"},
|
||||
{map[string]delivery.Want{"shanks": wanted.Wants["shanks"]}, assign.Outcome{}, ""},
|
||||
{nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Bus: true, Words: "the bus moves only by the operator's put"}}}, "rofi on shanks gets no want: the bus moves only by the operator's put"},
|
||||
} {
|
||||
logged = nil
|
||||
p.sayNoWant("rofi", []string{"shanks"}, c.before, c.out)
|
||||
@@ -412,6 +452,7 @@ func TestThePassGathersTheModulesOnNeitherList(t *testing.T) {
|
||||
// nothing it said, so it clears, once, the neither condition of every assigned module not on neither list and the held condition of
|
||||
// every module that is not held. Clearing a condition that is not open is quiet at the router.
|
||||
func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) {
|
||||
t.Run("a held module", testAHeldModuleRaisesNoNeitherAndClearsAStandingOne)
|
||||
wantsKV := &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerWantsBucket: wantsKV, delivery.AssignerAsksBucket: &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
var said []asks.Condition
|
||||
@@ -449,7 +490,7 @@ func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) {
|
||||
// 1945/AC24: given a module held at F under rule 2 on a node that runs it with no want, when the assigner passes, then it raises no
|
||||
// neither condition for it, a neither condition this process raised for it before is cleared, and so is one a previous process
|
||||
// raised (the first pass after a start), each once.
|
||||
func TestAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
|
||||
func testAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
|
||||
setup := func() (*passer, *[]asks.Condition) {
|
||||
p := newPasser(buckets{delivery.AssignerWantsBucket: &memKV{values: map[string][]byte{}}, delivery.AssignerAsksBucket: &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
said := &[]asks.Condition{}
|
||||
@@ -606,6 +647,36 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("a clear that failed to be said is said at the next pass, and the ask goes after it", func(t *testing.T) {
|
||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
|
||||
var said []asks.Condition
|
||||
fail := true
|
||||
p.say = func(c asks.Condition) error {
|
||||
if fail {
|
||||
fail = false
|
||||
return errors.New("router unreachable")
|
||||
}
|
||||
said = append(said, c)
|
||||
return nil
|
||||
}
|
||||
a := assign.HoldAsk(holdOld, "web", fpOld, "novox", "w", 1, time.Now())
|
||||
v, err := delivery.EncodeAssignerAsk(holdOld, a)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asksKV.values[holdOld] = v
|
||||
p.w.put(delivery.AssignerAsksBucket, holdOld, v)
|
||||
p.staleAsks(context.Background(), trunkOf(fpNew))
|
||||
if _, there := asksKV.values[holdOld]; !there || len(said) != 0 {
|
||||
t.Fatalf("after a failed clear: ask left = %v, conditions said = %d; want the ask kept and nothing said", there, len(said))
|
||||
}
|
||||
p.staleAsks(context.Background(), trunkOf(fpNew))
|
||||
if _, there := asksKV.values[holdOld]; there || len(said) != 1 || said[0].State != asks.StateCleared {
|
||||
t.Errorf("at the next pass: ask left = %v, conditions said = %v; want the ask gone and the clear said", there, said)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a lift takes the restore asks of its fingerprint on every node", func(t *testing.T) {
|
||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
|
||||
@@ -635,6 +706,8 @@ func mustKey(key string, err error) string {
|
||||
// (ADR 0545). That record ask is not about the trunk's fingerprint: it survives every pass, so the operator is asked, and on a
|
||||
// yes the want names the revised digest.
|
||||
func TestARecordAskOfAHeldBackNodeSurvivesPassesAndItsYesMovesTheWant(t *testing.T) {
|
||||
// A placed node's yes comes from its have (issue 1126): folded here to hold the count.
|
||||
aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t)
|
||||
fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
revised, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64)
|
||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||
@@ -683,3 +756,133 @@ func TestARecordAskOfAHeldBackNodeSurvivesPassesAndItsYesMovesTheWant(t *testing
|
||||
t.Error("the answered record ask was taken away")
|
||||
}
|
||||
}
|
||||
|
||||
// 1126 (ADR 0549 §5): until the third milestone a pair the controller's assignments and the catalogue's placement do not
|
||||
// both name gets no want and one condition, raised once and cleared once both name it; before any placement was read the
|
||||
// assignments are the one source and nothing is raised.
|
||||
func aPairOneSourceNamesAloneGetsNoWantAndOneCondition(t *testing.T) {
|
||||
p := newPasser(buckets{}, time.Date(2026, 10, 26, 18, 0, 0, 0, time.UTC))
|
||||
var said []asks.Condition
|
||||
p.say = func(c asks.Condition) error { said = append(said, c); return nil }
|
||||
assigned := func(node, module string) {
|
||||
raw, _ := json.Marshal(map[string]string{"node": node, "module": module})
|
||||
p.w.put(provider.AssignmentsBucket, node+"."+module, raw)
|
||||
}
|
||||
place := func(node string, modules ...string) {
|
||||
pl := delivery.Placement{Node: node}
|
||||
for _, m := range modules {
|
||||
pl.Modules = append(pl.Modules, delivery.PlacedModule{Module: m})
|
||||
}
|
||||
raw, err := delivery.EncodePlacement(pl)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := delivery.PlacementKey(node)
|
||||
p.w.put(delivery.PlacementBucket, key, raw)
|
||||
}
|
||||
assigned("g14", "a")
|
||||
assigned("g14", "b")
|
||||
if got, _ := p.w.assigned(); !slices.Equal(got["g14"], []string{"a", "b"}) || len(p.sayPlacement(p.w.placementDiffers())) != 0 || len(said) != 0 {
|
||||
t.Fatalf("before any placement: assigned %v, said %v", got, said)
|
||||
}
|
||||
none := func(context.Context, string) ([]byte, error) { return nil, errNotThere }
|
||||
if m := p.w.mesh(context.Background(), none); m.Input != nil || m.InputUnread != "" {
|
||||
t.Errorf("before any placement the check composes over %+v (%q)", m.Input, m.InputUnread)
|
||||
}
|
||||
place("g14", "a", "c")
|
||||
got, _ := p.w.assigned()
|
||||
frozen := p.sayPlacement(p.w.placementDiffers())
|
||||
p.sayPlacement(p.w.placementDiffers())
|
||||
if !slices.Equal(got["g14"], []string{"a"}) || !frozen["g14.b"] || !frozen["g14.c"] || frozen["g14.a"] || len(said) != 2 ||
|
||||
said[0].State != asks.StateRaised || !strings.Contains(said[0].Explanation, "the controller's assignments names b on g14") ||
|
||||
!strings.Contains(said[1].Explanation, "the catalogue's placement names c on g14") {
|
||||
t.Fatalf("one source alone: assigned %v, frozen %v, said %+v", got, frozen, said)
|
||||
}
|
||||
place("g14", "a", "b", "c")
|
||||
assigned("g14", "c")
|
||||
if frozen := p.sayPlacement(p.w.placementDiffers()); len(frozen) != 0 || len(said) != 4 || said[2].State != asks.StateCleared || said[3].State != asks.StateCleared {
|
||||
t.Errorf("both agree again: frozen %v, said %+v", frozen, said)
|
||||
}
|
||||
|
||||
// 1126 (ADR 0547 §2): placed, the check composes over the input the nodes compose from (frombus), once each placed node
|
||||
// wrote its record; until then it says why and composes the assignments alone.
|
||||
if m := p.w.mesh(context.Background(), none); m.Input != nil || !strings.Contains(m.InputUnread, "g14") {
|
||||
t.Errorf("a placed node with no record: input %+v, unread %q", m.Input, m.InputUnread)
|
||||
}
|
||||
rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "g14", TunnelKey: "k"}})
|
||||
p.w.put(delivery.HaveBucket, "g14.node", rec)
|
||||
m := p.w.mesh(context.Background(), none)
|
||||
if m.Input == nil || len(m.Input.Nodes) != 1 || !slices.Equal(m.Input.Nodes[0].Assigned, []string{"a", "b", "c"}) || m.Input.Nodes[0].Place.Key != "k" {
|
||||
t.Errorf("the check's input from the bus: %+v (%q)", m.Input, m.InputUnread)
|
||||
}
|
||||
}
|
||||
|
||||
// 1126 (ADR 0547 §2 to §4): a node the catalogue places composes its own: its wants name the build alone and its own want
|
||||
// names no composition; a have waiting for a yes raises the record ask once, and the yes is written into the want and
|
||||
// kept there while the want stays at its build.
|
||||
func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) {
|
||||
fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
waiting, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64)
|
||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
|
||||
p.logf = t.Logf
|
||||
put := func(bucket, key string, v []byte) { p.w.put(bucket, key, v) }
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
put(delivery.ModulesBucket, "web", mod) // the trunk's build is not done: rule 1 holds g14 at fpOld
|
||||
put(provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-1"}`))
|
||||
pl, _ := delivery.EncodePlacement(delivery.Placement{Node: "g14", Modules: []delivery.PlacedModule{{Module: "web"}}})
|
||||
put(delivery.PlacementBucket, "placement.g14", pl)
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
put(delivery.PresenceBucket, "g14", on)
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: was, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
put(delivery.HaveBucket, "g14.web", have)
|
||||
wkey, _ := delivery.WantKey("g14", "web")
|
||||
want, _ := delivery.EncodeWant(delivery.Want{FP: fpOld, Composed: was})
|
||||
wantsKV.values[wkey] = want
|
||||
put(delivery.AssignerWantsBucket, wkey, want)
|
||||
sync := func() {
|
||||
for k, v := range wantsKV.values {
|
||||
put(delivery.AssignerWantsBucket, k, v)
|
||||
}
|
||||
}
|
||||
|
||||
p.wants(context.Background())
|
||||
p.nodeWants(context.Background())
|
||||
sync()
|
||||
got, err := delivery.DecodeWant(wantsKV.values[wkey])
|
||||
if err != nil || got.FP != fpOld || got.Composed != "" || !got.Self {
|
||||
t.Fatalf("the placed node's want is %+v (%v), want it at %s, the node composing it", got, err, fpOld)
|
||||
}
|
||||
nkey, _ := delivery.WantNodeKey("g14")
|
||||
if nw, err := delivery.DecodeNodeWant(wantsKV.values[nkey]); err != nil || nw.Composed != "" || !nw.Self {
|
||||
t.Fatalf("the placed node's own want is %+v (%v), want one naming no composition", nw, err)
|
||||
}
|
||||
|
||||
waits, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: was, Phase: delivery.PhaseWaitingYes, Health: delivery.HealthOK,
|
||||
Judgement: delivery.JudgedRecreates, Waits: waiting, Detail: "it recreates web.server"})
|
||||
put(delivery.HaveBucket, "g14.web", waits)
|
||||
akey, _ := delivery.RecordAskKey("g14", "web", waiting)
|
||||
p.wants(context.Background())
|
||||
if _, there := asksKV.values[akey]; !there {
|
||||
t.Fatal("a have waiting for a yes raised no record ask")
|
||||
}
|
||||
a, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
put(delivery.AssignerAsksBucket, akey, asksKV.values[akey])
|
||||
yes, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a.Ask.ID + "@2026-10-26T19:00:00Z", Words: "the operator"})
|
||||
put(delivery.AnswersBucket, akey, yes)
|
||||
p.wants(context.Background())
|
||||
sync()
|
||||
if got, err = delivery.DecodeWant(wantsKV.values[wkey]); err != nil || got.Yes != waiting || got.FP != fpOld {
|
||||
t.Fatalf("after the yes the want is %+v (%v), want it to name the yes %s", got, err, waiting)
|
||||
}
|
||||
// The node applied it: its have runs the composition; the yes stays while the want stays at its build.
|
||||
ran, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: waiting, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
put(delivery.HaveBucket, "g14.web", ran)
|
||||
p.wants(context.Background())
|
||||
if got, _ = delivery.DecodeWant(wantsKV.values[wkey]); got.Yes != waiting {
|
||||
t.Errorf("a pass after the move dropped the yes: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,6 +14,8 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
|
||||
@@ -162,8 +164,56 @@ func (w *world) manifest(ctx context.Context, get getter, digest string) parsed
|
||||
return p
|
||||
}
|
||||
|
||||
// assigned is each node's modules with an open assignment, read from the controller's assignments, in name order.
|
||||
// assigned is each node's modules both the controller's assignments and the indexer's placement name, in name order
|
||||
// (ADR 0549 §5; the assignments alone before any placement was read), with every open assignment by node and module.
|
||||
func (w *world) assigned() (map[string][]string, map[string]assignment) {
|
||||
byAssignment, all := w.assignedByController()
|
||||
placed, seeded := w.placed()
|
||||
both, _ := assign.Placed(byAssignment, placed, seeded)
|
||||
return both, all
|
||||
}
|
||||
|
||||
// placementDiffers is every pair the controller's assignments and the indexer's placement do not both name.
|
||||
func (w *world) placementDiffers() []assign.Pair {
|
||||
byAssignment, _ := w.assignedByController()
|
||||
placed, seeded := w.placed()
|
||||
_, differ := assign.Placed(byAssignment, placed, seeded)
|
||||
return differ
|
||||
}
|
||||
|
||||
// placed is each node's modules in the indexer's placement (ADR 0549 §2), and whether any placement key was read: a key
|
||||
// that does not read places nothing on its node and is said by the indexer's own read tool.
|
||||
func (w *world) placed() (map[string][]string, bool) {
|
||||
out := map[string][]string{}
|
||||
values := w.read(delivery.PlacementBucket)
|
||||
for key, value := range values {
|
||||
node, err := delivery.ParsePlacementKey(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
p, err := delivery.DecodePlacement(value)
|
||||
if err != nil || p.Node != node {
|
||||
continue
|
||||
}
|
||||
out[node] = p.Assigned()
|
||||
}
|
||||
return out, len(values) > 0
|
||||
}
|
||||
|
||||
// placedNodes is every node the indexer's placement names (ADR 0549 §2): those compose their own modules (ADR 0547 §2),
|
||||
// so their wants name the build alone and the seventh rule does not hold them.
|
||||
func (w *world) placedNodes() map[string]bool {
|
||||
placed, _ := w.placed()
|
||||
out := make(map[string]bool, len(placed))
|
||||
for node := range placed {
|
||||
out[node] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// assignedByController is each node's modules with an open assignment, read from the controller's assignments, in
|
||||
// name order.
|
||||
func (w *world) assignedByController() (map[string][]string, map[string]assignment) {
|
||||
out, all := map[string][]string{}, map[string]assignment{}
|
||||
for _, value := range w.read(provider.AssignmentsBucket) {
|
||||
var a assignment
|
||||
@@ -210,9 +260,86 @@ func (w *world) mesh(ctx context.Context, manifests getter) assign.Mesh {
|
||||
}
|
||||
m.Manifests[name] = p.manifest
|
||||
}
|
||||
// Once the catalogue places the nodes, the check composes over the input they compose from (ADR 0547 §2, ADR 0549
|
||||
// §3): one composer's input for the check and the nodes. An input that cannot be read is said, and the nodes and their
|
||||
// assignments alone are composed, as before any placement.
|
||||
if _, seeded := w.placed(); seeded {
|
||||
in, err := frombus.Input(ctx, worldReader{w: w, manifests: manifests}, frombus.Mesh{}, frombus.Scope{Every: true})
|
||||
if err != nil {
|
||||
m.InputUnread = err.Error()
|
||||
} else {
|
||||
m.Input = &in
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// worldReader is what the assigner watched as frombus reads the bus: the indexer's placement and modules, each node's
|
||||
// record in its own have key, the manifests by digest; the check reads no config value, own secret or credential (the
|
||||
// composition it asks resolves, and stands in for what it does not carry).
|
||||
type worldReader struct {
|
||||
w *world
|
||||
manifests getter
|
||||
}
|
||||
|
||||
func (r worldReader) Placements(context.Context) (map[string]delivery.Placement, error) {
|
||||
out := map[string]delivery.Placement{}
|
||||
for key, value := range r.w.read(delivery.PlacementBucket) {
|
||||
node, err := delivery.ParsePlacementKey(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if p, err := delivery.DecodePlacement(value); err == nil && p.Node == node {
|
||||
out[node] = p
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r worldReader) Records(context.Context) (map[string]delivery.NodeRecord, error) {
|
||||
out := map[string]delivery.NodeRecord{}
|
||||
for key, value := range r.w.read(delivery.HaveBucket) {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
if h, err := delivery.DecodeHaveNode(value); err == nil && h.Record != nil {
|
||||
out[node] = *h.Record
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r worldReader) Modules(context.Context) (map[string]delivery.Module, error) {
|
||||
return r.w.trunk(), nil
|
||||
}
|
||||
|
||||
func (r worldReader) Manifest(ctx context.Context, digest string) ([]byte, error) {
|
||||
key, err := delivery.ManifestsKey(digest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := r.manifests(ctx, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m, err := delivery.DecodeManifest(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []byte(m.JSON), nil
|
||||
}
|
||||
|
||||
func (worldReader) Values(context.Context, string, string) (map[string]any, error) { return nil, nil }
|
||||
|
||||
func (worldReader) OwnSecrets(context.Context, string) ([]compose.OwnSecretRow, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (worldReader) Credentials(context.Context, string, []compose.CredentialKey, []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error) {
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
// change is a candidate's change for the composition check, or why the composition waits for it.
|
||||
func (w *world) change(ctx context.Context, manifests getter, c delivery.Candidate, trunk map[string]delivery.Module) (assign.Change, string) {
|
||||
ch := assign.Change{Group: c.Group, Heads: c.HeadsHash, Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{},
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026193323-03dce1237803
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,13 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025195051-20a6dc4920f8 h1:jA7V7hlfJAigVUvJDpKwfgd9ylQ9bqppg3rK9XgLWoA=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025195051-20a6dc4920f8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8 h1:vhYk/iv43u2WGSTrFyQGDvGBxLaRITgMAiiI9DjuA10=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 h1:gE80bR0tPME7MTMBLcdP62v+F/9soVXmdcBYNMOye44=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026193323-03dce1237803 h1:RInAboHBk5THYf6Agvy05iO9p2XV5zGb0ilYa+iSO7o=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026193323-03dce1237803/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -96,6 +96,14 @@ func NeitherCondition(node, module string, h Holding, raised bool) asks.Conditio
|
||||
if n := len([]rune(headline)); n > asks.HeadlineLength {
|
||||
headline = string([]rune(headline)[:asks.HeadlineLength])
|
||||
}
|
||||
if h.Bus {
|
||||
needs := fmt.Sprintf("put %s on %s with assigner_put", module, node)
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline, Needs: needs,
|
||||
Explanation: fmt.Sprintf("%s%s. %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
|
||||
"hold it either, so nothing applies it there. The assigner never moves the bus: %s. This clears once the node has a want for it.",
|
||||
asks.NeedsYou, needs, module, node, h.Words)}
|
||||
}
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline,
|
||||
Explanation: fmt.Sprintf("%s %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
|
||||
@@ -103,6 +111,17 @@ func NeitherCondition(node, module string, h Holding, raised bool) asks.Conditio
|
||||
"rule holds, and clears this.", asks.NothingToDo, module, node, h.Rule, h.Words)}
|
||||
}
|
||||
|
||||
// NeitherLeftCondition clears the neither condition of module on node when it is no longer on neither list: why is "" when the
|
||||
// node has a live want for it again, else why it has none (it is held, its move waits for an ask, the node no longer runs it), so
|
||||
// the clear never says a want the node does not hold (issue 1945: "nats on novox has a want again" while it had none).
|
||||
func NeitherLeftCondition(node, module, why string) asks.Condition {
|
||||
c := NeitherCondition(node, module, Holding{}, false)
|
||||
if why != "" {
|
||||
c.Resolved = fmt.Sprintf("%s on %s is no longer on neither list and has no want: %s.", module, node, why)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// LiftedCondition clears the condition of a held fingerprint the assigner lifted by itself, saying why (issue 1906).
|
||||
func LiftedCondition(module, fp string) asks.Condition {
|
||||
c := HeldCondition(module, fp, "", "", false, false)
|
||||
|
||||
@@ -35,6 +35,13 @@ type Mesh struct {
|
||||
// Digests is every trunk module's manifest digest in the indexer's manifests bucket, readable or not: what a
|
||||
// composer at another mesh-sdk reads (issue 1622).
|
||||
Digests map[string]string
|
||||
// Input is the rest of the mesh as the nodes compose from it, filled by mesh-sdk's frombus from the catalogue's
|
||||
// placement and the nodes' records (novox/hq ADR 0547 §2, ADR 0549 §3, issue 1126), so the check and the nodes share
|
||||
// one composer's input; nil before any placement is read, when the nodes and their assignments alone are composed.
|
||||
Input *compose.Input
|
||||
// InputUnread says why that input could not be read from the bus, when it could not: the check then composes the
|
||||
// nodes and their assignments alone, and says so in its warnings.
|
||||
InputUnread string
|
||||
}
|
||||
|
||||
// Change is a group's change as the indexer's candidate holds it: the manifest of every module it moves or adds, those
|
||||
|
||||
@@ -42,8 +42,15 @@ func HeadJob(m Mesh, c Change) (delivery.ComposeJob, error) {
|
||||
for _, name := range c.Removed {
|
||||
delete(change, name)
|
||||
}
|
||||
req := composer.Request{V: composer.Version, Input: plainInput(m.Assigned), Depth: composer.DepthResolve,
|
||||
HoldingsFromClaims: true, Machines: Touched(m, c),
|
||||
in, fromClaims := plainInput(m.Assigned), true
|
||||
if m.Input != nil {
|
||||
// The input the nodes compose from, without its catalogue: the cases carry the manifests (issue 1126).
|
||||
in = *m.Input
|
||||
in.Catalogue = nil
|
||||
fromClaims = len(in.Holdings) == 0
|
||||
}
|
||||
req := composer.Request{V: composer.Version, Input: in, Depth: composer.DepthResolve,
|
||||
HoldingsFromClaims: fromClaims, Machines: Touched(m, c),
|
||||
Cases: []composer.Case{{Name: CaseTrunk, Manifests: trunk}, {Name: CaseChange, Manifests: change}}}
|
||||
return job(SDKRepository, c.SDKHead, SDKDir, req, "the mesh-sdk head of "+c.Group)
|
||||
}
|
||||
@@ -195,6 +202,10 @@ func FromComposer(m Mesh, c Change, head Ran, readers []Reader, took time.Durati
|
||||
readerFails, readerWarnings := judgeReaders(c, change, readers)
|
||||
fails = append(fails, readerFails...)
|
||||
v.Warnings = append(v.Warnings, readerWarnings...)
|
||||
if m.InputUnread != "" {
|
||||
v.Warnings = append(v.Warnings, "composed over the nodes and their assignments alone: the input the nodes compose from "+
|
||||
"cannot be read from the bus ("+m.InputUnread+")")
|
||||
}
|
||||
return finish(v, m, c, machines, fails)
|
||||
}
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
@@ -100,6 +101,25 @@ func TestAGroupWithAnSDKMemberAsksTheComposerAtItsHead(t *testing.T) {
|
||||
if !slices.Equal(req.Machines, []string{"g14"}) || req.Depth != composer.DepthResolve || !req.HoldingsFromClaims {
|
||||
t.Errorf("request = machines %v, depth %q, holdings from claims %v", req.Machines, req.Depth, req.HoldingsFromClaims)
|
||||
}
|
||||
|
||||
// 1126 (ADR 0547 §2, ADR 0549 §3): once the catalogue places the nodes, the check composes over the input the nodes
|
||||
// compose from (frombus), its catalogue left to the cases, and the seats held as placement says.
|
||||
m := digestedMesh()
|
||||
m.Input = &compose.Input{Catalogue: map[string]catalogue.Manifest{"web": {Module: "web"}},
|
||||
Nodes: []compose.Node{{Name: "g14", Assigned: []string{"web"}, Place: compose.Place{Endpoint: "g14.example:51820"}}},
|
||||
Holdings: []catalogue.Held{{Claim: "web-seat", Scope: "mesh", Node: "g14", Module: "web"}}}
|
||||
placedJob, err := HeadJob(m, c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
placed, err := composer.Decode(placedJob.Request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if placed.Input.Catalogue != nil || len(placed.Input.Nodes) != 1 || placed.Input.Nodes[0].Place.Endpoint != "g14.example:51820" ||
|
||||
len(placed.Input.Holdings) != 1 || placed.HoldingsFromClaims {
|
||||
t.Errorf("the placed check's request = %+v, holdings from claims %v", placed.Input, placed.HoldingsFromClaims)
|
||||
}
|
||||
}
|
||||
|
||||
// 1622/AC2, AC3: a catalogue member using a field only the SDK head reads passes with the head's composer, and the
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Until ADR 0547's third milestone two sources name which module runs on which node: the controller's assignments and
|
||||
// the indexer's placement, read from the catalogue (ADR 0549 §5). The assigner writes wants only for a pair both name:
|
||||
// for a pair one source names alone it writes no want and touches none it wrote before, so nothing moves and nothing is
|
||||
// removed on one source's word, and it raises a condition naming the pair until both agree. Before the first placement
|
||||
// file reaches the indexer (no placement key at all) the controller's assignments are the one source, as before.
|
||||
|
||||
// Pair is a module on a node that one source names and the other does not; Only says which: OnlyAssignments or
|
||||
// OnlyPlacement.
|
||||
type Pair struct {
|
||||
Node, Module, Only string
|
||||
}
|
||||
|
||||
// The source that names a pair alone.
|
||||
const (
|
||||
OnlyAssignments = "the controller's assignments"
|
||||
OnlyPlacement = "the catalogue's placement"
|
||||
)
|
||||
|
||||
// Placed is, by node, the modules both sources name, in name order, and every pair one names alone, by node then
|
||||
// module. With no placement read yet (seeded false) it is the assignments, and no pair differs.
|
||||
func Placed(assigned, placed map[string][]string, seeded bool) (map[string][]string, []Pair) {
|
||||
if !seeded {
|
||||
return assigned, nil
|
||||
}
|
||||
set := func(m map[string][]string) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for node, modules := range m {
|
||||
for _, module := range modules {
|
||||
out[node+"\x00"+module] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
inAssigned, inPlaced := set(assigned), set(placed)
|
||||
both := map[string][]string{}
|
||||
var differ []Pair
|
||||
for node, modules := range assigned {
|
||||
for _, module := range modules {
|
||||
if inPlaced[node+"\x00"+module] {
|
||||
both[node] = append(both[node], module)
|
||||
} else {
|
||||
differ = append(differ, Pair{Node: node, Module: module, Only: OnlyAssignments})
|
||||
}
|
||||
}
|
||||
}
|
||||
for node, modules := range placed {
|
||||
for _, module := range modules {
|
||||
if !inAssigned[node+"\x00"+module] {
|
||||
differ = append(differ, Pair{Node: node, Module: module, Only: OnlyPlacement})
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, modules := range both {
|
||||
sort.Strings(modules)
|
||||
}
|
||||
sort.Slice(differ, func(i, j int) bool {
|
||||
if differ[i].Node != differ[j].Node {
|
||||
return differ[i].Node < differ[j].Node
|
||||
}
|
||||
return differ[i].Module < differ[j].Module
|
||||
})
|
||||
return both, differ
|
||||
}
|
||||
|
||||
// PlacementCondition is the condition of a pair the two sources do not both name: raised while they differ, cleared once
|
||||
// both name it or neither does.
|
||||
func PlacementCondition(p Pair, raised bool) asks.Condition {
|
||||
key := delivery.AssignerModule + ".placement." + p.Node + "." + p.Module
|
||||
subject := asks.Scope{Scope: "module", ID: p.Module, Machine: p.Node}
|
||||
if !raised {
|
||||
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared,
|
||||
Resolved: fmt.Sprintf("the controller's assignments and the catalogue's placement agree on %s on %s.", p.Module, p.Node)}
|
||||
}
|
||||
headline := fmt.Sprintf("%s on %s is placed by one source only", p.Module, p.Node)
|
||||
if n := len([]rune(headline)); n > asks.HeadlineLength {
|
||||
headline = string([]rune(headline)[:asks.HeadlineLength])
|
||||
}
|
||||
var needs string
|
||||
if p.Only == OnlyAssignments {
|
||||
needs = fmt.Sprintf("add %s to the catalogue's placement file of %s, or unassign it at the controller", p.Module, p.Node)
|
||||
} else {
|
||||
needs = fmt.Sprintf("assign %s to %s at the controller, as the merged placement file says", p.Module, p.Node)
|
||||
}
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline, Needs: needs,
|
||||
Explanation: fmt.Sprintf("%s%s. %s names %s on %s and the other source does not. Until ADR 0547's third milestone both must "+
|
||||
"name a module on a node (ADR 0549 §5): the assigner writes no want for it there and leaves the one it wrote before, so "+
|
||||
"nothing moves or is removed. This clears once both name it or neither does.", asks.NeedsYou, needs, p.Only, p.Module, p.Node)}
|
||||
}
|
||||
@@ -192,10 +192,27 @@ type RecordNeed struct {
|
||||
Words string
|
||||
}
|
||||
|
||||
// Holding is a rule holding a node back, and why.
|
||||
// Holding is a rule holding a node back, and why. Bus marks the bar on the bus instead of one of the seven rules (Rule is then
|
||||
// 0): the bus moves only by the operator's put (ADR 0539 §15, ADR 0546 §8), never by the assigner (issue 1945: nats on novox,
|
||||
// running an old-form build, got no want and the pass said nothing).
|
||||
type Holding struct {
|
||||
Rule int
|
||||
Words string
|
||||
Bus bool
|
||||
}
|
||||
|
||||
// Reason is the holding in words: the rule and its words, or for the bus bar its words alone.
|
||||
func (h Holding) Reason() string {
|
||||
if h.Bus {
|
||||
return h.Words
|
||||
}
|
||||
return fmt.Sprintf("rule %d: %s", h.Rule, h.Words)
|
||||
}
|
||||
|
||||
// BusBarWords say why the assigner writes no want moving the bus on node: it runs prev, the trunk builds fp, and only the
|
||||
// operator's put moves it (ADR 0539 §15, ADR 0546 §8). The words reach the operator, so they name no record.
|
||||
func BusBarWords(node, prev, fp string) string {
|
||||
return fmt.Sprintf("only your put moves the bus: %s runs %s, the trunk builds %s; put it with assigner_put", node, short(prev), short(fp))
|
||||
}
|
||||
|
||||
// Wants is the pass of one module m: for each of its nodes, the want by the six rules, starting from the want written
|
||||
@@ -303,6 +320,14 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
if need, wait := at.mustAsk(node, asked, hadLive, ran, running.FP, running.Composed == "", d); wait {
|
||||
if need != nil {
|
||||
out.Record = append(out.Record, *need)
|
||||
} else if prev, barred := at.busBar(asked, hadLive, running.FP); barred {
|
||||
// The bus is barred: no want moves it but the operator's put. Said, never silent (issue 1945): its holding is named,
|
||||
// and a node that runs it with no live want has it on neither list, so its condition stays raised.
|
||||
h := Holding{Bus: true, Words: BusBarWords(node, prev, at.FP)}
|
||||
out.Rules[node] = h
|
||||
if !hadLive && running.Phase != delivery.PhaseRemoved && !held {
|
||||
out.Neither[node] = h
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
@@ -466,12 +491,7 @@ func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, runnin
|
||||
if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == delivery.JudgedUnchanged {
|
||||
return nil, false
|
||||
}
|
||||
// The fingerprint the bus ran before: its want's, or with no readable want the one its have names (ADR 0539 §15 bars a new one).
|
||||
prev := before.FP
|
||||
if !had {
|
||||
prev = running
|
||||
}
|
||||
if catalogue.ProvidesBus(m.Manifest) && prev != m.FP {
|
||||
if _, barred := m.busBar(before, had, running); barred {
|
||||
return nil, true // the bus is never moved by the assigner; the operator writes its want (§15)
|
||||
}
|
||||
if !m.recorded(d) || d.Judgement == delivery.JudgedNeither || d.Judgement == delivery.JudgedUnchanged {
|
||||
@@ -483,6 +503,16 @@ func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, runnin
|
||||
return &RecordNeed{Node: node, Digest: d.Digest, Judgement: d.Judgement, Words: d.Words}, true
|
||||
}
|
||||
|
||||
// busBar says whether m is the bus and the move would take it off the fingerprint it ran before, prev: its want's, or with no
|
||||
// readable want the one its have names (ADR 0539 §15 bars a new one).
|
||||
func (m Module) busBar(before delivery.Want, had bool, running string) (prev string, barred bool) {
|
||||
prev = before.FP
|
||||
if !had {
|
||||
prev = running
|
||||
}
|
||||
return prev, catalogue.ProvidesBus(m.Manifest) && prev != m.FP
|
||||
}
|
||||
|
||||
// recorded says whether the module is of policy record: as the controller holds it beside the composition's digest, the
|
||||
// operator's own choice included (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it.
|
||||
func (m Module) recorded(d delivery.ComposedDigest) bool {
|
||||
|
||||
@@ -427,6 +427,23 @@ func TestTheBusIsNotMovedButItsUserListRevisionIsWrittenAtOnce(t *testing.T) {
|
||||
t.Errorf("a held bus did not take its user-list revision as a put: %+v", w)
|
||||
}
|
||||
m.Puts = nil
|
||||
// 1945 (the bus bar made loud): a node that runs the bus at an old-form build and holds no want for it gets none, since only
|
||||
// the operator's put moves the bus, and the pass says so: a holding naming the bar, and the node on neither list so its
|
||||
// condition stays raised. A node with a live want keeps it and is not on neither list.
|
||||
m.Composed = composedEverywhere
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: "b-1700000000-oldform", Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
out = Wants(m, n, map[string]delivery.Want{"g14": {FP: fOld, Composed: dig(fOld)}}, nil)
|
||||
if w, wrote := out.Wants["ace"]; wrote {
|
||||
t.Errorf("the bus bar wrote ace's want %+v", w)
|
||||
}
|
||||
h, on := out.Neither["ace"]
|
||||
if !on || !h.Bus || out.Rules["ace"] != h || !strings.Contains(h.Words, "only your put moves the bus") || !strings.Contains(h.Words, "ace runs b-1700000000-oldform") ||
|
||||
!strings.Contains(h.Words, "assigner_put") {
|
||||
t.Errorf("the barred bus on ace: neither %+v (%v), rules %+v", h, on, out.Rules["ace"])
|
||||
}
|
||||
if _, on := out.Neither["g14"]; on || out.Wants["g14"].FP == fNew {
|
||||
t.Errorf("g14 has a live want: neither %+v, its want %+v", out.Neither, out.Wants["g14"])
|
||||
}
|
||||
stepsOfAPut(t)
|
||||
}
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ func TestManifest(t *testing.T) {
|
||||
slices.Sort(reads)
|
||||
want := []string{"mesh-build-go.steps", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants",
|
||||
"mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "node-engine.have"}
|
||||
"mesh-delivery-indexer.modules", "mesh-delivery-indexer.placement", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "node-engine.have"}
|
||||
if !reflect.DeepEqual(reads, want) {
|
||||
t.Errorf("reads = %v, want exactly %v", reads, want)
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
"mesh-delivery-indexer.candidates",
|
||||
"mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-indexer.modules",
|
||||
"mesh-delivery-indexer.placement",
|
||||
"mesh-delivery-teller.answers",
|
||||
"mesh-delivery-watchman.presence",
|
||||
"node-engine.have"
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
@@ -31,7 +32,8 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q
|
||||
Description: "Every build key that waits (building or queued), oldest first: its module, fingerprint, step words and since when, and each ask it " +
|
||||
"waits on by language with its id, when it was put, its asking (1 for the first; the builder asks an ask again when it can no longer be " +
|
||||
"answered, at most 3 times) and what its build module's steps hold of it (none, or its state on its node since its start). The " +
|
||||
"seat workers' delivery counts are not readable by the builder. With module, that module's keys only. Read-only.",
|
||||
"seat workers' delivery counts are not readable by the builder. dead_letters says whether the builder hears the asks the build seats give up " +
|
||||
"on (issue 734), and why not: heard, a given-up ask is asked again at once; deaf, at its check bound. With module, that module's keys only. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "only this module's keys"}}},
|
||||
Run: func(args map[string]any) (any, error) { return r.Stuck(str(args, "module")) }},
|
||||
@@ -166,7 +168,8 @@ func main() {
|
||||
steps[bm] = read(stdio.State(bm + ".steps"))
|
||||
}
|
||||
reasks := stdio.State(build.ReAsksState)
|
||||
r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps}
|
||||
r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps,
|
||||
Hearing: read(stdio.State(buildrun.HearingState))}
|
||||
put := func(key string, value []byte) error {
|
||||
_, err := reasks.Put(key, json.RawMessage(value))
|
||||
return err
|
||||
|
||||
@@ -149,6 +149,18 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// Each build module's part of a build is an ask on its build seat's stream, pulled by a build node with a free slot
|
||||
// (novox/hq ADR 0525): the builder chooses no node, so it reads neither presence nor have and holds no capacity.
|
||||
asks := buildrun.JetStreamAsks{JS: js}
|
||||
// The asks the build seats gave up on (issue 734): heard from DEAD_LETTERS, each ends the wait for its step at once,
|
||||
// and the lane asks its part again (Builder.waitOnce) instead of waiting for the check bound.
|
||||
// Heard in the background, and again at a reconnect or an ask while it is not; whether it is heard is kept in the
|
||||
// state hearing, which builder_stuck says (dead_letters).
|
||||
dead := &buildrun.DeadLetters{}
|
||||
// The state may not be on the bus yet when a delivery started the builder (the grants part creates it, issue 1790): it
|
||||
// is bound when the bus says it was created, and the latest hearing is written then.
|
||||
hearingKV := process.AwaitBucket(ctx, js, conn, delivery.BuilderModule+"_"+buildrun.HearingState, "builder_stuck cannot say whether DEAD_LETTERS is heard", logger.Printf)
|
||||
hearing := buildrun.NewDeadLetterHearing(ctx, buildrun.JetStreamDeadLetters{JS: js}, delivery.AskLanguages, dead, logger.Printf,
|
||||
buildrun.SayHearing(ctx, hearingKV, logger.Printf))
|
||||
defer hearing.Stop()
|
||||
conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again() })
|
||||
// The builder asks the controller, as its own account, whether the artifact store still holds a done key's blobs
|
||||
// (the seat verb held, novox/hq issue 1937) before it leaves the key done.
|
||||
account, err := process.ReadAccount(path)
|
||||
@@ -162,7 +174,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
defer calls.Close()
|
||||
b := &build.Builder{
|
||||
Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
|
||||
Asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: bound},
|
||||
Asks: buildrun.Asker{Publisher: asks, Waiter: buildrun.LostAskWaiter{Inner: asks, Dead: dead, Hearing: hearing}, Bound: bound},
|
||||
Held: heldOnTheBus{js},
|
||||
// An ask that can no longer be answered is asked again (issue 1945), read at the step bound: a context's bound.
|
||||
Gone: heldOnTheBus{js},
|
||||
@@ -280,24 +292,25 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
// A person asks a key's build again with builder_ask_again, which writes the reasks state (issue 1945): watched, as
|
||||
// every other input. The builder asks again by itself without it.
|
||||
if kv := process.OptionalBucket(ctx, js, build.ReAsksBucket, "builder_ask_again is not heard until the builder starts again; an ask that can no longer be answered is still asked again by itself", logger.Printf); kv != nil {
|
||||
// A state not on the bus yet at this start is watched once it appears (issue 1790).
|
||||
process.AwaitBucket(ctx, js, conn, build.ReAsksBucket, "builder_ask_again is not heard; an ask that can no longer be answered is still asked again by itself", logger.Printf).Ready(func(kv jetstream.KeyValue) {
|
||||
eachReAsk := func(e jetstream.KeyValueEntry) { reAsked(ctx, b, kv, e, logger.Printf) }
|
||||
if rw, err := settle(ctx, kv, eachReAsk); err != nil {
|
||||
process.NotWatched(logger.Printf, build.ReAsksBucket, err, "builder_ask_again is not heard until the builder starts again")
|
||||
} else {
|
||||
go func() { errs <- follow(ctx, rw, eachReAsk) }()
|
||||
}
|
||||
}
|
||||
})
|
||||
// A person reads a build seat's stream with builder_seat_queue, which writes the queues state (issue 1945 note 62): the
|
||||
// builder alone may read the stream, so it answers there.
|
||||
if kv := process.OptionalBucket(ctx, js, build.QueuesBucket, "builder_seat_queue is not answered until the builder starts again", logger.Printf); kv != nil {
|
||||
process.AwaitBucket(ctx, js, conn, build.QueuesBucket, "builder_seat_queue is not answered", logger.Printf).Ready(func(kv jetstream.KeyValue) {
|
||||
eachRead := func(e jetstream.KeyValueEntry) { queueRead(ctx, heldOnTheBus{js}, kv, e, logger.Printf) }
|
||||
if qw, err := settle(ctx, kv, eachRead); err != nil {
|
||||
process.NotWatched(logger.Printf, build.QueuesBucket, err, "builder_seat_queue is not answered until the builder starts again")
|
||||
} else {
|
||||
go func() { errs <- follow(ctx, qw, eachRead) }()
|
||||
}
|
||||
}
|
||||
})
|
||||
logger.Printf("started on %s", cred.Node)
|
||||
go func() { errs <- follow(ctx, w, each) }()
|
||||
select {
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-builder
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142434-65ee43f07583
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1 h1:fzOkrz9CSp3mEjeV7NfZx8yoivI1sD1AOtIPtyyF8RI=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142434-65ee43f07583 h1:ikD/ggf9/5k23sxJOX1PFcGzb4M3+QeC66AT0MyQ9Zs=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142434-65ee43f07583/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// **An ask that can no longer be answered is asked again** (novox/hq issue 1945). A build seat's stream gives an ask up
|
||||
@@ -20,7 +21,8 @@ import (
|
||||
// restart take over the same id and wait again. The builder notices it at the check bound (CheckAfter, the step bound),
|
||||
// a context's bound and no timer of its own: it asks Gone, which reads the bus, and asks such an ask again under a new
|
||||
// id, written into the key, said in its log and as the event build.asked-again, at most Askings times; then the key is
|
||||
// failed with why. A person asks the same by hand with the tool builder_ask_again (AskAgain).
|
||||
// failed with why. An ask DEAD_LETTERS says the seat gave up (buildrun.LostAskWaiter, issue 734) is asked again at once.
|
||||
// A person asks the same by hand with the tool builder_ask_again (AskAgain).
|
||||
|
||||
// DefaultAskings is how many times a build module's part of a build is asked, the first included, when Builder.Askings is 0.
|
||||
const DefaultAskings = 3
|
||||
@@ -315,6 +317,11 @@ func (b *Builder) waitOnce(ctx context.Context, ask delivery.StepAsk, again <-ch
|
||||
defer stop()
|
||||
}
|
||||
step, err := b.Asks.Wait(wait, ask)
|
||||
if err == nil && buildrun.IsLost(step) {
|
||||
// The build seat gave the ask up (issue 734): it can no longer be answered, so it is asked again now, with the
|
||||
// words of why, not at the check bound.
|
||||
return step, step.Reason, errAgain
|
||||
}
|
||||
if err == nil || ctx.Err() != nil {
|
||||
return step, "", err
|
||||
}
|
||||
|
||||
@@ -690,12 +690,14 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||
step string
|
||||
byHand bool
|
||||
always bool // Gone says every ask cannot be answered, not only the first
|
||||
dead bool // DEAD_LETTERS keeps the first ask: the seat gave it up (issue 734)
|
||||
puts int
|
||||
done bool
|
||||
asking int // the asking the event says, 0 when none is said
|
||||
}{
|
||||
{name: "given up by its stream: asked again under a new id and done from it", puts: 2, done: true, asking: 2},
|
||||
{name: "asked again by hand: the same, with no check bound", byHand: true, puts: 2, done: true, asking: 2},
|
||||
{name: "given up into DEAD_LETTERS: its wait ends failed with the words and it is asked again at once", dead: true, puts: 2, done: true, asking: 2},
|
||||
{name: "never answerable: failed after its askings", askings: 2, always: true, puts: 2},
|
||||
{name: "taken over at its last asking: failed, not asked again", askings: 3, always: true, step: "asked mesh-build-go again (asking 3): x", puts: 0},
|
||||
} {
|
||||
@@ -714,7 +716,11 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||
r.st.vals[r.key] = data
|
||||
}
|
||||
gone := &goneAsks{}
|
||||
if !c.byHand {
|
||||
dead := &buildrun.DeadLetters{}
|
||||
if c.dead {
|
||||
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: buildrun.LostAskWaiter{Inner: r.asks, Dead: dead}, Bound: 2 * time.Second}
|
||||
}
|
||||
if !c.byHand && !c.dead {
|
||||
gone.why = func(a delivery.StepAsk) string {
|
||||
if c.always || a.ID == first || first == "" {
|
||||
return "it waits first in line while a node took a later ask"
|
||||
@@ -724,6 +730,10 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||
r.b.Gone, r.b.CheckAfter = gone, 5*time.Millisecond
|
||||
}
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
if c.dead {
|
||||
dead.Heard(buildrun.GivenUp{AskID: waitForAsks(t, r, 1)[0].ID, Stream: "SEAT_NODE_BUILD_GO", Deliveries: 5,
|
||||
At: time.Date(2026, 10, 26, 12, 26, 40, 0, time.UTC)})
|
||||
}
|
||||
if c.byHand {
|
||||
waitForAsks(t, r, 1)
|
||||
for deadline := time.Now().Add(2 * time.Second); len(r.b.AskAgain(r.key, "stuck since 19:06")) == 0; time.Sleep(time.Millisecond) {
|
||||
@@ -757,13 +767,16 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
// Note 62: the old message of an ask asked again because its stream gave it up is dropped from the stream.
|
||||
if gone.mu.Lock(); c.done && !c.byHand && (len(gone.dropped) != 1 || gone.dropped[0] != first) {
|
||||
if gone.mu.Lock(); c.done && !c.byHand && !c.dead && (len(gone.dropped) != 1 || gone.dropped[0] != first) {
|
||||
t.Errorf("dropped %v from the stream, want the old ask %s", gone.dropped, first)
|
||||
}
|
||||
gone.mu.Unlock()
|
||||
if c.asking > 0 && (len(said) != 1 || said[0].Asking != c.asking || said[0].New == said[0].Old || said[0].Key != r.key || said[0].Why == "") {
|
||||
t.Errorf("said %+v, want once at asking %d", said, c.asking)
|
||||
}
|
||||
if c.dead && (len(said) != 1 || !strings.Contains(said[0].Why, buildrun.LostWords+" after 5 deliveries at 2026-10-26T12:26:40Z")) {
|
||||
t.Errorf("said %+v, want the ask asked again saying the seat gave it up", said)
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("what the bus says of an ask", func(t *testing.T) { // 1945
|
||||
@@ -836,12 +849,17 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||
running, _ := json.Marshal(delivery.Step{V: delivery.Version, ID: "mvo53jcm70b5f8", Tool: "delivery ask", Node: "g14", State: delivery.StepRunning, Started: at})
|
||||
_, reask, _ := NewReAsk("m", fp1, "stuck", "shanks.lead", at)
|
||||
r := StuckReader{Builds: state(map[string]string{key: string(building), done: string(finished)}), ReAsks: state(map[string]string{key: string(reask)}),
|
||||
Steps: map[string]StateRead{buildrun.GoBuildModule: state(map[string]string{}), ImageBuildModule: state(map[string]string{"g14.mvo53jcm70b5f8": string(running)})},
|
||||
Now: func() time.Time { return at.Add(time.Hour) }}
|
||||
Steps: map[string]StateRead{buildrun.GoBuildModule: state(map[string]string{}), ImageBuildModule: state(map[string]string{"g14.mvo53jcm70b5f8": string(running)})},
|
||||
Hearing: state(map[string]string{buildrun.HearingKey: `{"heard":false,"streams":["SEAT_NODE_BUILD_GO"],"why":"DEAD_LETTERS is not heard: SEAT_NODE_BUILD_GO: permissions violation","at":"2026-10-26T13:00:00Z"}`}),
|
||||
Now: func() time.Time { return at.Add(time.Hour) }}
|
||||
out, err := r.Stuck("")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Issue 734, review round 1: a builder that does not hear DEAD_LETTERS says so, with why.
|
||||
if dl := out["dead_letters"].(map[string]any); dl["heard"] != false || !strings.Contains(dl["why"].(string), "permissions violation") {
|
||||
t.Errorf("dead_letters = %v, want not heard, saying why", dl)
|
||||
}
|
||||
stuck := out["stuck"].([]Stuck)
|
||||
if len(stuck) != 1 || stuck[0].Key != key || stuck[0].Age != "1h0m0s" || stuck[0].NoneStep != 1 || stuck[0].ReAsk == nil || stuck[0].ReAsk.Why != "stuck" {
|
||||
t.Fatalf("stuck = %+v, want the building key alone, an hour old, one ask with no step, with its ask again", stuck)
|
||||
|
||||
@@ -106,7 +106,9 @@ type StuckReader struct {
|
||||
ReAsks StateRead
|
||||
// Steps are the build modules' steps states, by build module.
|
||||
Steps map[string]StateRead
|
||||
Now func() time.Time
|
||||
// Hearing is the builder's state hearing: whether it hears DEAD_LETTERS (issue 734); without it none is said.
|
||||
Hearing StateRead
|
||||
Now func() time.Time
|
||||
// Logf, when set, gets the one line a call logs: the values it read and the time it took.
|
||||
Logf func(format string, args ...any)
|
||||
}
|
||||
@@ -149,6 +151,20 @@ func parallel(n int, read func(i int) error) error {
|
||||
}
|
||||
|
||||
// Stuck is every key building or queued with its asks, oldest first; with module, that module's only.
|
||||
// deadLetters says whether the builder hears the asks the build seats give up on (issue 734), and why not: deaf, a lane
|
||||
// whose ask is given up waits for its check bound (buildrun.HearingHealth over the kept hearing).
|
||||
func (r StuckReader) deadLetters() map[string]any {
|
||||
if r.Hearing.Get == nil {
|
||||
return map[string]any{"heard": false, "why": "the builder's state hearing is not readable here"}
|
||||
}
|
||||
raw, _, err := r.Hearing.Get(buildrun.HearingKey)
|
||||
if err != nil {
|
||||
return map[string]any{"heard": false, "why": "the builder's state hearing cannot be read: " + err.Error()}
|
||||
}
|
||||
a := buildrun.HearingHealth(raw)
|
||||
return map[string]any{"heard": a.Healthy, "why": a.Why}
|
||||
}
|
||||
|
||||
func (r StuckReader) Stuck(module string) (map[string]any, error) {
|
||||
started := time.Now()
|
||||
reads := 0
|
||||
@@ -268,6 +284,7 @@ func (r StuckReader) Stuck(module string) (map[string]any, error) {
|
||||
})
|
||||
answer := map[string]any{"stuck": out, "count": len(out),
|
||||
"delivery_counts": "not readable by the builder: the build seats' workers' information is granted to the delivery view alone (its backlog); an ask given up on is told by the builder at its check bound and asked again"}
|
||||
answer["dead_letters"] = r.deadLetters()
|
||||
if cut {
|
||||
answer["cut"], answer["note"] = true, fmt.Sprintf("read %d of %d build keys in key order and stopped; give a module to read its keys alone", MaxReads, total)
|
||||
}
|
||||
|
||||
@@ -87,7 +87,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
if m.Module != "mesh-delivery-builder" {
|
||||
t.Errorf("module = %q", m.Module)
|
||||
}
|
||||
if want := []state{{Name: "builds", History: 5}, {Name: "instances", History: 1}, {Name: "queues", History: 1}, {Name: "reasks", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
|
||||
if want := []state{{Name: "builds", History: 5}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "queues", History: 1}, {Name: "reasks", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
}
|
||||
// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
],
|
||||
"state": [
|
||||
{ "name": "builds", "history": 5 },
|
||||
{ "name": "hearing", "history": 1 },
|
||||
{ "name": "instances", "history": 1 },
|
||||
{ "name": "queues", "history": 1 },
|
||||
{ "name": "reasks", "history": 1 }
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
|
||||
"mesh-delivery-checker/internal/check"
|
||||
)
|
||||
@@ -53,10 +54,36 @@ func read(s stdio.KeptState) check.StateRead {
|
||||
}}
|
||||
}
|
||||
|
||||
// runsOf says which commit the mesh runs of a module from the indexer's modules state, as the checker's own record of it does
|
||||
// (the commit the module's record was computed at); "" when the record is not readable.
|
||||
type runsOf struct{ modules check.StateRead }
|
||||
|
||||
func (r runsOf) CommitOf(module string) string {
|
||||
if r.modules.Get == nil {
|
||||
return ""
|
||||
}
|
||||
raw, err := r.modules.Get(module)
|
||||
if err != nil || len(raw) == 0 {
|
||||
return ""
|
||||
}
|
||||
m, err := delivery.DecodeModule(raw)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return m.Commit
|
||||
}
|
||||
|
||||
// stateStore is a state the tools write: the tries.
|
||||
type stateStore struct {
|
||||
check.StateRead
|
||||
create func(key string, value []byte) error
|
||||
entry func(key string) (json.RawMessage, uint64, error)
|
||||
update func(key string, value []byte, revision uint64) error
|
||||
}
|
||||
|
||||
func (s stateStore) Entry(key string) (json.RawMessage, uint64, error) { return s.entry(key) }
|
||||
func (s stateStore) Update(key string, value []byte, revision uint64) error {
|
||||
return s.update(key, value, revision)
|
||||
}
|
||||
|
||||
func (s stateStore) Keys() ([]string, error) { return s.StateRead.Keys() }
|
||||
@@ -66,11 +93,12 @@ func (s stateStore) Create(key string, value []byte) error { return s.create(k
|
||||
// Trying is what the try's tools work on: the tries, and the gate's trunk heads.
|
||||
type Trying struct {
|
||||
Tries check.TryStore
|
||||
Ender check.TryEnder
|
||||
Trunks func() (map[string]string, error)
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// TryTools are the try's two tools (ADR 0501 §5): ask one, read one. mesh-controller's try calls them.
|
||||
// TryTools are the try's tools (ADR 0501 §5): ask one, read one, and end one by hand (issue 734). mesh-controller's try calls them.
|
||||
func TryTools(tr Trying) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return s }
|
||||
return []stdio.Tool{
|
||||
@@ -78,7 +106,8 @@ func TryTools(tr Trying) []stdio.Tool {
|
||||
Description: "Try a pushed head before its pull request is opened: the checker judges it as it judges a pull request's head, merged onto its trunk's head, " +
|
||||
"its unit and integration levels through the build modules, and owes the forge nothing. Answers the try's id at once; " + check.TryReadTool + " reads " +
|
||||
"where it stands. A try of the same head that waits or runs is the answer to a second ask; at most 4 wait or run at once, mesh-wide. A repository " +
|
||||
"the checker checks no level of (one whose trunk the gate does not name) is refused.",
|
||||
"the checker checks no level of (one whose trunk the gate does not name) is refused. A try whose ask no build node took within the step " +
|
||||
"bound ends as error; its ask stays in the build seat's stream and may still run later, judged by nobody (issue 734).",
|
||||
Input: map[string]any{"type": "object", "required": []string{"repository", "head"}, "properties": map[string]any{
|
||||
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
|
||||
"head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits"},
|
||||
@@ -115,6 +144,28 @@ func TryTools(tr Trying) []stdio.Tool {
|
||||
}
|
||||
return check.ReadTry(tr.Tries, str(args, "id"))
|
||||
}},
|
||||
{Name: check.TryEndTool,
|
||||
Description: "End a try that waits or runs, as error, by hand: say why (kept in what it says). Its place among the 4 is free at once, " +
|
||||
"the checker stops waiting for its asks, and the next " + check.TryTool + " of its head asks anew with a new id. For a try whose ask " +
|
||||
"was lost (no step runs for it in checker_status); the checker ends one by itself when the build seat gives its ask up, or no node " +
|
||||
"takes it within the step bound. A try that ended already is refused.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"id", "why"}, "properties": map[string]any{
|
||||
"id": map[string]any{"type": "string", "description": "the try's id, try-<digits>"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is ended, in words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
if tr.Ender == nil {
|
||||
return nil, fmt.Errorf("the tries state cannot be written here")
|
||||
}
|
||||
now := time.Now
|
||||
if tr.Now != nil {
|
||||
now = tr.Now
|
||||
}
|
||||
t, err := check.EndTry(tr.Ender, str(args, "id"), str(args, "why"), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"id": t.ID, "state": t.State, "said": t.Said}, nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -174,7 +225,10 @@ func Tools(r check.Reader) []stdio.Tool {
|
||||
"and the candidates whose check is owed by kind (waiting_by_kind: new, nodes, and trunk, a re-judge pulled after the other two). A key a " +
|
||||
"build module left running (started past the step bound, or its build module not running on its node) is listed apart under " +
|
||||
"left_running with why, and not counted. Nothing holds the checker and it has no capacity of its own: a check is an ask on the " +
|
||||
"build seats' streams, whose backlog the delivery view shows (novox/hq ADR 0525). Read-only.",
|
||||
"build seats' streams, whose backlog the delivery view shows (novox/hq ADR 0525). held_by_turn names the candidates whose check an approved " +
|
||||
"merge's turn holds off (novox/hq ADR 0548 §3), with the repository and the merge that waits; the gate's turns tool reads every turn and its " +
|
||||
"hand-now tool steers one. dead_letters says whether the checker hears the asks the build seats give up on (issue 734), and why not: " +
|
||||
"deaf, a try or check whose ask is given up waits until its bound. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) { return r.Status() }},
|
||||
}
|
||||
@@ -193,7 +247,9 @@ func main() {
|
||||
r := check.Reader{VerdictState: read(stdio.State("verdicts")), KeptState: read(stdio.State("kept")), CandidateState: read(stdio.State("mesh-delivery-indexer.candidates")),
|
||||
PreviewState: read(stdio.State("previews")),
|
||||
StepStates: map[string]check.StateRead{"mesh-build-go": read(stdio.State("mesh-build-go.steps")), "mesh-build-node": read(stdio.State("mesh-build-node.steps"))},
|
||||
HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound()}
|
||||
HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound(),
|
||||
TurnState: read(stdio.State("mesh-delivery-gate.turns")), HearingState: read(stdio.State(buildrun.HearingState))}
|
||||
r.Runs = runsOf{read(stdio.State("mesh-delivery-indexer.modules"))} // the node-engine at the commit the mesh runs, as the checker reads it
|
||||
tries := stdio.State(check.TriesState)
|
||||
source := stdio.State("mesh-delivery-gate.source")
|
||||
trunks := func() (map[string]string, error) {
|
||||
@@ -205,13 +261,21 @@ func main() {
|
||||
return s.Heads, err
|
||||
}
|
||||
r.Trunks = trunks
|
||||
trying := Trying{
|
||||
Tries: stateStore{StateRead: read(tries), create: func(key string, value []byte) error {
|
||||
_, err := tries.Create(key, json.RawMessage(value))
|
||||
return err
|
||||
}},
|
||||
Trunks: trunks,
|
||||
}
|
||||
store := stateStore{StateRead: read(tries), create: func(key string, value []byte) error {
|
||||
_, err := tries.Create(key, json.RawMessage(value))
|
||||
return err
|
||||
}, entry: func(key string) (json.RawMessage, uint64, error) {
|
||||
reads.Add(1)
|
||||
e, err := tries.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return e.Value, e.Revision, nil
|
||||
}, update: func(key string, value []byte, revision uint64) error {
|
||||
_, err := tries.Update(key, json.RawMessage(value), revision)
|
||||
return err
|
||||
}}
|
||||
trying := Trying{Tries: store, Ender: store, Trunks: trunks}
|
||||
if err := stdio.Serve("", logged(append(Tools(r), TryTools(trying)...))); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
|
||||
@@ -53,13 +53,19 @@ func (m *memStore) Create(key string, value []byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *memStore) Entry(key string) (json.RawMessage, uint64, error) { return m.vals[key], 1, nil }
|
||||
func (m *memStore) Update(key string, value []byte, revision uint64) error {
|
||||
m.vals[key] = value
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed interface with mesh-controller's try
|
||||
store := &memStore{vals: map[string]json.RawMessage{}}
|
||||
tools := TryTools(Trying{Tries: store, Now: func() time.Time { return time.Unix(1, 5) },
|
||||
tools := TryTools(Trying{Tries: store, Ender: store, Now: func() time.Time { return time.Unix(1, 5) },
|
||||
Trunks: func() (map[string]string, error) {
|
||||
return map[string]string{"novox/mesh-delivery": strings.Repeat("a", 40)}, nil
|
||||
}})
|
||||
if len(tools) != 2 || tools[0].Name != "checker_try" || tools[1].Name != "checker_try_read" {
|
||||
if len(tools) != 3 || tools[0].Name != "checker_try" || tools[1].Name != "checker_try_read" || tools[2].Name != "checker_try_end" {
|
||||
t.Fatalf("tools = %+v", tools)
|
||||
}
|
||||
out, err := tools[0].Run(map[string]any{"repository": "novox/mesh-delivery", "head": strings.Repeat("1", 40), "trunk": "main"})
|
||||
@@ -74,6 +80,14 @@ func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed i
|
||||
if err != nil || !strings.Contains(string(raw.(json.RawMessage)), `"state":"queued"`) {
|
||||
t.Errorf("read = %s, %v", raw, err)
|
||||
}
|
||||
// Issue 734: a try whose ask was lost is ended by hand with why, and its head is asked anew.
|
||||
ended, err := tools[2].Run(map[string]any{"id": id, "why": "its ask was lost"})
|
||||
if err != nil || ended.(map[string]any)["state"] != "error" {
|
||||
t.Errorf("end = %v, %v", ended, err)
|
||||
}
|
||||
if _, err := tools[2].Run(map[string]any{"id": id, "why": "again"}); err == nil {
|
||||
t.Error("an ended try was ended again")
|
||||
}
|
||||
if _, err := tools[0].Run(map[string]any{"repository": "novox/hq", "head": strings.Repeat("1", 40)}); err == nil ||
|
||||
err.Error() != "no level of novox/hq is checked by the checker" {
|
||||
t.Errorf("hq: %v", err)
|
||||
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
|
||||
"mesh-delivery-checker/internal/check"
|
||||
)
|
||||
@@ -79,6 +80,8 @@ type fakeBuildModules struct {
|
||||
js jetstream.JetStream
|
||||
ctx context.Context
|
||||
conflict atomic.Bool
|
||||
// hold is a commit whose fetch never answers: its ask stays taken, as one lost in a bus reload (issue 734).
|
||||
hold atomic.Value
|
||||
mu sync.Mutex
|
||||
calls map[string]int // by `<module>.<tool>`
|
||||
testTool *testing.T
|
||||
@@ -98,6 +101,9 @@ func (f *fakeBuildModules) result(t *testing.T, tool string, args map[string]any
|
||||
var v any
|
||||
switch {
|
||||
case strings.HasSuffix(tool, "_fetch"):
|
||||
if h, _ := f.hold.Load().(string); h != "" && args["commit"] == h {
|
||||
<-f.ctx.Done()
|
||||
}
|
||||
name := "go-fetch-clean"
|
||||
if f.conflict.Load() {
|
||||
name = "go-fetch-conflict"
|
||||
@@ -298,6 +304,10 @@ func startBed(t *testing.T) *bed {
|
||||
}
|
||||
b.tools[module] = kv
|
||||
}
|
||||
// What the controller keeps of the messages a consumer gave up on (issue 734).
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buildrun.DeadLettersStream, Subjects: []string{"mesh.events.dead.>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The checker's stream of events, which the indexer and the forge bridge hear.
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "CHECKS", Subjects: []string{"mesh.stream.mesh-delivery-checker.event.>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -305,7 +315,8 @@ func startBed(t *testing.T) *bed {
|
||||
for name, history := range map[string]int{delivery.CheckerVerdictsBucket: delivery.CheckerVerdictsHistory, delivery.CheckerKeptBucket: delivery.CheckerKeptHistory,
|
||||
delivery.CheckerOwedBucket: delivery.CheckerOwedHistory, delivery.PreviewsBucket: delivery.PreviewsHistory, delivery.CandidatesBucket: delivery.CandidatesHistory,
|
||||
delivery.ModulesBucket: delivery.ModulesHistory, delivery.ManifestsBucket: delivery.ManifestsHistory, delivery.SourceBucket: delivery.SourceHistory,
|
||||
delivery.PresenceBucket: delivery.PresenceHistory, delivery.HaveBucket: delivery.HaveHistory, check.TriesBucket: 1} {
|
||||
delivery.PresenceBucket: delivery.PresenceHistory, delivery.HaveBucket: delivery.HaveHistory, check.TriesBucket: 1,
|
||||
delivery.TurnsBucket: delivery.TurnsHistory} {
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: uint8(history)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -507,9 +518,12 @@ func TestCheckerJudgesACandidateAndOwesItsStatus(t *testing.T) { // N6/AC5, 1300
|
||||
}
|
||||
}
|
||||
|
||||
// TestACheckOfEqualInputsRunsNothing: a second candidate (another head, the same digests) is checked with every package
|
||||
// taken from the first's kept verdicts: no test step runs and no second kept key appears.
|
||||
func TestACheckOfEqualInputsRunsNothing(t *testing.T) { // 1287/AC5, 1288/AC6, 1929/AC2
|
||||
// TestACheckOrACarryOfEqualInputsRunsNothing: a second candidate (another head, the same digests) is checked with every
|
||||
// package taken from the first's kept verdicts: no test step runs and no second kept key appears. Then, through the real bus
|
||||
// (novox/hq ADR 0548 §3, §5): a third candidate is not claimed while the gate's merge turn on its repository stands, and is
|
||||
// claimed when the turn's key is deleted; and a move of the trunk carries the first verdict on the recheck stream by its
|
||||
// unchanged digests, running no test, the verdict naming the trunk it was carried from.
|
||||
func TestACheckOrACarryOfEqualInputsRunsNothing(t *testing.T) { // 1287/AC5, 1288/AC6, 1929/AC2, ADR 0548 §3, §5
|
||||
b := startBed(t)
|
||||
first := b.candidate(t, "feat/x", pull(repo, 7, head1))
|
||||
b.start(t, nil)
|
||||
@@ -526,6 +540,42 @@ func TestACheckOfEqualInputsRunsNothing(t *testing.T) { // 1287/AC5, 1288/AC6, 1
|
||||
if got := keptCount(t, b); got != kept {
|
||||
t.Errorf("%d kept keys, was %d", got, kept)
|
||||
}
|
||||
|
||||
// §3: the gate's turn on the repository holds a new check off; its delete claims it.
|
||||
at := time.Now().UTC()
|
||||
turn, err := delivery.EncodeTurn(delivery.Turn{Repository: repo, Waiting: []delivery.TurnWait{{Group: "fix/other", Heads: "0123456789ab", Approved: at, Since: at}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tk, _ := delivery.TurnsKey(repo)
|
||||
if _, err := b.buckets[delivery.TurnsBucket].Put(b.ctx, tk, turn); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(300 * time.Millisecond) // the checker hears the turn
|
||||
third := b.candidate(t, "feat/z", pull(repo, 9, "3333333333333333333333333333333333333333"))
|
||||
time.Sleep(time.Second)
|
||||
if _, err := b.buckets[delivery.CheckerVerdictsBucket].Get(b.ctx, third); !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
t.Fatalf("verdicts.%s exists (%v) while a merge turn holds its repository", third, err)
|
||||
}
|
||||
if err := b.buckets[delivery.TurnsBucket].Delete(b.ctx, tk); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.verdict(t, third, ended)
|
||||
|
||||
// §5: the trunk moves; the first verdict is carried by its digests, nothing runs.
|
||||
tests = b.fake.count("mesh-build-go.build_go_test")
|
||||
const moved = "abababababababababababababababababababab"
|
||||
b.setTrunk(t, map[string]string{repo: moved})
|
||||
c := b.verdict(t, first, func(v delivery.Verdict) bool { return ended(v) && v.Trunk[repo] == moved })
|
||||
if c.State != delivery.VerdictPass || c.CarriedFrom[repo] != trunk || c.Levels[delivery.LevelUnit].Run != 0 || c.Levels[delivery.LevelIntegration].Run != 0 {
|
||||
t.Fatalf("the carried verdict = %+v, want a pass carried from %s with no test run", c, trunk[:12])
|
||||
}
|
||||
if got := b.fake.count("mesh-build-go.build_go_test"); got != tests {
|
||||
t.Errorf("build_go_test called %d times on the carry, was %d: a carry of unchanged digests runs nothing", got, tests)
|
||||
}
|
||||
if o := b.owed(t, 7, head1, "success"); !strings.HasPrefix(o.Text, "pass 111111111111 on abababababab | carried from aaaaaaaaaaaa") {
|
||||
t.Errorf("the carried status says %q", o.Text)
|
||||
}
|
||||
}
|
||||
|
||||
func keptCount(t *testing.T, b *bed) int {
|
||||
@@ -676,6 +726,49 @@ func TestATryIsJudgedAndItsEndWrittenWithNoStatusOwed(t *testing.T) {
|
||||
t.Errorf("a try wrote %s: %v", name, keys)
|
||||
}
|
||||
}
|
||||
// Issue 734: a try whose ask the build seat gives up on (the controller keeps its copy in DEAD_LETTERS with the ask's
|
||||
// message id) ends as error saying so, and its head asked again is a new try.
|
||||
b.fake.hold.Store(head2)
|
||||
lost, err := check.AskTry(store, map[string]string{repo: trunk}, check.TryAsk{Repository: repo, Head: head2}, time.Now())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
steps, err := b.js.KeyValue(b.ctx, delivery.StepsBucket("mesh-build-go"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
askID := ""
|
||||
for deadline := time.Now().Add(20 * time.Second); askID == "" && time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
keys, _ := steps.Keys(b.ctx)
|
||||
for _, k := range keys {
|
||||
if e, err := steps.Get(b.ctx, k); err == nil && strings.Contains(string(e.Value()), `"running"`) {
|
||||
askID = strings.TrimPrefix(k, "n1.")
|
||||
}
|
||||
}
|
||||
}
|
||||
if askID == "" {
|
||||
t.Fatal("the held try's ask was never taken")
|
||||
}
|
||||
dead := &nats.Msg{Subject: "mesh.events.dead.SEAT_NODE_BUILD_GO.SEAT_NODE_BUILD_GO_CHECK_worker", Header: nats.Header{}}
|
||||
dead.Header.Set(buildrun.DeadHeaderMsgID, askID)
|
||||
dead.Header.Set(buildrun.DeadHeaderDeliveries, "5")
|
||||
dead.Header.Set(buildrun.DeadHeaderStream, "SEAT_NODE_BUILD_GO")
|
||||
dead.Header.Set(buildrun.DeadHeaderGaveUp, time.Now().UTC().Format(time.RFC3339Nano))
|
||||
if _, err := b.js.PublishMsg(b.ctx, dead); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
raw, _ := store.Get(lost)
|
||||
if last, err = check.DecodeTry(raw); err == nil && last.Over() {
|
||||
break
|
||||
}
|
||||
}
|
||||
if last.State != check.TryError || !strings.Contains(last.Said, buildrun.LostWords+" after 5 deliveries") {
|
||||
t.Fatalf("the try whose ask was given up = %+v, want error saying so", last)
|
||||
}
|
||||
if again, err := check.AskTry(store, map[string]string{repo: trunk}, check.TryAsk{Repository: repo, Head: head2}, time.Now()); err != nil || again == lost {
|
||||
t.Errorf("its head asked again answered %q, %v, want a new try", again, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTheCheckerSaysOwedReadableOnTheBus: started, the checker says on its own states subject what it keeps, owed among
|
||||
@@ -702,7 +795,7 @@ func TestTheCheckerSaysOwedReadableOnTheBus(t *testing.T) { // R2-1
|
||||
break
|
||||
}
|
||||
if said.Module != "mesh-delivery-checker" || said.Node != "n1" || !said.Keeps("owed") || !said.Readable("owed") ||
|
||||
!said.Readable("previews") || !said.Readable("verdicts") || !said.Readable("kept") || said.Readable("tries") || len(said.States) != 5 {
|
||||
t.Fatalf("the checker said %+v, want its five states with kept, owed, previews and verdicts readable (1929/AC4; 1887 A5b)", said)
|
||||
!said.Readable("previews") || !said.Readable("verdicts") || !said.Readable("kept") || !said.Readable("tries") || said.Readable("hearing") || len(said.States) != 6 {
|
||||
t.Fatalf("the checker said %+v, want its six states with kept, owed, previews, tries and verdicts readable, tries by the delivery view (1929/AC4; 1887 A5b; build_usage)", said)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,8 +37,8 @@ func TestTheCheckerAnnouncesEveryStateItKeepsAndOwedReadable(t *testing.T) { //
|
||||
if !equal(manifest, announced) {
|
||||
t.Errorf("announced %v, the manifest keeps %v", announced, manifest)
|
||||
}
|
||||
// kept is read by the Go build module for a check ask (issue 1887 design limit 1).
|
||||
if want := []string{"kept", "owed", "previews", "verdicts"}; !equal(readable, want) {
|
||||
// kept is read by the Go build module for a check ask (issue 1887 design limit 1); tries by the delivery view (build_usage).
|
||||
if want := []string{"kept", "owed", "previews", "tries", "verdicts"}; !equal(readable, want) {
|
||||
t.Errorf("readable %v, want %v", readable, want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-checker/internal/check"
|
||||
)
|
||||
|
||||
// lazyTurns is the gate's merge turns as the checker reads them (novox/hq ADR 0548 §3), opened when the gate's bucket is
|
||||
// there: before each claim it is opened again until it is, so a checker that started before the gate made the bucket holds
|
||||
// checks from the gate's first turn on, with no timer and no restart. Until then it holds nothing, and says so once, loudly,
|
||||
// and once again when it opens.
|
||||
type lazyTurns struct {
|
||||
turns *check.Turns
|
||||
open func() error
|
||||
logf func(format string, args ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
opened bool
|
||||
said bool
|
||||
}
|
||||
|
||||
// ensure opens the bucket once it is there; a failure is said once.
|
||||
func (l *lazyTurns) ensure() {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
if l.opened || l.open == nil {
|
||||
return
|
||||
}
|
||||
if err := l.open(); err != nil {
|
||||
if !l.said {
|
||||
l.said = true
|
||||
l.logf("NO MERGE TURN HOLDS A CHECK: the gate's bucket %s cannot be read (%v); every check is claimed as if no approved merge waited, "+
|
||||
"and the bucket is opened again before each claim until it is there", delivery.TurnsBucket, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
l.opened = true
|
||||
if l.said {
|
||||
l.logf("the gate's merge turns are read from %s from now on: an approved merge's turn holds checks off its repositories again", delivery.TurnsBucket)
|
||||
}
|
||||
}
|
||||
|
||||
// Holding opens the bucket when it is not yet, then answers what the turns hold.
|
||||
func (l *lazyTurns) Holding(repositories []string, heads string) (string, delivery.TurnWait, bool) {
|
||||
l.ensure()
|
||||
return l.turns.Holding(repositories, heads)
|
||||
}
|
||||
@@ -112,14 +112,17 @@ func durationEnv(getenv func(string) string, name string, def time.Duration) (ti
|
||||
|
||||
// run is the program: it returns when ctx ends or the bus cannot be used.
|
||||
// keptStates is every state the checker keeps and which others may read (ADR 0495 §2-§3): owed by the forge module,
|
||||
// which writes mesh/check from it (ADR 0521), and by the forge bridge; previews by the indexer; verdicts by the gate.
|
||||
// which writes mesh/check from it (ADR 0521), and by the forge bridge; previews by the indexer; verdicts by the gate; tries by the delivery view.
|
||||
var keptStates = []announce.State{
|
||||
// The kept verdicts are read by the Go build module, which takes them for a check ask's unchanged packages (issue 1887's
|
||||
// design limit 1); the checker alone writes them (ADR 0444 clause 5).
|
||||
// Whether DEAD_LETTERS is heard (issue 734): read by the checker's own status tool alone.
|
||||
{Name: "hearing"},
|
||||
{Name: "kept", Readable: true},
|
||||
{Name: "owed", Readable: true},
|
||||
{Name: "previews", Readable: true},
|
||||
{Name: "tries"},
|
||||
// The tries are read by the delivery view, which tells a try's asks from a candidate's by the asks each names (build_usage).
|
||||
{Name: "tries", Readable: true},
|
||||
{Name: "verdicts", Readable: true},
|
||||
}
|
||||
|
||||
@@ -193,13 +196,28 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
tools := &buildrun.ToolchainSet{}
|
||||
presence := &buildrun.PresenceSet{}
|
||||
trunks := &check.Trunks{}
|
||||
turns := &check.Turns{}
|
||||
lazy := &lazyTurns{turns: turns, logf: logger.Printf}
|
||||
// A check is one ask per pull request and build module on the build seats' check stream, pulled by a build node with a
|
||||
// free slot (novox/hq ADR 0525 §1, §3): the checker chooses no node and starts no step.
|
||||
asks := buildrun.JetStreamAsks{JS: js}
|
||||
// The asks the build seats gave up on (issue 734): heard from DEAD_LETTERS, each ends the step its asker waits for.
|
||||
// Heard in the background, and again at a reconnect or an ask while it is not; whether it is heard is kept in the
|
||||
// state hearing, which checker_status says (dead_letters).
|
||||
dead := &buildrun.DeadLetters{}
|
||||
// The state may not be on the bus yet when a delivery started the checker (the grants part creates it, issue 1790): it
|
||||
// is bound when the bus says it was created, and the latest hearing is written then.
|
||||
hearingKV := process.AwaitBucket(ctx, js, conn, check.HearingBucket, "checker_status cannot say whether DEAD_LETTERS is heard", logger.Printf)
|
||||
hearing := buildrun.NewDeadLetterHearing(ctx, buildrun.JetStreamDeadLetters{JS: js}, []string{delivery.AskGo, delivery.AskNode}, dead,
|
||||
logger.Printf, buildrun.SayHearing(ctx, hearingKV, logger.Printf))
|
||||
defer hearing.Stop()
|
||||
conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again() })
|
||||
waiter := buildrun.LostAskWaiter{Inner: asks, Dead: dead, Hearing: hearing}
|
||||
ch := &check.Checker{
|
||||
Verdicts: kvStore{buckets[delivery.CheckerVerdictsBucket]},
|
||||
Owed: kvStore{buckets[delivery.CheckerOwedBucket]},
|
||||
Trunks: trunks,
|
||||
Turns: lazy,
|
||||
Bound: checkBound,
|
||||
Link: link,
|
||||
Logf: logger.Printf,
|
||||
@@ -228,7 +246,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return cw, nil
|
||||
}}
|
||||
// The tries (ADR 0501 §5) are judged by the same judge, held by the same deliveries.
|
||||
tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, Logf: logger.Printf}
|
||||
tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, NotTaken: stepBound, Logf: logger.Printf}
|
||||
announce := func(ctx context.Context, event, key string) {
|
||||
body, _ := json.Marshal(map[string]string{"key": key})
|
||||
if _, err := js.Publish(ctx, eventSubject(event), body); err != nil {
|
||||
@@ -236,8 +254,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
}
|
||||
repos := &repoModules{manifests: kvStore{buckets[delivery.ManifestsBucket]}, logf: logger.Printf}
|
||||
ch.Runs = repos // the node-engine a check reads beside at its running commit (ADR 0548 §6)
|
||||
ch.Judge = newJudge(judgeDeps{
|
||||
asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: stepBound}, tools: tools, trunks: trunks,
|
||||
asks: buildrun.Asker{Publisher: asks, Waiter: waiter, Bound: stepBound}, tools: tools, trunks: trunks,
|
||||
kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]},
|
||||
repos: repos, builds: builds, forge: forgeReads{call: caller.Call}, logf: logger.Printf,
|
||||
announce: announce, compositions: compositions,
|
||||
@@ -313,13 +332,42 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}()
|
||||
}
|
||||
}
|
||||
// The gate's merge turns (novox/hq ADR 0548 §3): no check is claimed of a repository an approved merge waits on, and the
|
||||
// delete of its turn claims it. Read before the candidates, so no check starts past a turn written before this start.
|
||||
// Where the gate's bucket is not there yet it is read as empty, and opened again before each claim until it is there
|
||||
// (lazyTurns), so a gate delivered after the checker started holds checks from its first turn on.
|
||||
lazy.open = func() error {
|
||||
kv, err := js.KeyValue(ctx, delivery.TurnsBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { turns.Put(e.Key(), e.Value(), !opOf(e)) })
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func() {
|
||||
errs <- follow(ctx, tw, func(e jetstream.KeyValueEntry) {
|
||||
if turns.Put(e.Key(), e.Value(), !opOf(e)) {
|
||||
ch.Reconsider(ctx) // a turn went or changed: a check it held is claimed now
|
||||
}
|
||||
})
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
lazy.ensure()
|
||||
// The modules say which repository holds a TypeScript module, so the Node build module judges it too.
|
||||
mw, err := settle(ctx, buckets[delivery.ModulesBucket], func(e jetstream.KeyValueEntry) { repos.Put(ctx, e.Key(), e.Value(), !opOf(e)) })
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func() {
|
||||
errs <- follow(ctx, mw, func(e jetstream.KeyValueEntry) { repos.Put(ctx, e.Key(), e.Value(), !opOf(e)) })
|
||||
errs <- follow(ctx, mw, func(e jetstream.KeyValueEntry) {
|
||||
before := repos.CommitOf(check.NodeEngineModule)
|
||||
repos.Put(ctx, e.Key(), e.Value(), !opOf(e))
|
||||
if after := repos.CommitOf(check.NodeEngineModule); after != before {
|
||||
ch.TrunkMoved(ctx) // the node-engine the mesh runs moved: a verdict that read it beside is carried (ADR 0548 §6)
|
||||
}
|
||||
})
|
||||
}()
|
||||
each := func(e jetstream.KeyValueEntry) {
|
||||
if opOf(e) {
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-checker
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142434-65ee43f07583
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93 h1:FAiFY2KBKEoZSH49OZTt7/zRBIQxyl5g/KhYrpggbN4=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142434-65ee43f07583 h1:ikD/ggf9/5k23sxJOX1PFcGzb4M3+QeC66AT0MyQ9Zs=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142434-65ee43f07583/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -0,0 +1,355 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// One check per delivery (novox/hq ADR 0548): the checker's part. A merge turn holds new checks off its repository (§3);
|
||||
// a trunk or sibling move never ends a running check and carries its verdict once it ended (§1, §5, §6); only a head git
|
||||
// refused gets a whole check after a move (§7); the line of a carried verdict says so (§8).
|
||||
|
||||
// turnOf is a turn's value on a repository for a delivery of the heads hash.
|
||||
func turnOf(t testing.TB, repo, group, heads string) []byte {
|
||||
t.Helper()
|
||||
at := time.Date(2026, 10, 26, 9, 0, 0, 0, time.UTC)
|
||||
data, err := delivery.EncodeTurn(delivery.Turn{Repository: repo, Waiting: []delivery.TurnWait{{Group: group, Heads: heads, Approved: at, Since: at}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func turnKey(t testing.TB, repo string) string {
|
||||
t.Helper()
|
||||
k, err := delivery.TurnsKey(repo)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return k
|
||||
}
|
||||
|
||||
// waitFor polls a condition the checker's goroutines make true, failing after two seconds.
|
||||
func waitFor(t testing.TB, what string, ok func() bool) {
|
||||
t.Helper()
|
||||
for deadline := time.Now().Add(2 * time.Second); !ok(); time.Sleep(time.Millisecond) {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("%s did not happen", what)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// pullWithFP is a pull request touching the module m at the fingerprint `src2:` and 64 times the hex digit d.
|
||||
func pullWithFP(repo string, number int, head string, d string) delivery.CandidatePull {
|
||||
p := pullOf(repo, number, head)
|
||||
p.Modules = []string{"m"}
|
||||
p.Fps = map[string]delivery.CandidateModule{"m": {FP2: "src2:" + strings.Repeat(d, 64)}}
|
||||
return p
|
||||
}
|
||||
|
||||
func TestOneCheckPerDelivery(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
for name, run := range map[string]func(t *testing.T){
|
||||
// §3: no claim while a turn names one of its repositories; a claim on the turn's delete event.
|
||||
"a turn on its repository holds a new check off until the turn goes": func(t *testing.T) {
|
||||
j := passJudge()
|
||||
r := newRig(t, j)
|
||||
turns := &Turns{}
|
||||
r.ch.Turns = turns
|
||||
turns.Put(turnKey(t, repoA), turnOf(t, repoA, "fix/other", "0123456789ab"), false)
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
r.ch.Wait()
|
||||
if j.count() != 0 || len(r.verdicts.vals) != 0 {
|
||||
t.Fatalf("judged %d times with a turn on %s, verdicts %v", j.count(), repoA, r.verdicts.vals)
|
||||
}
|
||||
if held := r.ch.Held()[keyOf(t, "feat/x", cand)]; !strings.Contains(held, repoA) || !strings.Contains(held, "fix/other") {
|
||||
t.Errorf("the hold is said as %q, want the turn's repository and group", held)
|
||||
}
|
||||
if !turns.Put(turnKey(t, repoA), nil, true) {
|
||||
t.Fatal("the turn's delete changed nothing")
|
||||
}
|
||||
r.ch.Reconsider(ctx) // what the watch of turns does on the delete
|
||||
r.ch.Wait()
|
||||
if j.count() != 1 || r.verdict(t, keyOf(t, "feat/x", cand)).State != delivery.VerdictPass {
|
||||
t.Errorf("judged %d times after the turn went, want once", j.count())
|
||||
}
|
||||
},
|
||||
"a turn on another repository, or of its own heads, holds nothing": func(t *testing.T) {
|
||||
j := passJudge()
|
||||
r := newRig(t, j)
|
||||
turns := &Turns{}
|
||||
r.ch.Turns = turns
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
c, _ := delivery.DecodeCandidate(cand)
|
||||
turns.Put(turnKey(t, repoSDK), turnOf(t, repoSDK, "fix/other", "0123456789ab"), false)
|
||||
turns.Put(turnKey(t, repoA), turnOf(t, repoA, "feat/x", GateHeads(c)), false)
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
r.ch.Wait()
|
||||
if j.count() != 1 {
|
||||
t.Errorf("judged %d times, want once: the turns hold another repository and its own merge", j.count())
|
||||
}
|
||||
},
|
||||
// §3, the race: a check claimed before the gate's turn is judged on the commit it read, never restarted.
|
||||
"a check claimed before a turn was written runs to its verdict": func(t *testing.T) {
|
||||
release, started := make(chan struct{}), make(chan struct{})
|
||||
j := &judgeFunc{f: func(context.Context, *Case, int) (Outcome, error) {
|
||||
close(started)
|
||||
<-release
|
||||
return Outcome{Verdict: passing()}, nil
|
||||
}}
|
||||
r := newRig(t, j)
|
||||
turns := &Turns{}
|
||||
r.ch.Turns = turns
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
<-started
|
||||
turns.Put(turnKey(t, repoA), turnOf(t, repoA, "fix/other", "0123456789ab"), false)
|
||||
r.ch.Reconsider(ctx)
|
||||
close(release)
|
||||
r.ch.Wait()
|
||||
if j.count() != 1 || r.verdict(t, keyOf(t, "feat/x", cand)).State != delivery.VerdictPass {
|
||||
t.Errorf("judged %d times; the running check is not ended by the turn", j.count())
|
||||
}
|
||||
},
|
||||
// §1, §5: a trunk move during a running check calls the judge once, writes its verdict, keeps the status pending (a pass
|
||||
// on an old trunk is never shown), then asks one carry, on the recheck stream, and no check.
|
||||
"a trunk move during a check ends nothing and asks one carry after it": func(t *testing.T) {
|
||||
var r *rig
|
||||
var kinds []string
|
||||
var from []map[string]string
|
||||
judged := make(chan struct{})
|
||||
j := &judgeFunc{f: func(ctx context.Context, c *Case, call int) (Outcome, error) {
|
||||
kinds, from = append(kinds, c.Kind), append(from, c.CarriedFrom)
|
||||
if call == 1 {
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) // the trunk moves while it runs
|
||||
r.ch.TrunkMoved(ctx) // as the watch of the source says it
|
||||
close(judged)
|
||||
} else if s := owedState(t, r, repoA, 7, head1); s != "pending" {
|
||||
t.Errorf("the status during the carry is %q, want pending: the pass on the old trunk is never shown", s)
|
||||
}
|
||||
return Outcome{Verdict: passing()}, nil
|
||||
}}
|
||||
r = newRig(t, j)
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
<-judged
|
||||
r.ch.Wait()
|
||||
v := r.verdict(t, keyOf(t, "feat/x", cand))
|
||||
if j.count() != 2 || kinds[0] != KindNew || kinds[1] != KindTrunk || from[0] != nil || from[1][repoA] != trunkA {
|
||||
t.Fatalf("judged %d times as %v carried from %v; want a whole check, then one carry from %s", j.count(), kinds, from, trunkA[:12])
|
||||
}
|
||||
if v.State != delivery.VerdictPass || v.Trunk[repoA] != trunkA2 || v.CarriedFrom[repoA] != trunkA {
|
||||
t.Errorf("the verdict is %s on %v carried from %v; want a pass on the new trunk carried from the old", v.State, v.Trunk, v.CarriedFrom)
|
||||
}
|
||||
if s := owedState(t, r, repoA, 7, head1); s != "success" {
|
||||
t.Errorf("the status after the carry is %q, want success", s)
|
||||
}
|
||||
if r.verdicts.creates[keyOf(t, "feat/x", cand)] != 1 {
|
||||
t.Error("the verdict key was made again: a carry rewrites it")
|
||||
}
|
||||
},
|
||||
// §5: the composition is kept when no fingerprint of a module it composed changed, and runs again otherwise.
|
||||
"a carry keeps a passed composition while the fingerprints stay the same": func(t *testing.T) {
|
||||
var kept []bool
|
||||
j := &judgeFunc{f: func(_ context.Context, c *Case, _ int) (Outcome, error) {
|
||||
kept = append(kept, c.Composition != nil)
|
||||
v := passing()
|
||||
v.Composition = &delivery.Composition{State: delivery.CompositionPass, SDK: "v0.1.14-x", FPSet: c.FPSet}
|
||||
return Outcome{Verdict: v}, nil
|
||||
}}
|
||||
r := newRig(t, j)
|
||||
cand := candidateOf(t, "feat/x", pullWithFP(repoA, 7, head1, "a"))
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
r.ch.Wait()
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS})
|
||||
r.ch.TrunkMoved(ctx) // the indexer's fingerprints of the candidate stay the same
|
||||
r.ch.Wait()
|
||||
r.ch.Candidate(ctx, "feat%2Fx", candidateOf(t, "feat/x", pullWithFP(repoA, 7, head1, "b"))) // a fingerprint changed with the move
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS})
|
||||
r.ch.TrunkMoved(ctx)
|
||||
r.ch.Wait()
|
||||
if len(kept) != 3 || kept[0] || !kept[1] || kept[2] {
|
||||
t.Errorf("the composition was kept in the checks %v; want composed, kept on the carry, composed again once a fingerprint changed", kept)
|
||||
}
|
||||
},
|
||||
// §6: a sibling's move puts the verdict out of date, its status pending saying it is carried, and the carry's verdict
|
||||
// names the sibling's old commit; the status follows the carry's verdict. Which packages run is the digest's (only
|
||||
// those that name MESH_CHECK_BESIDE change with a sibling), the build module's part.
|
||||
"a sibling's move is carried and the status follows the carry": func(t *testing.T) {
|
||||
var r *rig
|
||||
var from map[string]string
|
||||
j := &judgeFunc{f: func(_ context.Context, c *Case, call int) (Outcome, error) {
|
||||
if call == 2 {
|
||||
from = c.CarriedFrom
|
||||
if s := owedState(t, r, repoA, 7, head1); s != "pending" {
|
||||
t.Errorf("the status during the sibling's carry is %q, want pending", s)
|
||||
}
|
||||
}
|
||||
return Outcome{Verdict: passing(), Beside: map[string]string{catalogueRepository: r.trunks.Heads()[catalogueRepository]}}, nil
|
||||
}}
|
||||
r = newRig(t, j)
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS, catalogueRepository: trunkS})
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
r.ch.Wait()
|
||||
if b := r.verdict(t, keyOf(t, "feat/x", cand)).Beside; b[catalogueRepository] != trunkS {
|
||||
t.Fatalf("the verdict names the siblings %v, want the catalogue at %s", b, trunkS[:12])
|
||||
}
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS, catalogueRepository: trunkS2})
|
||||
r.ch.TrunkMoved(ctx)
|
||||
r.ch.Wait()
|
||||
v := r.verdict(t, keyOf(t, "feat/x", cand))
|
||||
if j.count() != 2 || from[catalogueRepository] != trunkS || v.CarriedFrom[catalogueRepository] != trunkS || v.Beside[catalogueRepository] != trunkS2 {
|
||||
t.Errorf("judged %d times, carried from %v, verdict beside %v carried from %v; want one carry over the catalogue's move", j.count(), from, v.Beside, v.CarriedFrom)
|
||||
}
|
||||
if s := owedState(t, r, repoA, 7, head1); s != "success" {
|
||||
t.Errorf("the status after the carry is %q, want success", s)
|
||||
}
|
||||
},
|
||||
// §7: a carry whose merge conflicts fails naming the files and asks no check; a later move onto which the head merges
|
||||
// cleanly asks a whole check, not a carry.
|
||||
"a carry git refuses fails naming the files, and the next move asks a whole check": func(t *testing.T) {
|
||||
var kinds []string
|
||||
var from []map[string]string
|
||||
j := &judgeFunc{f: func(_ context.Context, c *Case, call int) (Outcome, error) {
|
||||
kinds, from = append(kinds, c.Kind), append(from, c.CarriedFrom)
|
||||
if call == 2 {
|
||||
return Outcome{Verdict: delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "ace",
|
||||
Words: repoA + "#7 " + ConflictWords + "go.mod"}}}, nil
|
||||
}
|
||||
return Outcome{Verdict: passing()}, nil
|
||||
}}
|
||||
r := newRig(t, j)
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
r.ch.Wait()
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS})
|
||||
r.ch.TrunkMoved(ctx)
|
||||
r.ch.Wait()
|
||||
r.ch.Reconsider(ctx) // anything else that looks again asks nothing of a conflict
|
||||
r.ch.Wait()
|
||||
if v := r.verdict(t, keyOf(t, "feat/x", cand)); j.count() != 2 || !IsConflictFailure(v) || !strings.Contains(v.Failed.Words, "go.mod") {
|
||||
t.Fatalf("judged %d times, verdict %+v; want the carry failed naming go.mod and nothing asked after it", j.count(), v.Failed)
|
||||
}
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS})
|
||||
r.ch.TrunkMoved(ctx)
|
||||
r.ch.Wait()
|
||||
if j.count() != 3 || kinds[1] != KindTrunk || from[1] == nil || kinds[2] != KindNew || from[2] != nil {
|
||||
t.Errorf("checks %v carried from %v; want a whole check, a carry, then a whole check after the conflict", kinds, from)
|
||||
}
|
||||
},
|
||||
// §8, §6: the line of a carried verdict names the head on the new trunk commit first, as the gate and the merge tool
|
||||
// read a line, then where it was carried from and what the carry ran and kept; while it is carried it is pending and says so.
|
||||
"the line of a carry says where it was carried from": func(t *testing.T) {
|
||||
cs, p := lineCase(t, pullOf(repoA, 7, head1))
|
||||
v := passVerdict()
|
||||
v.Trunk = map[string]string{repoA: trunkA}
|
||||
v.Beside = map[string]string{catalogueRepository: trunkS2}
|
||||
v.CarriedFrom = map[string]string{repoA: trunkA2, catalogueRepository: trunkS}
|
||||
line := Line(v, cs, p, nil, nil, nil)
|
||||
if !strings.HasPrefix(line, "pass 111111111111 on aaaaaaaaaaaa | carried from abababababab, mesh-catalog 555555555555 | unit 12 run") {
|
||||
t.Errorf("the carried line = %q", line)
|
||||
}
|
||||
if head, trunk := checkWordsOf(line); head != head1[:12] || trunk != trunkA[:12] {
|
||||
t.Errorf("the line names %s on %s first, want the head on the new trunk", head, trunk)
|
||||
}
|
||||
cs.CarriedFrom = map[string]string{repoA: trunkA2}
|
||||
pending := Line(delivery.Verdict{State: delivery.VerdictRunning}, cs, p, nil, nil, nil)
|
||||
if !strings.HasPrefix(pending, "being carried to 111111111111 on aaaaaaaaaaaa | from abababababab") {
|
||||
t.Errorf("the pending line of a carry = %q", pending)
|
||||
}
|
||||
},
|
||||
// §3 (review round 1): two approved merges in one turn whose trunk moved each get their carry; neither holds the other's.
|
||||
"two merges waiting in one turn each get their carry": func(t *testing.T) {
|
||||
var r *rig
|
||||
j := &judgeFunc{f: func(context.Context, *Case, int) (Outcome, error) { return Outcome{Verdict: passing()}, nil }}
|
||||
r = newRig(t, j)
|
||||
candA, candC := candidateOf(t, "feat/a", pullOf(repoA, 7, head1)), candidateOf(t, "feat/c", pullOf(repoA, 8, head2))
|
||||
r.ch.Candidate(ctx, "feat%2Fa", candA)
|
||||
r.ch.Candidate(ctx, "feat%2Fc", candC)
|
||||
r.ch.Wait()
|
||||
a, _ := delivery.DecodeCandidate(candA)
|
||||
c, _ := delivery.DecodeCandidate(candC)
|
||||
at := time.Date(2026, 10, 26, 9, 0, 0, 0, time.UTC)
|
||||
turn, err := delivery.EncodeTurn(delivery.Turn{Repository: repoA, Waiting: []delivery.TurnWait{
|
||||
{Group: "feat/a", Heads: GateHeads(a), Approved: at, Since: at}, {Group: "feat/c", Heads: GateHeads(c), Approved: at.Add(time.Minute), Since: at}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
turns := &Turns{}
|
||||
turns.Put(turnKey(t, repoA), turn, false)
|
||||
r.ch.Turns = turns
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) // a third merge moves the trunk under both
|
||||
r.ch.TrunkMoved(ctx)
|
||||
r.ch.Wait()
|
||||
for _, cand := range [][]byte{candA, candC} {
|
||||
group := map[bool]string{true: "feat/a", false: "feat/c"}[string(cand) == string(candA)]
|
||||
if v := r.verdict(t, keyOf(t, group, cand)); v.Trunk[repoA] != trunkA2 || v.CarriedFrom[repoA] != trunkA {
|
||||
t.Errorf("%s: verdict on %v carried from %v; want its carry claimed and done", group, v.Trunk, v.CarriedFrom)
|
||||
}
|
||||
}
|
||||
},
|
||||
// §6 (review round 1): the node-engine read at the commit the mesh runs is named, and a new running commit is carried.
|
||||
"a new running node-engine is carried": func(t *testing.T) {
|
||||
runs := runsAt{nodeEngineModule: trunkS}
|
||||
j := &judgeFunc{f: func(_ context.Context, c *Case, _ int) (Outcome, error) {
|
||||
return Outcome{Verdict: passing(), Beside: map[string]string{nodeEngineRepo: runs[nodeEngineModule]}}, nil
|
||||
}}
|
||||
r := newRig(t, j)
|
||||
r.ch.Runs = runs
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
r.ch.Wait()
|
||||
r.ch.TrunkMoved(ctx) // nothing moved: nothing is carried
|
||||
r.ch.Wait()
|
||||
runs[nodeEngineModule] = trunkS2
|
||||
r.ch.TrunkMoved(ctx) // what the indexer's watch does when mesh-host's running commit changes
|
||||
r.ch.Wait()
|
||||
v := r.verdict(t, keyOf(t, "feat/x", cand))
|
||||
if j.count() != 2 || v.CarriedFrom[nodeEngineRepo] != trunkS || v.Beside[nodeEngineRepo] != trunkS2 {
|
||||
t.Errorf("judged %d times, verdict beside %v carried from %v; want one carry over the node-engine's new running commit", j.count(), v.Beside, v.CarriedFrom)
|
||||
}
|
||||
},
|
||||
// §5: a carry that fails names the move it failed on.
|
||||
"a carry that fails names the move": func(t *testing.T) {
|
||||
j := &judgeFunc{f: func(_ context.Context, c *Case, call int) (Outcome, error) {
|
||||
if call == 1 {
|
||||
return Outcome{Verdict: passing()}, nil
|
||||
}
|
||||
return Outcome{Verdict: delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Test: "unit: m/a.TestX", Words: "want 4"}}}, nil
|
||||
}}
|
||||
r := newRig(t, j)
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
r.ch.Wait()
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS})
|
||||
r.ch.TrunkMoved(ctx)
|
||||
r.ch.Wait()
|
||||
v := r.verdict(t, keyOf(t, "feat/x", cand))
|
||||
if v.State != delivery.VerdictFail || v.CarriedFrom != nil || !strings.HasPrefix(v.Failed.Words, "want 4") ||
|
||||
!strings.Contains(v.Failed.Words, repoA+" "+trunkA[:12]+" to "+trunkA2[:12]) {
|
||||
t.Errorf("the failed carry is %s carried from %v with words %q; want the test's words, then the move", v.State, v.CarriedFrom, v.Failed.Words)
|
||||
}
|
||||
},
|
||||
} {
|
||||
t.Run(name, run)
|
||||
}
|
||||
}
|
||||
|
||||
// checkWordsOf is the head and trunk a line names first, as the gate reads it (mesh-delivery-gate onestatus.go checkLine).
|
||||
func checkWordsOf(line string) (string, string) {
|
||||
for _, part := range strings.Split(line, " | ") {
|
||||
f := strings.Fields(part)
|
||||
for i := 0; i+2 < len(f); i++ {
|
||||
if f[i+1] == "on" && len(f[i]) == 12 && len(f[i+2]) == 12 {
|
||||
return f[i], f[i+2]
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", ""
|
||||
}
|
||||
@@ -127,6 +127,9 @@ func (ch *Checker) Composed(ctx context.Context, comp delivery.CompositionVerdic
|
||||
return
|
||||
}
|
||||
withComposition(&v, comp)
|
||||
if v.Composition != nil && v.Composition.State != delivery.CompositionNotRun {
|
||||
v.Composition.FPSet = cs.FPSet
|
||||
}
|
||||
data, err := delivery.EncodeVerdict(v)
|
||||
if err != nil {
|
||||
ch.logf("%s: the verdict with its composition cannot be written: %v", cs.Key, err)
|
||||
|
||||
@@ -12,8 +12,18 @@ type BesideRef struct {
|
||||
Repository string `json:"repository"`
|
||||
Commit string `json:"commit"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
// atTrunk says the sibling was read at its trunk's head, so a move of that trunk puts the verdict out of date and is
|
||||
// carried (novox/hq ADR 0548 §6). Never sent: the node-engine at the commit the mesh runs, mesh-sdk at a pin and a
|
||||
// group's member at its head do not move with a trunk.
|
||||
atTrunk bool
|
||||
// runs says the sibling was read at the commit the mesh runs (the node-engine, the indexer's record of mesh-host): a new
|
||||
// running commit puts the verdict out of date and is carried (ADR 0548 §6). Never sent.
|
||||
runs bool
|
||||
}
|
||||
|
||||
// NodeEngineModule is the module whose running commit the node-engine sibling is read at.
|
||||
const NodeEngineModule = nodeEngineModule
|
||||
|
||||
const (
|
||||
catalogueRepository = "novox/mesh-catalog"
|
||||
nodeEngineRepo = "novox/mesh-host"
|
||||
@@ -51,18 +61,21 @@ func (j *Examiner) besideFor(ctx context.Context, c *Case, p Pull) ([]BesideRef,
|
||||
switch {
|
||||
case repo == catalogueRepository:
|
||||
if head := heads[repo]; head != "" {
|
||||
out = append(out, BesideRef{Repository: repo, Commit: head})
|
||||
out = append(out, BesideRef{Repository: repo, Commit: head, atTrunk: c.member(repo) == nil})
|
||||
}
|
||||
case repo == nodeEngineRepo:
|
||||
commit := ""
|
||||
if j.Runs != nil {
|
||||
commit = j.Runs.CommitOf(nodeEngineModule)
|
||||
}
|
||||
atTrunk := false
|
||||
if commit == "" {
|
||||
commit = heads[repo]
|
||||
commit, atTrunk = heads[repo], true
|
||||
}
|
||||
if commit != "" {
|
||||
out = append(out, BesideRef{Repository: repo, Commit: commit})
|
||||
// Named in the verdict either way (ADR 0548 §6, issue 568's own example): at the commit the mesh runs, a new
|
||||
// running commit puts the verdict out of date; at the trunk, a trunk move does (Checker.besideNow).
|
||||
out = append(out, BesideRef{Repository: repo, Commit: commit, atTrunk: atTrunk, runs: !atTrunk})
|
||||
}
|
||||
case repo == SDKRepository && p.Repository == controllerRepo && j.Forge != nil:
|
||||
text, err := j.Forge.File(ctx, p.Repository, "go.mod", p.Head)
|
||||
@@ -82,7 +95,7 @@ func (j *Examiner) besideFor(ctx context.Context, c *Case, p Pull) ([]BesideRef,
|
||||
if member := c.member(repo); member != nil {
|
||||
out = append(out, BesideRef{Repository: repo, Commit: member.Head, Ref: refOf(member.Number)})
|
||||
} else if head := heads[repo]; head != "" {
|
||||
out = append(out, BesideRef{Repository: repo, Commit: head})
|
||||
out = append(out, BesideRef{Repository: repo, Commit: head, atTrunk: true})
|
||||
} else if repo == controllerRepo && p.Repository == nodeEngineRepo {
|
||||
// The node-engine's step-nine test judges the controller's own manifest and fails without it (issue 1295): a
|
||||
// checker that cannot place it says so, where placing none left that test to skip.
|
||||
@@ -108,3 +121,13 @@ func refOf(number int) string {
|
||||
}
|
||||
return "refs/pull/" + strconv.Itoa(number) + "/head"
|
||||
}
|
||||
|
||||
// besideAtTrunk are the siblings of refs read at their trunk's head that the case does not merge, by repository: what a
|
||||
// verdict names as Beside (ADR 0548 §6).
|
||||
func besideAtTrunk(c *Case, refs []BesideRef, into map[string]string) {
|
||||
for _, r := range refs {
|
||||
if _, merged := c.Trunk[r.Repository]; (r.atTrunk || r.runs) && !merged {
|
||||
into[r.Repository] = r.Commit
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,11 +44,35 @@ type Case struct {
|
||||
|
||||
notesMu sync.Mutex
|
||||
notes []string
|
||||
// asks are the ids of the asks the check put on the build seats' streams, in order: a try names them (Try.Asks).
|
||||
asks []string
|
||||
// Progress says what the check is doing, so a check that ran past its bound names the step it was stuck at.
|
||||
Progress *Progress
|
||||
// Kind is what asked the check (KindNew, KindNodes, KindTrunk): a re-judge after a trunk move is asked on the build
|
||||
// seats' recheck stream, every other check on check (ADR 0536 §4).
|
||||
// Kind is what asked the check (KindNew, KindNodes, KindTrunk): a carry after a trunk or sibling move is asked on the
|
||||
// build seats' recheck stream, every other check on check (ADR 0536 §4, ADR 0548 §5).
|
||||
Kind string
|
||||
// CarriedFrom are, for the carry of a verdict that passed, the trunk and sibling commits it was judged on (ADR 0548 §5);
|
||||
// nil for a whole check.
|
||||
CarriedFrom map[string]string
|
||||
// FPSet is the candidate's fingerprint set (Candidate.FPSet): a composition names the set it judged.
|
||||
FPSet string
|
||||
// Composition is, for a carry, the composition that passed on the same fingerprint set: it is kept and not asked again,
|
||||
// since no fingerprint of a module it composed changed (ADR 0548 §5). Nil composes.
|
||||
Composition *delivery.Composition
|
||||
}
|
||||
|
||||
// Asked records the id of an ask the check put.
|
||||
func (c *Case) Asked(id string) {
|
||||
c.notesMu.Lock()
|
||||
c.asks = append(c.asks, id)
|
||||
c.notesMu.Unlock()
|
||||
}
|
||||
|
||||
// Asks are the ids of the asks the check put, in order.
|
||||
func (c *Case) Asks() []string {
|
||||
c.notesMu.Lock()
|
||||
defer c.notesMu.Unlock()
|
||||
return append([]string(nil), c.asks...)
|
||||
}
|
||||
|
||||
// Progress is the step a check is at, written by the check and read when its bound passes.
|
||||
@@ -80,7 +104,7 @@ func CaseOf(c delivery.Candidate, trunk map[string]string) *Case {
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
cs := &Case{Key: key, Group: c.Group, HeadsHash: c.HeadsHash, Heads: GateHeads(c), Trunk: map[string]string{}, Progress: &Progress{}}
|
||||
cs := &Case{Key: key, Group: c.Group, HeadsHash: c.HeadsHash, Heads: GateHeads(c), Trunk: map[string]string{}, Progress: &Progress{}, FPSet: c.FPSet}
|
||||
for _, p := range c.Pulls {
|
||||
switch head, known := trunk[p.Repository]; {
|
||||
case !known || head == "":
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -42,6 +43,9 @@ type Outcome struct {
|
||||
// Judged says, by pull request (pullKey), which program judged a change to a build module (ADR 0535 §5); a pull request
|
||||
// that changes none is not named.
|
||||
Judged map[string]string
|
||||
// Beside are the siblings the check read at their trunk's head, by repository (ADR 0548 §6): the verdict names them, and
|
||||
// a move of one is carried.
|
||||
Beside map[string]string
|
||||
}
|
||||
|
||||
// BuildNote is the state of a candidate's module build, by module: built, failed or waiting (ADR 0484 clause 7).
|
||||
@@ -65,6 +69,14 @@ type Checker struct {
|
||||
Owed KV
|
||||
Judge Judge
|
||||
Trunks interface{ Heads() map[string]string }
|
||||
// Runs says which commit the mesh runs of a module (the indexer's modules): the node-engine a verdict read beside at its
|
||||
// running commit is out of date when that commit moves (ADR 0548 §6); nil reads every sibling at its trunk.
|
||||
Runs interface{ CommitOf(module string) string }
|
||||
// Turns are the gate's merge turns (ADR 0548 §3): no check is claimed of a repository a turn names, but the check of the
|
||||
// waiting delivery's own heads; nil holds none.
|
||||
Turns interface {
|
||||
Holding(repositories []string, heads string) (string, delivery.TurnWait, bool)
|
||||
}
|
||||
// Bound is how long one check may run; none when 0 (DefaultBound).
|
||||
Bound time.Duration
|
||||
// Link is the base of the address a status points at; the verdict key follows.
|
||||
@@ -83,12 +95,13 @@ type Checker struct {
|
||||
tried map[string]bool // by verdict key: looked at since this process started
|
||||
epoch uint64 // changes of the nodes seen since this process started
|
||||
nodesFail map[string]uint64 // by verdict key: the epoch a failure of the nodes was written at
|
||||
held map[string]string // by verdict key: the turn that holds its check off, as last said
|
||||
}
|
||||
|
||||
func (ch *Checker) init() {
|
||||
ch.once.Do(func() {
|
||||
ch.known, ch.inflight, ch.again, ch.tried = map[string]delivery.Candidate{}, map[string]bool{}, map[string]bool{}, map[string]bool{}
|
||||
ch.nodesFail = map[string]uint64{}
|
||||
ch.nodesFail, ch.held = map[string]uint64{}, map[string]string{}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -191,7 +204,7 @@ func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidat
|
||||
// Every check is asked at once: the build seats' streams hold the backlog, a new head's check and a check after a
|
||||
// failure of the nodes on check, a re-judge after a trunk move on recheck, pulled after them (ADR 0525 §4, §9;
|
||||
// ADR 0536 §4). The checker holds no place and no capacity of its own.
|
||||
cs.Kind = ch.kindOf(ctx, cs)
|
||||
ch.prepare(ctx, cs)
|
||||
ch.try(ctx, key, cs)
|
||||
ch.mu.Lock()
|
||||
more := ch.again[cs.Key]
|
||||
@@ -211,29 +224,130 @@ func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidat
|
||||
}
|
||||
|
||||
// The kinds of a check (ADR 0536 §4): a head never judged and a head asked again after a failure of the nodes are asked on
|
||||
// the build seats' check stream, a head asked again only because its trunk moved on the recheck stream, pulled after.
|
||||
// the build seats' check stream; a carry, a verdict moved to a newer trunk or sibling commit by its package digests (ADR 0548
|
||||
// §5), on the recheck stream, pulled after.
|
||||
const (
|
||||
KindNew = "new"
|
||||
KindNodes = "nodes"
|
||||
KindTrunk = "trunk"
|
||||
)
|
||||
|
||||
// kindOf is the kind of a case's check, read from its verdict key: none (or one left running) is a head never judged; a
|
||||
// failure of the nodes is asked again after the nodes changed; a verdict on other trunk heads is a re-judge after a trunk
|
||||
// move. A verdict that stands is ranked with the re-judges: its claim finds nothing to do.
|
||||
func (ch *Checker) kindOf(ctx context.Context, cs *Case) string {
|
||||
// prepare reads the case's verdict key and says what its check is (ADR 0548 §1, §5, §7): none (or one left running) is a
|
||||
// head never judged, a whole check; a failure of the nodes is asked again after the nodes changed; a verdict judged on trunk
|
||||
// or sibling commits that moved since is carried, its packages run only where the move changed their digest, and a pass's
|
||||
// commits are named as the carry's CarriedFrom. A verdict that stands is ranked with the carries: its claim finds nothing
|
||||
// to do.
|
||||
func (ch *Checker) prepare(ctx context.Context, cs *Case) {
|
||||
cs.Kind, cs.CarriedFrom, cs.Composition = KindNew, nil, nil
|
||||
value, _, found, err := ch.Verdicts.Get(ctx, cs.Key)
|
||||
if err != nil || !found {
|
||||
return KindNew
|
||||
return
|
||||
}
|
||||
v, err := delivery.DecodeVerdict(value)
|
||||
switch {
|
||||
case err != nil || v.State == delivery.VerdictRunning:
|
||||
return KindNew
|
||||
return
|
||||
case IsNodesFailure(v):
|
||||
return KindNodes
|
||||
cs.Kind = KindNodes
|
||||
return
|
||||
case IsConflictFailure(v):
|
||||
return // a head git refused: its next check is whole (ADR 0548 §7)
|
||||
}
|
||||
return KindTrunk
|
||||
cs.Kind = KindTrunk
|
||||
if v.State == delivery.VerdictPass && ch.stale(v, cs) {
|
||||
cs.CarriedFrom = judgedOn(v)
|
||||
if c := v.Composition; c != nil && c.State == delivery.CompositionPass && c.FPSet != "" && c.FPSet == cs.FPSet {
|
||||
kept := *c
|
||||
cs.Composition = &kept
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// stale says whether a verdict was judged on commits that are no longer the case's: a trunk it merged moved, or a sibling it
|
||||
// read at its trunk's head (ADR 0548 §6).
|
||||
func (ch *Checker) stale(v delivery.Verdict, cs *Case) bool {
|
||||
return !sameTrunk(v.Trunk, cs.Trunk) || besideMoved(v.Beside, ch.besideNow())
|
||||
}
|
||||
|
||||
// besideNow are the commits a sibling is read at now, by repository: its trunk head, and for the node-engine the commit the
|
||||
// mesh runs when the indexer names one, as besideFor places them.
|
||||
func (ch *Checker) besideNow() map[string]string {
|
||||
return besideAt(ch.heads(), ch.Runs)
|
||||
}
|
||||
|
||||
// besideAt places the commits a sibling is read at: the trunk heads, with the node-engine at the commit the mesh runs when
|
||||
// runs names one (ADR 0548 §6). The checker's decision and the status read the same commits through it.
|
||||
func besideAt(now map[string]string, runs interface{ CommitOf(module string) string }) map[string]string {
|
||||
if runs != nil {
|
||||
if c := runs.CommitOf(nodeEngineModule); c != "" {
|
||||
out := make(map[string]string, len(now)+1)
|
||||
for r, h := range now {
|
||||
out[r] = h
|
||||
}
|
||||
out[nodeEngineRepo] = c
|
||||
return out
|
||||
}
|
||||
}
|
||||
return now
|
||||
}
|
||||
|
||||
// besideMoved says whether a sibling read at these commits moved: the commit it is read at now is known and another.
|
||||
func besideMoved(beside, now map[string]string) bool {
|
||||
for r, c := range beside {
|
||||
if h := now[r]; h != "" && h != c {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// judgedOn are the trunk and sibling commits a verdict was judged on, by repository.
|
||||
func judgedOn(v delivery.Verdict) map[string]string {
|
||||
out := make(map[string]string, len(v.Trunk)+len(v.Beside))
|
||||
for r, c := range v.Beside {
|
||||
out[r] = c
|
||||
}
|
||||
for r, c := range v.Trunk {
|
||||
out[r] = c
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// carried is what a carry's verdict names as carried from (ADR 0548 §5): the commits it was judged on before, of the
|
||||
// repositories the new verdict still judges or reads; nil when none of them moved.
|
||||
func carried(from map[string]string, v delivery.Verdict) map[string]string {
|
||||
out, moved := map[string]string{}, false
|
||||
for r, c := range from {
|
||||
now, ok := v.Trunk[r]
|
||||
if !ok {
|
||||
now, ok = v.Beside[r]
|
||||
}
|
||||
if ok && now != "" && c != "" {
|
||||
out[r] = c
|
||||
moved = moved || now != c
|
||||
}
|
||||
}
|
||||
if !moved {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// moveWords say a carry's moves, for the words of a failure: "carried from <old12> to <new12> of <repository>", each moved
|
||||
// repository, sorted.
|
||||
func moveWords(from map[string]string, v delivery.Verdict) string {
|
||||
var moves []string
|
||||
for r, c := range from {
|
||||
now, ok := v.Trunk[r]
|
||||
if !ok {
|
||||
now = v.Beside[r]
|
||||
}
|
||||
if now != "" && now != c {
|
||||
moves = append(moves, fmt.Sprintf("%s %s to %s", r, short12(c), short12(now)))
|
||||
}
|
||||
}
|
||||
sort.Strings(moves)
|
||||
return strings.Join(moves, ", ")
|
||||
}
|
||||
|
||||
func (ch *Checker) candidate(key string, fallback delivery.Candidate) delivery.Candidate {
|
||||
@@ -265,6 +379,9 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64
|
||||
return nil, fmt.Errorf("%s: %w", cs.Key, err)
|
||||
}
|
||||
if !found {
|
||||
if ch.heldByTurn(cs) {
|
||||
return nil, nil
|
||||
}
|
||||
rev, err := ch.Verdicts.Create(ctx, cs.Key, data)
|
||||
if errors.Is(err, ErrConflict) {
|
||||
return nil, nil // another checker made it first
|
||||
@@ -285,7 +402,7 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64
|
||||
return &claimed{rev}, nil
|
||||
}
|
||||
return nil, nil
|
||||
case !sameTrunk(cur.Trunk, cs.Trunk):
|
||||
case ch.stale(cur, cs):
|
||||
retake = true
|
||||
case cur.State == delivery.VerdictFail && IsNodesFailure(cur):
|
||||
ch.mu.Lock()
|
||||
@@ -293,7 +410,7 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64
|
||||
ch.mu.Unlock()
|
||||
retake = !ok || epoch > at
|
||||
}
|
||||
if !retake {
|
||||
if !retake || ch.heldByTurn(cs) {
|
||||
return nil, nil
|
||||
}
|
||||
rev, err = ch.Verdicts.Update(ctx, cs.Key, data, rev)
|
||||
@@ -306,6 +423,59 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64
|
||||
return &claimed{rev}, nil
|
||||
}
|
||||
|
||||
// heldByTurn says whether a merge turn holds the case's check off (ADR 0548 §3): an approved merge waits on one of the
|
||||
// repositories it merges heads into, and it is not the merge of the case's own heads. The check is claimed when the turn's
|
||||
// key goes (Reconsider on the turns watch). Said once per turn in the log. A check already claimed is never held: one
|
||||
// claimed in the moment before the gate wrote its turn is judged on the commit it read and carried once after.
|
||||
func (ch *Checker) heldByTurn(cs *Case) bool {
|
||||
if ch.Turns == nil {
|
||||
return false
|
||||
}
|
||||
repos := make([]string, 0, len(cs.Trunk))
|
||||
for r := range cs.Trunk {
|
||||
repos = append(repos, r)
|
||||
}
|
||||
repo, w, held := ch.Turns.Holding(repos, cs.Heads)
|
||||
why := ""
|
||||
if held {
|
||||
why = repo + " " + w.Group + " " + w.Heads
|
||||
}
|
||||
ch.mu.Lock()
|
||||
said := ch.held[cs.Key]
|
||||
if why == "" {
|
||||
delete(ch.held, cs.Key)
|
||||
} else {
|
||||
ch.held[cs.Key] = why
|
||||
}
|
||||
ch.mu.Unlock()
|
||||
if held && said != why {
|
||||
ch.logf("%s: not claimed: the approved merge of %s (heads %s) holds a turn on %s, so its check waits for that merge (ADR 0548 §3)",
|
||||
cs.Key, w.Group, w.Heads, repo)
|
||||
}
|
||||
return held
|
||||
}
|
||||
|
||||
// Held is every check a merge turn holds off now, by verdict key, with the turn's repository, group and heads.
|
||||
func (ch *Checker) Held() map[string]string {
|
||||
ch.init()
|
||||
ch.mu.Lock()
|
||||
defer ch.mu.Unlock()
|
||||
out := make(map[string]string, len(ch.held))
|
||||
for k, v := range ch.held {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ConflictWords are what the words of a head that git would not merge onto its trunk say before the files in conflict.
|
||||
const ConflictWords = "does not merge into the trunk: files in conflict: "
|
||||
|
||||
// IsConflictFailure says whether a verdict failed because its head did not merge onto its trunk (ADR 0548 §7): it has no
|
||||
// pass to carry, so its next check, at a push or at a later trunk move, is a whole check of its head.
|
||||
func IsConflictFailure(v delivery.Verdict) bool {
|
||||
return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, ConflictWords)
|
||||
}
|
||||
|
||||
// IsNodesFailure says whether a verdict failed because a part of the mesh the check reads was not heard (UnheardWords):
|
||||
// it is judged again when the nodes change. No check fails for want of a build node any more: its asks wait in the build
|
||||
// seats' streams (ADR 0525 §6, issue 1912).
|
||||
@@ -332,11 +502,41 @@ func (ch *Checker) try(ctx context.Context, key string, cs *Case) {
|
||||
began := ch.now()
|
||||
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
|
||||
out := ch.judge(ctx, cs)
|
||||
// An ask the build seat gave up on judged nothing (issue 734): the check is asked again under the same claim, its status
|
||||
// pending, and after LostAskTries it fails as not heard, judged again when the nodes change.
|
||||
for try := 1; IsLostAsk(out.Verdict) && ctx.Err() == nil; try++ {
|
||||
if try >= LostAskTries {
|
||||
out.Verdict.Failed.Words = strings.TrimSpace(out.Verdict.Failed.Words + "\nasked " + fmt.Sprint(LostAskTries) +
|
||||
" times and given up each time: " + UnheardWords)
|
||||
break
|
||||
}
|
||||
ch.logf("%s: %s; asked again (%d of %d)", cs.Key, out.Verdict.Failed.Words, try+1, LostAskTries)
|
||||
out = ch.judge(ctx, cs)
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
v := out.Verdict
|
||||
v.Group, v.Heads, v.Trunk = cs.Group, cs.Heads, cs.Trunk
|
||||
v.Group, v.Heads, v.Trunk, v.Beside = cs.Group, cs.Heads, cs.Trunk, nil
|
||||
for r, c := range out.Beside {
|
||||
if _, merged := cs.Trunk[r]; !merged && c != "" {
|
||||
if v.Beside == nil {
|
||||
v.Beside = map[string]string{}
|
||||
}
|
||||
v.Beside[r] = c
|
||||
}
|
||||
}
|
||||
if cs.CarriedFrom != nil {
|
||||
// A carry (ADR 0548 §5): a pass names the commits it was carried from; a failure names the move it failed on.
|
||||
switch v.State {
|
||||
case delivery.VerdictPass:
|
||||
v.CarriedFrom = carried(cs.CarriedFrom, v)
|
||||
case delivery.VerdictFail:
|
||||
if moves := moveWords(cs.CarriedFrom, v); moves != "" {
|
||||
v.Failed.Words = strings.TrimSpace(v.Failed.Words + "\nfailed on the carry over the move of " + moves + ": the move changed what these tests read")
|
||||
}
|
||||
}
|
||||
}
|
||||
if v.Took.Queue == 0 && !cs.Queued.IsZero() && began.After(cs.Queued) {
|
||||
v.Took.Queue = began.Sub(cs.Queued).Milliseconds()
|
||||
}
|
||||
@@ -358,11 +558,11 @@ func (ch *Checker) try(ctx context.Context, key string, cs *Case) {
|
||||
ch.nodesFail[cs.Key] = epoch
|
||||
ch.mu.Unlock()
|
||||
}
|
||||
if ch.movedSince(cs) {
|
||||
if ch.movedSince(cs) || besideMoved(v.Beside, ch.besideNow()) {
|
||||
// Judged on a trunk that moved while it ran: the verdict is kept as judged, and the status stays pending until the
|
||||
// check of the new trunk, which Reconsider starts, ends.
|
||||
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
|
||||
ch.logf("%s: %s on a trunk that moved while it ran; the status stays pending", cs.Key, v.State)
|
||||
ch.owePending(ctx, cs, v)
|
||||
ch.logf("%s: %s on a trunk that moved while it ran; the status stays pending until it is carried", cs.Key, v.State)
|
||||
return
|
||||
}
|
||||
ch.writeOwed(ctx, cs, v, out)
|
||||
@@ -502,14 +702,37 @@ func (ch *Checker) TrunkMoved(ctx context.Context) {
|
||||
if err != nil || !found {
|
||||
continue // never judged: no status to put out of date
|
||||
}
|
||||
if v, err := delivery.DecodeVerdict(value); err == nil && sameTrunk(v.Trunk, cs.Trunk) {
|
||||
v, err := delivery.DecodeVerdict(value)
|
||||
if err == nil && !ch.stale(v, cs) {
|
||||
continue
|
||||
}
|
||||
if err == nil && v.State == delivery.VerdictPass {
|
||||
cs.CarriedFrom = judgedOn(v) // the status says it is being carried to the new commit (ADR 0548 §6)
|
||||
}
|
||||
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
|
||||
}
|
||||
ch.Reconsider(ctx)
|
||||
}
|
||||
|
||||
// owePending owes mesh/check pending on the heads of a verdict judged on commits that moved while it ran: a pass says it is
|
||||
// being carried to the trunk's head now (ADR 0548 §6), anything else that it is checked again.
|
||||
func (ch *Checker) owePending(ctx context.Context, judged *Case, v delivery.Verdict) {
|
||||
next := judged
|
||||
ckey, err := delivery.CandidatesKey(v.Group)
|
||||
ch.mu.Lock()
|
||||
c, ok := ch.known[ckey]
|
||||
ch.mu.Unlock()
|
||||
if err == nil && ok {
|
||||
if now := CaseOf(c, ch.heads()); now != nil && now.Key == judged.Key {
|
||||
next = now
|
||||
}
|
||||
}
|
||||
if v.State == delivery.VerdictPass {
|
||||
next.CarriedFrom = judgedOn(v)
|
||||
}
|
||||
ch.writeOwed(ctx, next, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
|
||||
}
|
||||
|
||||
// movedSince says whether a trunk the case was judged on is no longer the trunk's head.
|
||||
func (ch *Checker) movedSince(cs *Case) bool {
|
||||
now := ch.heads()
|
||||
|
||||
@@ -28,6 +28,7 @@ func TestAPassingCheckTakesTheAssignersComposition(t *testing.T) {
|
||||
f := &fakeCompositions{v: delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionPass, SDK: "v0.1.14-x", Machines: []string{"g14"}, Took: 40}}
|
||||
b.ex.Compositions = f
|
||||
cs := caseOf(t, pullOf(repoA, 7, head1))
|
||||
cs.FPSet = "fps-of-the-candidate"
|
||||
v := b.judge(t, cs).Verdict
|
||||
if v.State != delivery.VerdictPass || v.Composition == nil || v.Composition.State != delivery.CompositionPass || v.Composition.SDK != "v0.1.14-x" || v.Took.Composition != 40 {
|
||||
t.Fatalf("verdict = %+v, composition %+v", v, v.Composition)
|
||||
@@ -38,6 +39,15 @@ func TestAPassingCheckTakesTheAssignersComposition(t *testing.T) {
|
||||
if !compositionMentions(v, "passed") {
|
||||
t.Errorf("the line says %q", compositionWords(v.Composition, v.Took.Composition))
|
||||
}
|
||||
if v.Composition.FPSet == "" || v.Composition.FPSet != cs.FPSet {
|
||||
t.Errorf("the composition names the fingerprint set %q, want the candidate's %q (ADR 0548 §5)", v.Composition.FPSet, cs.FPSet)
|
||||
}
|
||||
// ADR 0548 §5: a carry whose fingerprints did not change keeps the composition it passed and asks the assigner nothing.
|
||||
carry := caseOf(t, pullOf(repoA, 7, head1))
|
||||
carry.CarriedFrom, carry.Composition = map[string]string{repoA: trunkA2}, v.Composition
|
||||
if kept := b.judge(t, carry).Verdict; kept.Composition == nil || kept.Composition.SDK != "v0.1.14-x" || len(f.asked) != 1 {
|
||||
t.Errorf("the carry's composition is %+v and the assigner was asked %d times; want the kept one and no second ask", kept.Composition, len(f.asked))
|
||||
}
|
||||
}
|
||||
|
||||
func compositionMentions(v delivery.Verdict, word string) bool {
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// failingResult is a level's test result in which the named tests failed, each in its own package.
|
||||
func failingResult(t *testing.T, tests int, failed ...Failure) json.RawMessage {
|
||||
t.Helper()
|
||||
raw, err := json.Marshal(Tested{Level: delivery.LevelIntegration, Tests: tests, Elapsed: 2, Failed: failed, Passed: len(failed) == 0})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
func judgeLevel(t *testing.T, tests ...json.RawMessage) (*delivery.Verdict, *state) {
|
||||
t.Helper()
|
||||
st := &state{j: &Examiner{}, c: &Case{}, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{}}
|
||||
v, _ := st.level(t.Context(), &target{pull: Pull{Repository: "novox/x"}}, delivery.LevelAnswer{Level: delivery.LevelIntegration, Tests: tests})
|
||||
return v, st
|
||||
}
|
||||
|
||||
// Issue 1921: two failing tests in two packages are two entries of the verdict's Failures and two lines of a try's
|
||||
// failed list, each with its own first line (its cause, issue 1901), whether the build module answered them in one test
|
||||
// result (one go test run over both packages) or in two (the changed packages first, then the rest); every result is
|
||||
// counted, the ones after a failure too; and Failed keeps the first one's words whole.
|
||||
func TestTwoFailingTestsInTwoPackagesAreTwoEntries(t *testing.T) {
|
||||
a := Failure{Package: "x/a", Test: "TestA", Cause: "a_test.go:9: want 1, got 2", Output: " a_test.go:3: logged\n a_test.go:9: want 1, got 2\n--- FAIL: TestA (0.00s)"}
|
||||
b := Failure{Package: "x/b", Test: "TestB", Cause: "b_test.go:4: boom", Output: " b_test.go:4: boom\n--- FAIL: TestB (0.00s)"}
|
||||
for name, tests := range map[string][]json.RawMessage{
|
||||
"one result": {failingResult(t, 10, a, b)},
|
||||
"two results": {failingResult(t, 4, a), failingResult(t, 3), failingResult(t, 3, b)},
|
||||
} {
|
||||
v, st := judgeLevel(t, tests...)
|
||||
if v == nil {
|
||||
t.Fatalf("%s: the level passed", name)
|
||||
}
|
||||
want := []string{"integration: x/a.TestA: a_test.go:9: want 1, got 2", "integration: x/b.TestB: b_test.go:4: boom"}
|
||||
if got := failedLines(*v); strings.Join(got, "\n") != strings.Join(want, "\n") {
|
||||
t.Errorf("%s: failed lines\n%s\nwant\n%s", name, strings.Join(got, "\n"), strings.Join(want, "\n"))
|
||||
}
|
||||
if len(v.Failures) != 2 || v.Failed.Test != "integration: x/a.TestA" || !strings.Contains(v.Failed.Words, "a_test.go:3: logged") {
|
||||
t.Errorf("%s: verdict %+v", name, v)
|
||||
}
|
||||
if run := st.levels[delivery.LevelIntegration].run; run != 10 {
|
||||
t.Errorf("%s: %d tests counted, want all 10", name, run)
|
||||
}
|
||||
if _, err := delivery.EncodeVerdict(delivery.Verdict{Group: "g", Heads: "0123456789ab", State: v.State, Failed: v.Failed, Failures: v.Failures}); err != nil {
|
||||
t.Errorf("%s: the verdict is not written: %v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1921: past delivery.MaxFailures the list names the first fifty and how many more, the status line says how many
|
||||
// failed, and each entry's line is cut to MaxFailureLine.
|
||||
func TestFailuresAreCappedWithHowManyMore(t *testing.T) {
|
||||
var fs []Failure
|
||||
for i := range 60 {
|
||||
fs = append(fs, Failure{Package: fmt.Sprintf("x/p%d", i), Test: "TestT", Cause: strings.Repeat("y", 500)})
|
||||
}
|
||||
v, _ := judgeLevel(t, failingResult(t, 60, fs...))
|
||||
if len(v.Failures) != delivery.MaxFailures+1 || v.Failures[delivery.MaxFailures].Test != "and 10 more" {
|
||||
t.Fatalf("%d entries, last %+v", len(v.Failures), v.Failures[len(v.Failures)-1])
|
||||
}
|
||||
if len(v.Failures[0].Words) > MaxFailureLine {
|
||||
t.Errorf("an entry's line is %d bytes", len(v.Failures[0].Words))
|
||||
}
|
||||
if n := failedCount(*v); n != 60 {
|
||||
t.Errorf("failedCount = %d, want 60", n)
|
||||
}
|
||||
cs := &Case{Pulls: []Pull{{Repository: "novox/x", Head: "0123456789abcdef", Trunk: "fedcba9876543210"}}}
|
||||
if line := Line(*v, cs, cs.Pulls[0], nil, nil, nil); !strings.HasPrefix(line, "failed (60 tests): integration: x/p0.TestT") {
|
||||
t.Errorf("line %q", line)
|
||||
}
|
||||
}
|
||||
|
||||
// A verdict that names no list (a conflict, a refused step, a verdict written before issue 1921) is one failed line, as
|
||||
// before.
|
||||
func TestAVerdictWithoutFailuresIsOneLine(t *testing.T) {
|
||||
v := delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "g14", Words: "files in conflict: go.mod\nmore"}}
|
||||
if got := failedLines(v); len(got) != 1 || got[0] != "g14: files in conflict: go.mod" {
|
||||
t.Fatalf("%q", got)
|
||||
}
|
||||
}
|
||||
@@ -116,6 +116,9 @@ type bench struct {
|
||||
fetch Fetched
|
||||
digest map[string]Digested // by level
|
||||
test func(c stepCall) (any, error)
|
||||
// declines is how many asks naming needs a node lacking them declines before shanks, which has every need, takes
|
||||
// one (issue 890); an ask naming none runs on shanks, which then has none.
|
||||
declines int
|
||||
}
|
||||
|
||||
// refsOf reads the siblings of a fetch's arguments, as they arrive in an ask (JSON) or as the checker gave them.
|
||||
@@ -151,8 +154,17 @@ func (b *bench) answer(a delivery.StepAsk) delivery.Step {
|
||||
if module == buildrun.GoBuildModule {
|
||||
steps.Digest = tool("digest")
|
||||
}
|
||||
out, err := buildstep.RunAsk(context.Background(), steps, a)
|
||||
began := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC)
|
||||
if len(a.Needs) > 0 {
|
||||
if b.declines > 0 {
|
||||
b.declines--
|
||||
result, _ := json.Marshal(delivery.StepDeclined{Declined: true, Needs: a.Needs, Lacks: a.Needs})
|
||||
return delivery.Step{ID: a.ID, Tool: a.Kind + " ask", Node: "ace", State: delivery.StepRefused, Started: began, Ended: began,
|
||||
Reason: "declined", Result: result}
|
||||
}
|
||||
steps.Has = func(string) bool { return true }
|
||||
}
|
||||
out, err := buildstep.RunAsk(context.Background(), steps, a)
|
||||
step := delivery.Step{ID: a.ID, Tool: a.Kind + " ask", Node: node, State: delivery.StepPassed, Started: began, Ended: began.Add(2 * time.Second)}
|
||||
if err != nil {
|
||||
step.State, step.Reason = delivery.StepFailed, err.Error()
|
||||
@@ -276,6 +288,49 @@ func TestOnlyThePackageWithoutAKeptVerdictRuns(t *testing.T) { // 1288/AC1
|
||||
if u := out.Verdict.Levels[delivery.LevelUnit]; u.Run != 2 || u.Kept != 40 {
|
||||
t.Errorf("unit = %+v, want 2 run and 40 kept", u)
|
||||
}
|
||||
|
||||
// Issue 890/AC5: a package needing node-build-node is left unmet where the node lacks it; the check keeps the passes
|
||||
// and asks the integration level again for a node with it, puts it again when a node declines it, and that node tests
|
||||
// the package alone, its kept passes not counted twice.
|
||||
// fixture/outside has no kept verdict after its pass (it reads outside itself: its digest moves every call), so a
|
||||
// placed ask that ran more than its unmet packages would run and count it again.
|
||||
b = newBench(t)
|
||||
b.declines = 1
|
||||
outside := 0
|
||||
b.steps.handlers["build_go_digest"] = func(c stepCall) (any, error) {
|
||||
d := echoBeside(b, b.digest[c.args["level"].(string)])
|
||||
if d.Level == "integration" {
|
||||
outside++
|
||||
d.Packages = map[string]string{"fixture/slow": digB, "fixture/node": digC, "fixture/outside": fmt.Sprintf("%064x", outside)}
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
b.digest[delivery.LevelIntegration] = Digested{Level: "integration",
|
||||
Needs: []delivery.PackageNeeds{{Package: "fixture/node", Needs: []string{delivery.NeedNodeBuildNode}}}}
|
||||
out = b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
|
||||
asks := b.asks.Asks()
|
||||
if out.Verdict.State != delivery.VerdictPass || len(asks) != 3 || len(asks[0].Needs) != 0 {
|
||||
t.Fatalf("verdict %+v, asks %+v: want a pass of three asks, the first naming no need", out.Verdict, asks)
|
||||
}
|
||||
for _, a := range asks[1:] {
|
||||
if strings.Join(a.Needs, ",") != delivery.NeedNodeBuildNode || strings.Join(a.Check.Levels, ",") != delivery.LevelIntegration || strings.Join(a.Check.Packages, ",") != "fixture/node" {
|
||||
t.Errorf("a placed ask %+v, want the integration level needing node-build-node", a)
|
||||
}
|
||||
}
|
||||
tests = b.steps.tool("test")
|
||||
if last := tests[len(tests)-1]; len(tests) != 3 || strings.Join(packagesOf(last), ",") != "fixture/node" || last.args["level"] != "integration" {
|
||||
t.Errorf("test calls = %+v, want the unit level, the integration level of fixture/slow and fixture/outside, then fixture/node alone", tests)
|
||||
}
|
||||
slow := captured[Tested](t, "go-test-slow")
|
||||
if i := out.Verdict.Levels[delivery.LevelIntegration]; i.Run != 2*slow.Tests || i.Kept != 0 {
|
||||
t.Errorf("integration = %+v, want %d run and nothing counted kept", i, 2*slow.Tests)
|
||||
}
|
||||
// And where no node has the need, the check fails once the declines outlast the bound, naming the need.
|
||||
b.kept, b.declines = newMemKV(), 1000
|
||||
b.ex.Kept, b.ex.DeclineBound = Kept{Store: b.kept}, 200*time.Millisecond
|
||||
if v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict; v.State != delivery.VerdictFail || !strings.Contains(fmt.Sprint(v), "needs a node with node-build-node") {
|
||||
t.Errorf("with no node having the need: %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReAskRunsOnlyWhatFailedAndWhatChanged(t *testing.T) { // 1288/AC2, AC3
|
||||
@@ -341,7 +396,9 @@ func TestANodePackageIsNeverKeptAndRunsEveryCheck(t *testing.T) { // 1288/AC4, 1
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheChangesOwnFailingTestRunsFirstAndNothingElseRuns(t *testing.T) { // 1289/AC1, AC2
|
||||
// 1289/AC1, AC2 as issue 1921 changed it: the change's own failing test runs first, the level's other packages still run
|
||||
// after it (so one check names every failure), and no later level runs.
|
||||
func TestTheChangesOwnFailingTestRunsFirstThenTheRestAndNoLaterLevel(t *testing.T) {
|
||||
b := newBench(t)
|
||||
b.fetch.Changed = []string{"modules/m/internal/b/b_test.go"}
|
||||
b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"m/internal/a": digA, "m/internal/b": digB}}
|
||||
@@ -357,8 +414,8 @@ func TestTheChangesOwnFailingTestRunsFirstAndNothingElseRuns(t *testing.T) { //
|
||||
t.Fatalf("verdict = %+v", out.Verdict)
|
||||
}
|
||||
tests := b.steps.tool("test")
|
||||
if len(tests) != 1 || strings.Join(packagesOf(tests[0]), ",") != "m/internal/b" {
|
||||
t.Fatalf("test calls = %+v, want only m/internal/b, once", tests)
|
||||
if len(tests) != 2 || strings.Join(packagesOf(tests[0]), ",") != "m/internal/b" || strings.Join(packagesOf(tests[1]), ",") != "m/internal/a" {
|
||||
t.Fatalf("test calls = %+v, want m/internal/b, then m/internal/a, each once", tests)
|
||||
}
|
||||
if n := len(b.steps.tool("digest")); n != 1 {
|
||||
t.Errorf("%d digest calls, want the unit level's alone (the check stopped)", n)
|
||||
@@ -618,9 +675,8 @@ func TestNothingIsKeptForAWorkspaceGivenSiblingsWhenTheDigestDoesNotSayItCoversT
|
||||
// Beside): every check now places the change's own repository at its trunk (issue 1887 A12).
|
||||
func echoBeside(b *bench, d Digested) Digested {
|
||||
var lines []string
|
||||
for _, f := range b.steps.tool("fetch") {
|
||||
refs, _ := refsOf(f.args["beside"]), true
|
||||
for _, r := range refs {
|
||||
if fetches := b.steps.tool("fetch"); len(fetches) > 0 { // the workspace digested is the last fetch's (issue 890: a placed ask fetches again)
|
||||
for _, r := range refsOf(fetches[len(fetches)-1].args["beside"]) {
|
||||
lines = append(lines, r.Repository+" "+r.Commit)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
@@ -88,29 +87,3 @@ func TestATrunkMoveMarksEveryOpenPullRequestIntoItPendingAtOnce(t *testing.T) {
|
||||
t.Errorf("mesh-sdk#13, alone into a trunk that did not move, is %q, want its success to stand", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestACheckThatEndsAfterItsTrunkMovedOwesPendingNotItsVerdict(t *testing.T) { // gate (a): a pass on an old trunk is never shown
|
||||
ctx := context.Background()
|
||||
var r *rig
|
||||
release, judged := make(chan struct{}), make(chan struct{})
|
||||
j := &judgeFunc{f: func(ctx context.Context, c *Case, call int) (Outcome, error) {
|
||||
if call == 1 {
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) // the trunk moves while it runs
|
||||
defer close(judged)
|
||||
} else {
|
||||
<-release // the check asked again for the new trunk waits in the stream meanwhile
|
||||
}
|
||||
return Outcome{Verdict: passing()}, nil
|
||||
}}
|
||||
r = newRig(t, j)
|
||||
r.ch.Candidate(ctx, "feat%2Fx", candidateOf(t, "feat/x", pullOf(repoA, 7, head1)))
|
||||
<-judged
|
||||
for deadline := time.Now().Add(2 * time.Second); owedState(t, r, repoA, 7, head1) == "" && time.Now().Before(deadline); time.Sleep(time.Millisecond) {
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
if s := owedState(t, r, repoA, 7, head1); s != "pending" {
|
||||
t.Errorf("the status is %q, want pending: the pass was judged on a trunk that moved", s)
|
||||
}
|
||||
close(release)
|
||||
r.ch.Wait()
|
||||
}
|
||||
|
||||
@@ -80,6 +80,9 @@ var askLanguage = map[string]string{buildrun.GoBuildModule: delivery.AskGo, buil
|
||||
// stopping at the first failure; it judges the step's answer as it judged the tools' answers.
|
||||
type Examiner struct {
|
||||
Asks Asks
|
||||
// DeclineBound is how long a placed ask may go on being declined, from its first decline, before the check fails
|
||||
// saying no node with its need took it (issue 890, ADR 0552); DefaultDeclineBound when zero.
|
||||
DeclineBound time.Duration
|
||||
// Toolchains are the nodes' toolchains: a kept verdict names the toolchain the nodes agree on.
|
||||
Toolchains ToolchainEntries
|
||||
Kept Kept
|
||||
@@ -203,12 +206,15 @@ type state struct {
|
||||
nodes map[string]bool
|
||||
clone time.Duration
|
||||
slow []string
|
||||
// beside are the siblings read at their trunk's head, by repository (ADR 0548 §6).
|
||||
beside map[string]string
|
||||
}
|
||||
|
||||
// Judge runs the case (it is the Judge of a Checker). A failure of the change, of a merge or of a step is a verdict; an error is only a case that cannot
|
||||
// be judged at all.
|
||||
func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) {
|
||||
st := &state{j: j, c: c, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{}}
|
||||
st := &state{j: j, c: c, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{},
|
||||
beside: map[string]string{}}
|
||||
if v := st.judgeAll(ctx); v != nil {
|
||||
return st.finish(ctx, *v), nil
|
||||
}
|
||||
@@ -230,6 +236,13 @@ func (st *state) finish(ctx context.Context, v delivery.Verdict) Outcome {
|
||||
// fails the check, naming the machine it failed at. A try is not composed (it has no candidate); a composition that
|
||||
// cannot be had leaves it not run, said; with no assigner running it is not judged, said at once.
|
||||
func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
|
||||
if st.c.Composition != nil && st.c.Try == nil {
|
||||
// A carry whose candidate's fingerprints did not change keeps the composition it passed (ADR 0548 §5).
|
||||
kept := *st.c.Composition
|
||||
v.Composition = &kept
|
||||
st.c.note("the composition is kept: no fingerprint of a module it composed changed")
|
||||
return
|
||||
}
|
||||
if st.j.Compositions == nil || st.c.Try != nil {
|
||||
return
|
||||
}
|
||||
@@ -243,6 +256,9 @@ func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
|
||||
return
|
||||
}
|
||||
withComposition(v, comp)
|
||||
if v.Composition != nil && v.Composition.State != delivery.CompositionNotRun {
|
||||
v.Composition.FPSet = st.c.FPSet // a carry keeps it while the fingerprints stay the same (ADR 0548 §5)
|
||||
}
|
||||
}
|
||||
|
||||
// outcome fills the verdict's levels, times, machines and composition from what ran.
|
||||
@@ -282,6 +298,9 @@ func (st *state) outcome(v delivery.Verdict) Outcome {
|
||||
}
|
||||
}
|
||||
out := Outcome{Verdict: v, Slowest: slow}
|
||||
if len(st.beside) > 0 {
|
||||
out.Beside = st.beside
|
||||
}
|
||||
for _, t := range st.targets {
|
||||
if t.judged != "" {
|
||||
if out.Judged == nil {
|
||||
@@ -343,6 +362,7 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
|
||||
if len(beside) > 0 {
|
||||
args["beside"] = beside
|
||||
t.beside = besideLines(beside)
|
||||
besideAtTrunk(st.c, beside, st.beside)
|
||||
}
|
||||
if p.Repository == catalogueRepository {
|
||||
t.runner = RunnerWorkspace // used by the build module when the merge changes it (ADR 0535 §1)
|
||||
@@ -364,11 +384,25 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
|
||||
return nil
|
||||
}
|
||||
|
||||
// DefaultDeclineBound is an Examiner's DeclineBound when it names none: longer than a check runs, so a node with the
|
||||
// need that is busy is waited for, and short enough that a mesh where no node has it says so.
|
||||
const DefaultDeclineBound = 30 * time.Minute
|
||||
|
||||
// ask puts the target's check ask, waits for the step of the node that pulled it and judges its answer: the fetch, the
|
||||
// pins of its go.mod files, then each level's digest, kept verdicts and tests.
|
||||
// pins of its go.mod files, then each level's digest, kept verdicts and tests. When the answer leaves packages unmet,
|
||||
// their tests needing what that node lacks (issue 890), the passes are kept and the levels from the first such one are
|
||||
// asked again of a node that has it (placed).
|
||||
func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *delivery.Verdict {
|
||||
return st.askNeeding(ctx, t, merged, st.levelsOf(), nil, nil)
|
||||
}
|
||||
|
||||
// askNeeding is ask with the levels to run, what the node must have and the packages it tests: none for the first ask
|
||||
// of a target; for the placed one the unmet packages' needs and those packages alone, so no package the first ask judged
|
||||
// is run or counted again (ADR 0552 §4).
|
||||
func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, levels, needs, packages []string) *delivery.Verdict {
|
||||
p := t.pull
|
||||
job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: st.levelsOf(), Kept: true}
|
||||
placed := len(needs) > 0
|
||||
job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: levels, Kept: true, Packages: packages}
|
||||
if t.runner != "" {
|
||||
job.Runner, job.RunnerWhen = t.runner, buildGoDir
|
||||
}
|
||||
@@ -380,17 +414,47 @@ func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *deliver
|
||||
kind = delivery.AskRecheck // a re-judge after a trunk move goes after every new head's check (ADR 0536 §4)
|
||||
}
|
||||
a := delivery.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: askLanguage[t.module], Asker: delivery.CheckerModule,
|
||||
At: st.j.now().UTC(), Check: job}
|
||||
st.c.Progress.Set(fmt.Sprintf("the %s ask %s of %s#%d", kind, a.ID, p.Repository, p.Number), "")
|
||||
if err := st.j.Asks.Put(ctx, a); err != nil {
|
||||
return fail(t.module, fmt.Sprintf("the check of %s#%d could not be asked: %v", p.Repository, p.Number, err))
|
||||
}
|
||||
step, err := st.j.Asks.Wait(ctx, a)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return fail(t.module, ctx.Err().Error())
|
||||
At: st.j.now().UTC(), Check: job, Needs: needs}
|
||||
var step delivery.Step
|
||||
var firstDecline time.Time
|
||||
for {
|
||||
what := fmt.Sprintf("the %s ask %s of %s#%d", kind, a.ID, p.Repository, p.Number)
|
||||
if placed {
|
||||
what += " for a node with " + strings.Join(needs, ", ")
|
||||
}
|
||||
return fail(t.module, fmt.Sprintf("the check of %s#%d: %v", p.Repository, p.Number, err))
|
||||
st.c.Progress.Set(what, "")
|
||||
st.c.Asked(a.ID)
|
||||
if err := st.j.Asks.Put(ctx, a); err != nil {
|
||||
return fail(t.module, fmt.Sprintf("the check of %s#%d could not be asked: %v", p.Repository, p.Number, err))
|
||||
}
|
||||
var err error
|
||||
step, err = st.j.Asks.Wait(ctx, a)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return fail(t.module, ctx.Err().Error())
|
||||
}
|
||||
return fail(t.module, fmt.Sprintf("the check of %s#%d: %v", p.Repository, p.Number, err))
|
||||
}
|
||||
d, declined := delivery.DeclinedOf(step)
|
||||
if !declined {
|
||||
break
|
||||
}
|
||||
// A node that lacks the need dropped the ask (issue 890): it is put again, for a node that has it to take; the
|
||||
// node that declined looks at the stream again only after its next pull's wait.
|
||||
now := st.j.now()
|
||||
if firstDecline.IsZero() {
|
||||
firstDecline = now
|
||||
}
|
||||
bound := st.j.DeclineBound
|
||||
if bound <= 0 {
|
||||
bound = DefaultDeclineBound
|
||||
}
|
||||
if now.Sub(firstDecline) > bound {
|
||||
return fail(step.Node, fmt.Sprintf("the check of %s#%d needs a node with %s, and none took it in %s: every node that pulled it lacks it (last %s)",
|
||||
p.Repository, p.Number, strings.Join(needs, ", "), bound, step.Node))
|
||||
}
|
||||
st.j.logf("%s#%d: the ask %s was declined on %s, which lacks %s; asked again", p.Repository, p.Number, a.ID, step.Node, strings.Join(d.Lacks, ", "))
|
||||
a.ID, a.At = buildstep.NewID(), now.UTC()
|
||||
}
|
||||
// The fetch and the digests are what of the ask's time its levels' running did not take (Took.Clone).
|
||||
took := time.Duration(0)
|
||||
@@ -414,18 +478,28 @@ func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *deliver
|
||||
if st.c.Try == nil {
|
||||
st.j.writePreview(ctx, p, append(*merged, t.fetched))
|
||||
}
|
||||
return fail(step.Node, fmt.Sprintf("%s#%d does not merge into the trunk: files in conflict: %s", p.Repository, p.Number, strings.Join(t.fetched.Conflicts, ", ")))
|
||||
return fail(step.Node, fmt.Sprintf("%s#%d %s%s", p.Repository, p.Number, ConflictWords, strings.Join(t.fetched.Conflicts, ", ")))
|
||||
}
|
||||
t.changed = t.fetched.Changed
|
||||
if !changesBuildGo(p.Repository, t.changed) {
|
||||
t.runner = "" // the build module used no runner: the merge does not change it
|
||||
}
|
||||
*merged = append(*merged, t.fetched)
|
||||
if v := st.checkPinsOf(ctx, t); v != nil {
|
||||
return v
|
||||
if !placed { // the placed ask fetched the same merge: its preview and pins were read from the first
|
||||
*merged = append(*merged, t.fetched)
|
||||
if v := st.checkPinsOf(ctx, t); v != nil {
|
||||
return v
|
||||
}
|
||||
}
|
||||
ran := 0.0
|
||||
for _, la := range answer.Levels {
|
||||
if placed {
|
||||
if len(la.Unmet) > 0 {
|
||||
st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
|
||||
return fail(step.Node, fmt.Sprintf("the %s level of %s#%d left %s untested on %s, which was asked for a node with %s",
|
||||
la.Level, p.Repository, p.Number, unmetNames(la.Unmet), step.Node, strings.Join(needs, ", ")))
|
||||
}
|
||||
la.Kept = nil // it names its packages and takes no other's kept verdict; this is only a guard against an older module
|
||||
}
|
||||
v, elapsed := st.level(ctx, t, la)
|
||||
ran += elapsed
|
||||
if v != nil {
|
||||
@@ -434,12 +508,52 @@ func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *deliver
|
||||
}
|
||||
}
|
||||
st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
|
||||
if unmet := delivery.UnmetNeeds(answer.Levels); len(unmet) > 0 && !placed {
|
||||
// Every level passed, its passes kept, and some packages were left to a node with what their tests need: the levels
|
||||
// from the first that left one are asked of such a node, which takes the kept passes and tests those (issue 890).
|
||||
var again, packages []string
|
||||
seen := map[string]bool{}
|
||||
for _, la := range answer.Levels {
|
||||
if len(again) > 0 || len(la.Unmet) > 0 {
|
||||
again = append(again, la.Level)
|
||||
}
|
||||
for _, u := range la.Unmet {
|
||||
if !seen[u.Package] {
|
||||
seen[u.Package] = true
|
||||
packages = append(packages, u.Package)
|
||||
}
|
||||
}
|
||||
}
|
||||
st.j.logf("%s#%d: %s on %s left untested; asked of a node with %s", p.Repository, p.Number, unmetOf(answer.Levels), step.Node, strings.Join(unmet, ", "))
|
||||
return st.askNeeding(ctx, t, merged, again, unmet, packages)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// unmetNames names the unmet packages and their needs.
|
||||
func unmetNames(unmet []delivery.PackageNeeds) string {
|
||||
var out []string
|
||||
for _, u := range unmet {
|
||||
out = append(out, u.Package+" (needs "+strings.Join(u.Needs, ", ")+")")
|
||||
}
|
||||
return strings.Join(out, ", ")
|
||||
}
|
||||
|
||||
// unmetOf names the unmet packages of every level.
|
||||
func unmetOf(levels []delivery.LevelAnswer) string {
|
||||
var out []string
|
||||
for _, la := range levels {
|
||||
if len(la.Unmet) > 0 {
|
||||
out = append(out, "at "+la.Level+" "+unmetNames(la.Unmet))
|
||||
}
|
||||
}
|
||||
return strings.Join(out, "; ")
|
||||
}
|
||||
|
||||
// level judges one level of a check ask's answer as the tools' answers were judged: the digest keys the kept verdicts
|
||||
// of the packages that passed, the kept ones taken are counted, and the first test result that did not pass is the
|
||||
// verdict. It answers the level's running time (its tests' and setup's).
|
||||
// of the packages that passed, the kept ones taken are counted, and every test result is read and counted, also after
|
||||
// one that did not pass, so the verdict names every failed test of the level (issue 1921). It answers the level's
|
||||
// running time (its tests' and setup's).
|
||||
func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer) (*delivery.Verdict, float64) {
|
||||
level := la.Level
|
||||
a := st.levels[level]
|
||||
@@ -461,6 +575,7 @@ func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer)
|
||||
a.kept += k.Tests
|
||||
}
|
||||
elapsed := 0.0
|
||||
var failedResults []Tested
|
||||
for _, raw := range la.Tests {
|
||||
var res Tested
|
||||
if err := json.Unmarshal(raw, &res); err != nil {
|
||||
@@ -489,9 +604,12 @@ func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer)
|
||||
}
|
||||
}
|
||||
if !res.Passed {
|
||||
return levelFailure(t, level, res), elapsed
|
||||
failedResults = append(failedResults, res)
|
||||
}
|
||||
}
|
||||
if len(failedResults) > 0 {
|
||||
return levelFailure(t, level, failedResults...), elapsed
|
||||
}
|
||||
return nil, elapsed
|
||||
}
|
||||
|
||||
@@ -633,23 +751,60 @@ func firstLine(s string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// failure turns a level that did not pass into a verdict naming what failed first.
|
||||
func levelFailure(t *target, level string, res Tested) *delivery.Verdict {
|
||||
// MaxFailureLine is the most bytes of a failure's line in a verdict's Failures: the first line of its words, which is
|
||||
// its cause where the build module told one apart (issue 1901). Failed keeps the first failure's words whole.
|
||||
const MaxFailureLine = 300
|
||||
|
||||
// levelFailure turns a level that did not pass into a verdict: Failed names what failed first with its words whole,
|
||||
// and Failures every failed test of the level's results with its first line, at most delivery.MaxFailures and an entry
|
||||
// saying how many more (issue 1921).
|
||||
func levelFailure(t *target, level string, results ...Tested) *delivery.Verdict {
|
||||
v := delivery.Verdict{State: delivery.VerdictFail}
|
||||
switch {
|
||||
case len(res.Failed) > 0:
|
||||
f := res.Failed[0]
|
||||
name := failedName(f)
|
||||
if t.module == buildrun.NodeBuildModule && f.Package == "" {
|
||||
name = res.Module
|
||||
var all []delivery.FailedFirst
|
||||
for _, res := range results {
|
||||
if len(res.Failed) == 0 {
|
||||
why := res.Why
|
||||
if why == "" {
|
||||
why = "the " + level + " level did not pass"
|
||||
}
|
||||
all = append(all, delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: why})
|
||||
continue
|
||||
}
|
||||
v.Failed = delivery.FailedFirst{Test: level + ": " + name, Words: failureWords(f)}
|
||||
default:
|
||||
why := res.Why
|
||||
if why == "" {
|
||||
why = "the " + level + " level did not pass"
|
||||
for _, f := range res.Failed {
|
||||
name := failedName(f)
|
||||
if t.module == buildrun.NodeBuildModule && f.Package == "" {
|
||||
name = res.Module
|
||||
}
|
||||
all = append(all, delivery.FailedFirst{Test: level + ": " + name, Words: failureWords(f)})
|
||||
}
|
||||
v.Failed = delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: why}
|
||||
}
|
||||
if len(all) == 0 { // no result: a level said failed with nothing to name
|
||||
all = append(all, delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: "the " + level + " level did not pass"})
|
||||
}
|
||||
v.Failed = all[0]
|
||||
v.Failures = failureLines(all)
|
||||
return &v
|
||||
}
|
||||
|
||||
// failureLines is every failure with the first line of its words, cut to MaxFailureLine, at most delivery.MaxFailures
|
||||
// and then one entry saying how many more failed.
|
||||
func failureLines(all []delivery.FailedFirst) []delivery.FailedFirst {
|
||||
n := min(len(all), delivery.MaxFailures)
|
||||
out := make([]delivery.FailedFirst, 0, n+1)
|
||||
for _, f := range all[:n] {
|
||||
out = append(out, delivery.FailedFirst{Test: f.Test, Machine: f.Machine, Words: clipLine(firstLine(f.Words), MaxFailureLine)})
|
||||
}
|
||||
if more := len(all) - n; more > 0 {
|
||||
out = append(out, delivery.FailedFirst{Test: fmt.Sprintf("and %d more", more)})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// clipLine cuts a line to at most n bytes on a whole character, marking the cut.
|
||||
func clipLine(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
s = strings.ToValidUTF8(s[:n-3], "")
|
||||
return s + "..."
|
||||
}
|
||||
|
||||
@@ -56,12 +56,22 @@ func (k Kept) Keep(ctx context.Context, level, digest string, v delivery.Kept) (
|
||||
|
||||
// KeepPasses keeps, for the packages of a test result that passed with tests, a verdict under the digest the build module
|
||||
// named for the package at the level. A package that failed, has no tests or has no digest is never kept (ADR 0444 clause
|
||||
// 1): it runs every check. It answers the packages it kept now.
|
||||
// 1): it runs every check. A package with skipped tests is not kept either: a pass that skipped (a test needing a
|
||||
// toolchain the node lacks) would be reused for its digest and never run anywhere (issue 890). A build module that
|
||||
// counts skips only per result, not per package, keeps nothing of a result in which a test skipped. It answers the
|
||||
// packages it kept now.
|
||||
func (k Kept) KeepPasses(ctx context.Context, repository, level, toolchain string, res Tested, digests map[string]string, at time.Time) ([]string, error) {
|
||||
var kept []string
|
||||
perPackage := false
|
||||
for _, p := range res.Packages {
|
||||
perPackage = perPackage || p.Skipped > 0
|
||||
}
|
||||
if res.Skipped > 0 && !perPackage {
|
||||
return kept, nil // an older build module: the skip cannot be placed in a package
|
||||
}
|
||||
for _, p := range res.Packages {
|
||||
digest, ok := digests[p.Package]
|
||||
if !ok || !p.Passed || p.NoTests || p.Tests == 0 {
|
||||
if !ok || !p.Passed || p.NoTests || p.Tests == 0 || p.Skipped > 0 {
|
||||
continue
|
||||
}
|
||||
if failedIn(res, p.Package) {
|
||||
|
||||
@@ -74,6 +74,32 @@ func TestOnlyPassingPackagesWithADigestAreKept(t *testing.T) { // 1287/AC3, AC4
|
||||
}
|
||||
}
|
||||
|
||||
func TestAResultWithSkippedTestsKeepsNothing(t *testing.T) { // issue 890: a pass that skipped proved nothing
|
||||
ctx, st := context.Background(), newMemKV()
|
||||
k := Kept{Store: st}
|
||||
res := Tested{Level: "integration", Passed: true, Skipped: 2, Packages: []GoPackage{{Package: "m/a", Passed: true, Tests: 4}}}
|
||||
kept, err := k.KeepPasses(ctx, "novox/mesh-catalog", "integration", "go1.26.0", res, map[string]string{"m/a": digA}, at0)
|
||||
if err != nil || len(kept) != 0 {
|
||||
t.Fatalf("kept %v, %v, want nothing", kept, err)
|
||||
}
|
||||
if keys, _ := st.Keys(ctx); len(keys) != 0 {
|
||||
t.Errorf("the bucket holds %v, want none", keys)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyTheSkippingPackageOfAResultIsWithheld(t *testing.T) { // issue 890
|
||||
ctx, st := context.Background(), newMemKV()
|
||||
k := Kept{Store: st}
|
||||
res := Tested{Level: "integration", Passed: true, Skipped: 2, Packages: []GoPackage{
|
||||
{Package: "m/a", Passed: true, Tests: 4, Skipped: 2},
|
||||
{Package: "m/b", Passed: true, Tests: 3},
|
||||
}}
|
||||
kept, err := k.KeepPasses(ctx, "novox/mesh-catalog", "integration", "go1.26.0", res, map[string]string{"m/a": digA, "m/b": digB}, at0)
|
||||
if err != nil || strings.Join(kept, ",") != "m/b" {
|
||||
t.Fatalf("kept %v, %v, want only m/b", kept, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAKeptValueOfAnotherDigestIsNotRead(t *testing.T) { // 1287/AC7
|
||||
ctx, st := context.Background(), newMemKV()
|
||||
k := Kept{Store: st}
|
||||
|
||||
@@ -49,6 +49,34 @@ func firstFailed(v delivery.Verdict) string {
|
||||
return firstWordsLine(f.Words)
|
||||
}
|
||||
|
||||
// failedLines is everything a verdict names as failed, one line each: every failed test with its first line (issue
|
||||
// 1921), or what failed first when the verdict names no list (a conflict, a refused step, an older verdict).
|
||||
func failedLines(v delivery.Verdict) []string {
|
||||
if len(v.Failures) == 0 {
|
||||
if f := firstFailed(v); f != "" {
|
||||
return []string{f}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
lines := make([]string, 0, len(v.Failures))
|
||||
for _, f := range v.Failures {
|
||||
lines = append(lines, firstFailed(delivery.Verdict{Failed: f}))
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
// failedCount is how many tests a verdict names as failed, the ones after delivery.MaxFailures counted too.
|
||||
func failedCount(v delivery.Verdict) int {
|
||||
n := len(v.Failures)
|
||||
if n > delivery.MaxFailures {
|
||||
var more int
|
||||
if _, err := fmt.Sscanf(v.Failures[n-1].Test, "and %d more", &more); err == nil {
|
||||
return delivery.MaxFailures + more
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// firstWordsLine is the first line of a failure's words: the words are several lines (issue 1901) and the status line one.
|
||||
func firstWordsLine(w string) string { return firstLine(w) }
|
||||
|
||||
@@ -57,18 +85,20 @@ type pieces struct {
|
||||
head, commits, counts, shortCounts, times, shortTimes, over, builds, composition, shortComposition, group string
|
||||
// judged names the program that judged a change to a build module (ADR 0535 §5); never dropped.
|
||||
judged string
|
||||
// carried says, of a carried verdict, the commits it was carried from (ADR 0548 §8); never dropped.
|
||||
carried string
|
||||
}
|
||||
|
||||
// pullKey names a pull request of a case: `<owner/repo>#<number>`.
|
||||
func pullKey(p Pull) string { return fmt.Sprintf("%s#%d", p.Repository, p.Number) }
|
||||
|
||||
func (p pieces) join(drop int) string {
|
||||
parts := []string{p.head, p.commits, p.counts, p.judged}
|
||||
parts := []string{p.head, p.commits, p.carried, p.counts, p.judged}
|
||||
switch {
|
||||
case drop < 3 && p.times != "":
|
||||
parts = append(parts, p.times)
|
||||
case drop == 3 && p.shortTimes != "":
|
||||
parts = []string{p.head, p.commits, p.shortCounts, p.judged, p.shortTimes}
|
||||
parts = []string{p.head, p.commits, p.carried, p.shortCounts, p.judged, p.shortTimes}
|
||||
}
|
||||
if p.over != "" {
|
||||
parts = append(parts, p.over)
|
||||
@@ -143,6 +173,39 @@ func buildWords(builds []BuildNote) string {
|
||||
return "builds: " + strings.Join(parts, ", ")
|
||||
}
|
||||
|
||||
// carriedWords say the commits a verdict was carried from, for the line of pull request p (ADR 0548 §8): its own trunk's
|
||||
// commit as twelve characters, a sibling's with its name before it; only those that moved, sorted. Empty when none did.
|
||||
func carriedWords(from map[string]string, v delivery.Verdict, cs *Case, p Pull) string {
|
||||
var own string
|
||||
var others []string
|
||||
for r, old := range from {
|
||||
now, ok := v.Trunk[r]
|
||||
if cs != nil && !ok {
|
||||
now, ok = cs.Trunk[r]
|
||||
}
|
||||
if !ok {
|
||||
now = v.Beside[r]
|
||||
}
|
||||
if now == "" || now == old {
|
||||
continue
|
||||
}
|
||||
if strings.EqualFold(r, p.Repository) {
|
||||
own = short12(old)
|
||||
continue
|
||||
}
|
||||
name := r
|
||||
if i := strings.LastIndex(r, "/"); i >= 0 {
|
||||
name = r[i+1:]
|
||||
}
|
||||
others = append(others, name+" "+short12(old))
|
||||
}
|
||||
slices.Sort(others)
|
||||
if own != "" {
|
||||
others = append([]string{own}, others...)
|
||||
}
|
||||
return strings.Join(others, ", ")
|
||||
}
|
||||
|
||||
// parts makes the pieces of the line of a pull request of a case.
|
||||
func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string, judged map[string]string) pieces {
|
||||
var pc pieces
|
||||
@@ -151,10 +214,23 @@ func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []s
|
||||
switch v.State {
|
||||
case delivery.VerdictRunning:
|
||||
pc.head = "checking " + commits
|
||||
if cs != nil && cs.CarriedFrom != nil {
|
||||
// Out of date and being carried to the new commit (ADR 0548 §6): pending, and saying so.
|
||||
pc.head = "being carried to " + commits
|
||||
if w := carriedWords(cs.CarriedFrom, v, cs, p); w != "" {
|
||||
pc.carried = "from " + w
|
||||
}
|
||||
}
|
||||
case delivery.VerdictPass:
|
||||
pc.head = "pass " + commits
|
||||
if w := carriedWords(v.CarriedFrom, v, cs, p); w != "" {
|
||||
pc.carried = "carried from " + w // the counts that follow say what the carry ran and kept
|
||||
}
|
||||
default:
|
||||
pc.head = "failed: " + firstFailed(v)
|
||||
if n := failedCount(v); n > 1 {
|
||||
pc.head = fmt.Sprintf("failed (%d tests): %s", n, firstFailed(v))
|
||||
}
|
||||
pc.commits = commits
|
||||
}
|
||||
var counts []string
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// **An ask the build seat gave up on ends its step** (novox/hq issue 734): mesh-sdk buildrun.LostAskWaiter hands the
|
||||
// waiting asker a failed step on the node buildrun.LostNode when DEAD_LETTERS keeps the ask's copy (buildrun.HearDeadLetters,
|
||||
// granted by buscompose DeadLetterReads), and, under buildrun.NotTakenBound, when no build node took the ask within the
|
||||
// bound. What follows is the checker's: a try ends as error (Tryer.run), with the not-taken bound at the step bound
|
||||
// (Tryer.NotTaken); a check is asked again under its claim, at most LostAskTries times, then fails as not heard and is
|
||||
// judged again when the nodes change (Checker.try). A check's ask has no not-taken bound: it waits in the stream as long as
|
||||
// the backlog is (ADR 0525 §9), and a check ended for a backlog would owe its pull request a failure.
|
||||
|
||||
// LostAskTries is how often a check asks again whose ask was given up, before it fails as not heard (UnheardWords) and is
|
||||
// judged again when the nodes change.
|
||||
const LostAskTries = 3
|
||||
|
||||
// IsLostAsk says whether a verdict failed because the build seat gave its ask up (buildrun.LostWords).
|
||||
func IsLostAsk(v delivery.Verdict) bool {
|
||||
return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, buildrun.LostWords)
|
||||
}
|
||||
|
||||
// IsNotTaken says whether a verdict failed because no build node took its ask within the bound (buildrun.NotTakenWords).
|
||||
func IsNotTaken(v delivery.Verdict) bool {
|
||||
return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, buildrun.NotTakenWords)
|
||||
}
|
||||
|
||||
// HearingBucket is the checker's state hearing: whether it hears DEAD_LETTERS (buildrun.Hearing under
|
||||
// buildrun.HearingKey), which checker_status says (dead_letters).
|
||||
const HearingBucket = delivery.CheckerModule + "_" + buildrun.HearingState
|
||||
@@ -0,0 +1,170 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// losingAsks is the build seat over a bus that gives up the first `lose` asks put (DEAD_LETTERS keeps them, as on
|
||||
// 2026-10-26 during a bus reload: try-1793017340003993788) and answers the rest; with never set it neither answers nor
|
||||
// gives up (an ask no node takes).
|
||||
type losingAsks struct {
|
||||
*buildrun.FakeAsks
|
||||
answer func(delivery.StepAsk) delivery.Step
|
||||
dead *buildrun.DeadLetters
|
||||
never bool
|
||||
|
||||
mu sync.Mutex
|
||||
lose int
|
||||
}
|
||||
|
||||
func (l *losingAsks) PublishAsk(ctx context.Context, subject, id string, data []byte) error {
|
||||
if err := l.FakeAsks.PublishAsk(ctx, subject, id, data); err != nil {
|
||||
return err
|
||||
}
|
||||
l.mu.Lock()
|
||||
lost := l.lose > 0
|
||||
if lost {
|
||||
l.lose--
|
||||
}
|
||||
l.mu.Unlock()
|
||||
switch {
|
||||
case l.never:
|
||||
case lost:
|
||||
h := map[string][]string{buildrun.DeadHeaderMsgID: {id}, buildrun.DeadHeaderDeliveries: {"5"}, buildrun.DeadHeaderStream: {"SEAT_NODE_BUILD_GO"},
|
||||
buildrun.DeadHeaderGaveUp: {"2026-10-26T12:26:40Z"}}
|
||||
g, ok := buildrun.GivenUpOf(h)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
go l.dead.Heard(g)
|
||||
default:
|
||||
a, err := delivery.DecodeStepAsk(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
l.Write(l.answer(a))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func losing(b *bench, lose int) *losingAsks {
|
||||
l := &losingAsks{FakeAsks: &buildrun.FakeAsks{}, answer: b.answer, dead: &buildrun.DeadLetters{}, lose: lose}
|
||||
b.ex.Asks = buildrun.Asker{Publisher: l, Waiter: buildrun.LostAskWaiter{Inner: l.FakeAsks, Dead: l.dead}, Bound: 5 * time.Second}
|
||||
return l
|
||||
}
|
||||
|
||||
// endable is the tries state as the end tool sees it.
|
||||
type endable struct{ kv *memKV }
|
||||
|
||||
func (e endable) Entry(key string) (json.RawMessage, uint64, error) {
|
||||
v, rev, _, err := e.kv.Get(context.Background(), key)
|
||||
return v, rev, err
|
||||
}
|
||||
func (e endable) Update(key string, value []byte, revision uint64) error {
|
||||
_, err := e.kv.Update(context.Background(), key, value, revision)
|
||||
return err
|
||||
}
|
||||
|
||||
// Issue 734: a try whose ask the build seat gave up on ends as error saying so, and the next ask of its head is a new try;
|
||||
// one no node takes ends at the step bound; one ended by hand frees its place and stops being waited for.
|
||||
func TestAnAskGivenUpOrNeverTakenEndsItsTryAsErrorAndItsHeadIsAskedAnew(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
lose int
|
||||
never bool
|
||||
want string
|
||||
}{
|
||||
{"given up by the build seat", 1, false, buildrun.LostWords + " after 5 deliveries at 2026-10-26T12:26:40Z"},
|
||||
{"taken by no build node within the bound", 0, true, buildrun.NotTakenWords + " within 200ms"},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
kv := newMemKV()
|
||||
at := time.Date(2026, 10, 26, 12, 22, 20, 0, time.UTC)
|
||||
id, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, at)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := newBench(t)
|
||||
l := losing(b, row.lose)
|
||||
l.never = row.never
|
||||
tr := &Tryer{Tries: kv, Judge: b.ex, NotTaken: 200 * time.Millisecond}
|
||||
raw, _, _, _ := kv.Get(ctx, id)
|
||||
tr.Put(ctx, id, raw)
|
||||
tr.Wait()
|
||||
got := readTry(t, kv, id)
|
||||
if got.State != TryError || len(got.Failed) != 1 || !strings.Contains(got.Failed[0], row.want) || !strings.Contains(got.Said, row.want) {
|
||||
t.Fatalf("try = %+v, want error saying %q", got, row.want)
|
||||
}
|
||||
again, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, at.Add(time.Minute))
|
||||
if err != nil || again == id {
|
||||
t.Errorf("the head asked again answered %q, %v, want a new try", again, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("ended by hand", func(t *testing.T) {
|
||||
kv := newMemKV()
|
||||
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now())
|
||||
b := newBench(t)
|
||||
losing(b, 0).never = true
|
||||
tr := &Tryer{Tries: kv, Judge: b.ex} // no bound: only the hand ends it
|
||||
raw, _, _, _ := kv.Get(ctx, id)
|
||||
tr.Put(ctx, id, raw)
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for readTry(t, kv, id).State != TryRunning && time.Now().Before(deadline) {
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
if _, err := EndTry(endable{kv}, id, "", time.Now()); err == nil {
|
||||
t.Error("an end without a why was taken")
|
||||
}
|
||||
ended, err := EndTry(endable{kv}, id, "its ask was lost in the bus reload", time.Now())
|
||||
if err != nil || ended.State != TryError || !strings.HasPrefix(ended.Said, "ended by hand ("+TryEndTool+"): its ask was lost") {
|
||||
t.Fatalf("ended = %+v, %v", ended, err)
|
||||
}
|
||||
raw, _, _, _ = kv.Get(ctx, id)
|
||||
tr.Put(ctx, id, raw) // the watch of the tries hands the end over
|
||||
tr.Wait() // returns: the try's judging stopped
|
||||
if got := readTry(t, kv, id); got.State != TryError || got.Said != ended.Said {
|
||||
t.Errorf("try = %+v, want the hand's end kept", got)
|
||||
}
|
||||
if _, err := EndTry(endable{kv}, id, "again", time.Now()); err == nil || !strings.Contains(err.Error(), "ended already") {
|
||||
t.Errorf("a second end answered %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Issue 734: a pull request's check whose ask the build seat gave up on is asked again under its claim, its status
|
||||
// pending, and judged; one given up every time fails as not heard and is judged again when the nodes change.
|
||||
func TestACheckWhoseAskWasGivenUpIsAskedAgain(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
for _, row := range []struct {
|
||||
lose int
|
||||
state string
|
||||
asks int
|
||||
}{
|
||||
{1, delivery.VerdictPass, 2},
|
||||
{LostAskTries, delivery.VerdictFail, LostAskTries},
|
||||
} {
|
||||
b := newBench(t)
|
||||
l := losing(b, row.lose)
|
||||
r := newRig(t, b.ex)
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
r.ch.Wait()
|
||||
v := r.verdict(t, keyOf(t, "feat/x", cand))
|
||||
if v.State != row.state || len(l.Asks()) != row.asks {
|
||||
t.Errorf("lose %d: verdict %s (%s) after %d asks, want %s after %d", row.lose, v.State, v.Failed.Words, len(l.Asks()), row.state, row.asks)
|
||||
}
|
||||
if row.state == delivery.VerdictFail && (!IsLostAsk(v) || !IsNodesFailure(v)) {
|
||||
t.Errorf("a check given up every time = %q, want lost and judged again when the nodes change", v.Failed.Words)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -240,7 +240,12 @@ func TestTheSiblingsAreWhereTheTestsStateThem(t *testing.T) { // review D2, ADR
|
||||
catalogue + " go.mod " + head2: "module git.novox.be/novox/mesh-catalog\n"}, onTrunk: map[string]bool{"cb54756c3865": true}}
|
||||
b.ex.Forge = forge
|
||||
// A group of the controller and the catalogue: the controller's tests read the catalogue's trunk, not the group's head.
|
||||
b.judge(t, caseOf(t, pullOf(controller, 7, head1), pullOf(catalogue, 12, head2)))
|
||||
out := b.judge(t, caseOf(t, pullOf(controller, 7, head1), pullOf(catalogue, 12, head2)))
|
||||
// ADR 0548 §6, issue 568's own example: the verdict names the node-engine at the commit the mesh runs, and mesh-sdk at the
|
||||
// trunk the catalogue's tests read it at; the catalogue, a member here, is not named.
|
||||
if out.Beside[host] != hostRuns || out.Beside[catalogue] != "" || out.Beside[SDKRepository] != "9999999999999999999999999999999999999999" {
|
||||
t.Errorf("the siblings named for the verdict = %v, want the node-engine at %s and mesh-sdk at its trunk", out.Beside, hostRuns[:12])
|
||||
}
|
||||
got := besideOf(t, b, controller)
|
||||
if c := got[catalogue]; c.Commit != trunkS || c.Ref != "" {
|
||||
t.Errorf("the catalogue beside the controller = %+v, want its trunk %s", c, trunkS)
|
||||
@@ -324,7 +329,11 @@ func TestTheChangesOwnRepositoryIsBesideItAtItsTrunk(t *testing.T) {
|
||||
func TestACatalogueChangeHasTheCatalogueBesideItAtItsTrunk(t *testing.T) {
|
||||
b := newBench(t)
|
||||
b.ex.Trunks = trunksOf(t, map[string]string{catalogueRepository: trunkA, SDKRepository: trunkS, controllerRepo: trunkA2})
|
||||
b.judge(t, caseOf(t, pullOf(catalogueRepository, 12, head1)))
|
||||
out := b.judge(t, caseOf(t, pullOf(catalogueRepository, 12, head1)))
|
||||
// ADR 0548 §6: the outcome names the siblings read at their trunk's head, never the repository it merges.
|
||||
if out.Beside[SDKRepository] != trunkS || out.Beside[controllerRepo] != trunkA2 || out.Beside[catalogueRepository] != "" {
|
||||
t.Errorf("the siblings named for the verdict = %v, want mesh-sdk and mesh-controller at their trunks and not the catalogue", out.Beside)
|
||||
}
|
||||
got := besideOf(t, b, catalogueRepository)
|
||||
if c := got[catalogueRepository]; c.Commit != trunkA || c.Ref != "" {
|
||||
t.Fatalf("the catalogue beside its own change = %+v, want its trunk %s", c, trunkA)
|
||||
|
||||
@@ -75,3 +75,40 @@ func TestCheckerStatusNamesTheWaitingChecksByKind(t *testing.T) { // 1897/AC4
|
||||
t.Errorf("waiting = %v, want %v", st.Waiting, want)
|
||||
}
|
||||
}
|
||||
|
||||
// 568 note 21: a verdict that read the node-engine at its running commit is not waiting for a trunk move while that commit
|
||||
// is still the one the mesh runs, though the node-engine's trunk has moved on (ADR 0548 §6).
|
||||
func TestCheckerStatusReadsTheNodeEngineAtItsRunningCommit(t *testing.T) { // 568/21
|
||||
cand := candidateOf(t, group(1), pullOf(repoA, 1, head1))
|
||||
ckey, _ := delivery.CandidatesKey(group(1))
|
||||
vkey := keyOf(t, group(1), cand)
|
||||
dc, _ := delivery.DecodeCandidate(cand)
|
||||
judged, err := delivery.EncodeVerdict(delivery.Verdict{Group: group(1), Heads: dc.HeadsHash, Trunk: map[string]string{repoA: trunkA}, Beside: map[string]string{nodeEngineRepo: "run111"}, State: delivery.VerdictPass, Levels: passing().Levels})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
heads := func() (map[string]string, error) {
|
||||
return map[string]string{repoA: trunkA, nodeEngineRepo: "trunk222"}, nil
|
||||
}
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
runs runsAt
|
||||
want int
|
||||
}{
|
||||
{"the running commit the verdict read", runsAt{nodeEngineModule: "run111"}, 0},
|
||||
{"the running commit moved", runsAt{nodeEngineModule: "run999"}, 1},
|
||||
{"no running commit known: the trunk's", nil, 1},
|
||||
} {
|
||||
rd := Reader{CandidateState: fakeState{ckey: cand}.read(), VerdictState: fakeState{vkey: judged}.read(), Trunks: heads}
|
||||
if row.runs != nil {
|
||||
rd.Runs = row.runs
|
||||
}
|
||||
st, err := rd.Status()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := len(st.Waiting[KindTrunk]); got != row.want {
|
||||
t.Errorf("%s: waiting for a trunk move = %v, want %d", row.name, st.Waiting[KindTrunk], row.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// The read tools (novox/hq ADR 0484 clauses 8 and 13, issue 1686): what the checker stored, as stored, with the line of the
|
||||
@@ -39,7 +40,14 @@ type Reader struct {
|
||||
StepBound time.Duration // how long the checker waits for one step; DefaultStepBound when 0
|
||||
// Trunks are the trunk heads of every repository (the gate's source); nil names no re-judge after a trunk move.
|
||||
Trunks func() (map[string]string, error)
|
||||
Now func() time.Time
|
||||
// TurnState is the gate's merge turns (ADR 0548 §3); without it no check is named as held by a turn.
|
||||
TurnState StateRead
|
||||
// Runs says which commit the mesh runs of a module (the indexer's modules), as the checker's decision reads it: the
|
||||
// node-engine a verdict names is compared at its running commit, not at the trunk's (ADR 0548 §6); nil knows none.
|
||||
Runs interface{ CommitOf(module string) string }
|
||||
// HearingState is the checker's state hearing: whether it hears DEAD_LETTERS (issue 734).
|
||||
HearingState StateRead
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// VerdictRead is a verdict as stored with the full report of the status of each pull request it judged (Report), when the
|
||||
@@ -312,7 +320,68 @@ type Status struct {
|
||||
// Waiting are the candidates whose check waits, by kind (ADR 0536 §4): new (no verdict for its heads) and nodes (failed
|
||||
// for the nodes, asked again when they change) start before trunk (judged on trunk heads that moved since).
|
||||
Waiting map[string][]string `json:"waiting_by_kind"`
|
||||
Notes []string `json:"notes,omitempty"`
|
||||
// HeldByTurn are the candidates whose check an approved merge's turn holds off (ADR 0548 §3), by group: the repository and
|
||||
// the group of the merge that waits. Their check is claimed when that turn goes.
|
||||
HeldByTurn map[string]string `json:"held_by_turn"`
|
||||
// DeadLetters says whether the checker hears the asks the build seats give up on (issue 734): deaf, a try or check
|
||||
// whose ask is given up waits until its bound.
|
||||
DeadLetters DeadLettersHeard `json:"dead_letters"`
|
||||
Notes []string `json:"notes,omitempty"`
|
||||
}
|
||||
|
||||
// DeadLettersHeard is the checker's hearing of DEAD_LETTERS as the status says it: healthy or not, and why.
|
||||
type DeadLettersHeard struct {
|
||||
Heard bool `json:"heard"`
|
||||
Why string `json:"why"`
|
||||
}
|
||||
|
||||
// deadLetters reads the checker's kept hearing (buildrun.HearingHealth).
|
||||
func (r Reader) deadLetters() DeadLettersHeard {
|
||||
if r.HearingState.Get == nil {
|
||||
return DeadLettersHeard{Why: "the checker's state hearing is not readable here"}
|
||||
}
|
||||
raw, err := r.HearingState.Get(buildrun.HearingKey)
|
||||
if err != nil {
|
||||
return DeadLettersHeard{Why: "the checker's state hearing cannot be read: " + err.Error()}
|
||||
}
|
||||
a := buildrun.HearingHealth(raw)
|
||||
return DeadLettersHeard{Heard: a.Healthy, Why: a.Why}
|
||||
}
|
||||
|
||||
// turns reads the gate's merge turns, by repository; nil when they cannot be read, said in a note.
|
||||
func (r Reader) turns(st *Status) *Turns {
|
||||
if r.TurnState.Keys == nil || r.TurnState.Get == nil {
|
||||
st.Notes = append(st.Notes, "the gate's merge turns are not readable here, so no check is named as held by one")
|
||||
return nil
|
||||
}
|
||||
keys, err := r.TurnState.Keys()
|
||||
if err != nil {
|
||||
st.Notes = append(st.Notes, "the gate's merge turns cannot be listed, so no check is named as held by one: "+err.Error())
|
||||
return nil
|
||||
}
|
||||
t := &Turns{}
|
||||
for _, k := range keys {
|
||||
if raw, err := r.TurnState.Get(k); err == nil && len(raw) > 0 {
|
||||
t.Put(k, raw, false)
|
||||
}
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// heldBy says the turn that holds a candidate's check off, or "".
|
||||
func heldBy(t *Turns, c delivery.Candidate, heads map[string]string) string {
|
||||
cs := CaseOf(c, heads)
|
||||
if t == nil || cs == nil {
|
||||
return ""
|
||||
}
|
||||
repos := make([]string, 0, len(cs.Trunk))
|
||||
for r := range cs.Trunk {
|
||||
repos = append(repos, r)
|
||||
}
|
||||
if repo, w, ok := t.Holding(repos, cs.Heads); ok {
|
||||
return repo + ": the approved merge of " + w.Group + " waits on it"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Status reads the build modules' steps and the candidates against their verdicts.
|
||||
@@ -383,6 +452,8 @@ func (r Reader) Status() (Status, error) {
|
||||
}
|
||||
st.Unjudged = []string{}
|
||||
st.Waiting = map[string][]string{KindNew: {}, KindNodes: {}, KindTrunk: {}}
|
||||
st.HeldByTurn = map[string]string{}
|
||||
turns := r.turns(&st)
|
||||
var heads map[string]string
|
||||
if r.Trunks == nil {
|
||||
st.Notes = append(st.Notes, "the trunk heads are not readable here, so no check is named as waiting for a trunk move")
|
||||
@@ -435,6 +506,9 @@ func (r Reader) Status() (Status, error) {
|
||||
if len(raw) == 0 {
|
||||
st.Unjudged = append(st.Unjudged, c.Group)
|
||||
st.Waiting[KindNew] = append(st.Waiting[KindNew], c.Group)
|
||||
if why := heldBy(turns, c, heads); why != "" {
|
||||
st.HeldByTurn[c.Group] = why
|
||||
}
|
||||
continue
|
||||
}
|
||||
v, err := delivery.DecodeVerdict(raw)
|
||||
@@ -443,12 +517,16 @@ func (r Reader) Status() (Status, error) {
|
||||
case IsNodesFailure(v):
|
||||
st.Waiting[KindNodes] = append(st.Waiting[KindNodes], c.Group)
|
||||
case heads != nil:
|
||||
if cs := CaseOf(c, heads); cs != nil && !sameTrunk(v.Trunk, cs.Trunk) {
|
||||
if cs := CaseOf(c, heads); cs != nil && (!sameTrunk(v.Trunk, cs.Trunk) || besideMoved(v.Beside, besideAt(heads, r.Runs))) {
|
||||
st.Waiting[KindTrunk] = append(st.Waiting[KindTrunk], c.Group)
|
||||
if why := heldBy(turns, c, heads); why != "" {
|
||||
st.HeldByTurn[c.Group] = why
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
st.DeadLetters = r.deadLetters()
|
||||
st.Notes = append(st.Notes, "nothing holds the checker and it holds no place: running_steps are the build modules' keys that can still be running (started within the step bound, their build module running on the node); a candidate without a verdict has its check ask in a build seat's stream, waiting for a node with a free slot, whose backlog the delivery view shows (ADR 0525 §4, §9)")
|
||||
return st, nil
|
||||
}
|
||||
|
||||
@@ -144,7 +144,10 @@ func TestTheStatusToolReadsRunningSteps(t *testing.T) {
|
||||
CandidateState: fakeState{ckey: cand}.read(),
|
||||
VerdictState: fakeState{}.read(),
|
||||
HaveState: fakeState{"shanks.mesh-build-go": haveOf(t, delivery.PhaseRunning)}.read(),
|
||||
Now: func() time.Time { return time.Date(2026, 10, 22, 15, 10, 0, 0, time.UTC) },
|
||||
HearingState: fakeState{"dead-letters": []byte(`{"heard":false,"streams":["SEAT_NODE_BUILD_GO"],` +
|
||||
`"why":"DEAD_LETTERS is not heard, so an ask a build seat gives up on is waited for until its bound: SEAT_NODE_BUILD_GO: permissions violation",` +
|
||||
`"at":"2026-10-26T13:00:00Z"}`)}.read(),
|
||||
Now: func() time.Time { return time.Date(2026, 10, 22, 15, 10, 0, 0, time.UTC) },
|
||||
}
|
||||
st, err := rd.Status()
|
||||
if err != nil {
|
||||
@@ -156,9 +159,13 @@ func TestTheStatusToolReadsRunningSteps(t *testing.T) {
|
||||
if len(st.Unjudged) != 1 || st.Unjudged[0] != "feat/x" {
|
||||
t.Errorf("unjudged = %v", st.Unjudged)
|
||||
}
|
||||
// Issue 734, review round 1: a checker that does not hear DEAD_LETTERS says so in its status, with why.
|
||||
if st.DeadLetters.Heard || !strings.Contains(st.DeadLetters.Why, "permissions violation") {
|
||||
t.Errorf("dead letters = %+v, want not heard, saying why", st.DeadLetters)
|
||||
}
|
||||
// What cannot be read is said, not guessed.
|
||||
st, err = Reader{}.Status()
|
||||
if err != nil || len(st.Notes) == 0 {
|
||||
if err != nil || len(st.Notes) == 0 || st.DeadLetters.Heard {
|
||||
t.Errorf("empty status = %+v, %v", st, err)
|
||||
}
|
||||
}
|
||||
@@ -234,6 +241,18 @@ func TestTheStatusCountsAnHQOnlyGroupAsUnjudged(t *testing.T) {
|
||||
if len(st.Unjudged) != 1 || st.Unjudged[0] != "pb-x" {
|
||||
t.Errorf("unjudged = %v, want pb-x", st.Unjudged)
|
||||
}
|
||||
// ADR 0548 §3: a candidate whose check an approved merge's turn holds off is named with the turn.
|
||||
held := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
hkey, _ := delivery.CandidatesKey("feat/x")
|
||||
st, err = Reader{CandidateState: fakeState{hkey: held}.read(), VerdictState: fakeState{}.read(),
|
||||
Trunks: func() (map[string]string, error) { return map[string]string{repoA: trunkA}, nil },
|
||||
TurnState: fakeState{turnKey(t, repoA): turnOf(t, repoA, "fix/other", "0123456789ab")}.read()}.Status()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if why := st.HeldByTurn["feat/x"]; !strings.Contains(why, repoA) || !strings.Contains(why, "fix/other") {
|
||||
t.Errorf("held_by_turn = %v, want feat/x held by the merge of fix/other on %s", st.HeldByTurn, repoA)
|
||||
}
|
||||
}
|
||||
|
||||
// slowBucket wraps a state with a Get that takes delay and counts the values read, the lists and the most Gets in flight.
|
||||
|
||||
@@ -3,6 +3,8 @@ package check
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The shapes of what the build modules' steps answer, as their code writes them (mesh-catalog modules/mesh-build-go
|
||||
@@ -52,6 +54,8 @@ type GoPackage struct {
|
||||
Elapsed float64 `json:"elapsed"`
|
||||
Tests int `json:"tests"`
|
||||
NoTests bool `json:"no_tests,omitempty"`
|
||||
// Skipped is how many of the package's tests skipped; an older build module names none.
|
||||
Skipped int `json:"skipped,omitempty"`
|
||||
}
|
||||
|
||||
// Failure is a test that failed, or a package that failed with no test to name.
|
||||
@@ -107,4 +111,6 @@ type Digested struct {
|
||||
WithoutTests []string `json:"without_tests,omitempty"`
|
||||
// JudgedBy is the program that ran the digest, as Tested.JudgedBy says.
|
||||
JudgedBy string `json:"judged_by,omitempty"`
|
||||
// Needs are the packages whose tests need something of the node at this level (issue 890).
|
||||
Needs []delivery.PackageNeeds `json:"needs,omitempty"`
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// A try is a pushed head judged by the checker as a pull request's head is, before its pull request is opened (ADR 0501
|
||||
@@ -24,6 +25,7 @@ import (
|
||||
const (
|
||||
TryTool = "checker_try"
|
||||
TryReadTool = "checker_try_read"
|
||||
TryEndTool = "checker_try_end"
|
||||
// TriesState is the checker's state of tries: the bucket mesh-delivery-checker_tries, keys try-<unix nanoseconds>.
|
||||
TriesState = "tries"
|
||||
TriesBucket = delivery.CheckerModule + "_" + TriesState
|
||||
@@ -69,13 +71,18 @@ type Try struct {
|
||||
// Said is the try's report in words: what failed first, the tests run, kept and skipped per level, the time of each part.
|
||||
Said string `json:"said,omitempty"`
|
||||
// Took is the time from its start to its end, as Go writes a duration.
|
||||
Took string `json:"took,omitempty"`
|
||||
Took string `json:"took,omitempty"`
|
||||
// Failed is every test that failed, each with its first line, at most delivery.MaxFailures and then how many more
|
||||
// (issue 1921); or what else failed first (a conflict, a refused step).
|
||||
Failed []string `json:"failed,omitempty"`
|
||||
Levels map[string]delivery.LevelResult `json:"levels,omitempty"`
|
||||
Machines []string `json:"machines,omitempty"`
|
||||
Asked time.Time `json:"asked"`
|
||||
Started time.Time `json:"started,omitzero"`
|
||||
Ended time.Time `json:"ended,omitzero"`
|
||||
// Asks are the ids of the asks the try put on the build seats' streams, each ending on the step key <node>.<id> of its
|
||||
// build module: the delivery view counts those steps as a try's, not a candidate's check (build_usage).
|
||||
Asks []string `json:"asks,omitempty"`
|
||||
Asked time.Time `json:"asked"`
|
||||
Started time.Time `json:"started,omitzero"`
|
||||
Ended time.Time `json:"ended,omitzero"`
|
||||
}
|
||||
|
||||
// Over says whether the try came to its end.
|
||||
@@ -220,6 +227,52 @@ func ReadTry(store TryStore, id string) (json.RawMessage, error) {
|
||||
return raw, nil
|
||||
}
|
||||
|
||||
// TryEnder is the tries state as the end tool sees it: one key with its revision, and a write at that revision.
|
||||
type TryEnder interface {
|
||||
Entry(key string) (json.RawMessage, uint64, error)
|
||||
Update(key string, value []byte, revision uint64) error
|
||||
}
|
||||
|
||||
// EndTry ends a try that waits or runs as error, saying why and that it was ended by hand (issue 734): its place is
|
||||
// free at once, and the next checker_try of its head asks anew. A try that ended already is refused, naming its end.
|
||||
func EndTry(store TryEnder, id, why string, now time.Time) (Try, error) {
|
||||
if !tryKeyOK.MatchString(id) {
|
||||
return Try{}, fmt.Errorf("%q is not a try's id, try-<digits>", id)
|
||||
}
|
||||
why = strings.TrimSpace(why)
|
||||
if why == "" {
|
||||
return Try{}, errors.New("say why the try is ended: the why is kept in what it says")
|
||||
}
|
||||
raw, rev, err := store.Entry(id)
|
||||
if err != nil {
|
||||
return Try{}, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return Try{}, fmt.Errorf("no try %s: it was never asked, or it ended more than %s ago and was removed", id, TryKeep)
|
||||
}
|
||||
t, err := DecodeTry(raw)
|
||||
if err != nil {
|
||||
return Try{}, err
|
||||
}
|
||||
if t.Over() {
|
||||
return Try{}, fmt.Errorf("%s ended already, %s at %s", id, t.State, t.Ended.Format(time.RFC3339))
|
||||
}
|
||||
t.State, t.Ended = TryError, now.UTC()
|
||||
if !t.Started.IsZero() {
|
||||
t.Took = t.Ended.Sub(t.Started).Round(time.Second).String()
|
||||
}
|
||||
t.Said = strings.TrimSpace("ended by hand (" + TryEndTool + "): " + why + " | " + t.Said)
|
||||
t.Said = strings.TrimSuffix(t.Said, " |")
|
||||
data, err := encodeTry(t)
|
||||
if err != nil {
|
||||
return Try{}, err
|
||||
}
|
||||
if err := store.Update(id, data, rev); err != nil {
|
||||
return Try{}, fmt.Errorf("%s changed while it was ended, read it and end it again: %w", id, err)
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// TryOptions narrow what a try runs; a pull request's check has none.
|
||||
type TryOptions struct {
|
||||
Level string
|
||||
@@ -232,12 +285,14 @@ type Tryer struct {
|
||||
Tries KV
|
||||
Judge Judge
|
||||
Bound time.Duration
|
||||
Now func() time.Time
|
||||
Logf func(format string, args ...any)
|
||||
// NotTaken ends a try whose ask no build node took within it as error (issue 734); none when 0.
|
||||
NotTaken time.Duration
|
||||
Now func() time.Time
|
||||
Logf func(format string, args ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
wg sync.WaitGroup
|
||||
inflight map[string]bool
|
||||
inflight map[string]context.CancelFunc
|
||||
seen map[string]bool // looked at since this process started: a try left running is taken over the first time
|
||||
}
|
||||
|
||||
@@ -265,25 +320,31 @@ func (tr *Tryer) Put(ctx context.Context, key string, value []byte) {
|
||||
tr.logf("tries.%s holds what is not a try, left: %v", key, err)
|
||||
return
|
||||
}
|
||||
if t.Over() {
|
||||
return
|
||||
}
|
||||
tr.mu.Lock()
|
||||
if tr.inflight == nil {
|
||||
tr.inflight, tr.seen = map[string]bool{}, map[string]bool{}
|
||||
tr.inflight, tr.seen = map[string]context.CancelFunc{}, map[string]bool{}
|
||||
}
|
||||
if tr.inflight[key] || (t.State == TryRunning && tr.seen[key]) {
|
||||
if t.Over() {
|
||||
// Ended while it ran (checker_try_end): its judging stops, and its asks are no longer waited for.
|
||||
if stop := tr.inflight[key]; stop != nil {
|
||||
stop()
|
||||
}
|
||||
tr.mu.Unlock()
|
||||
return
|
||||
}
|
||||
if tr.inflight[key] != nil || (t.State == TryRunning && tr.seen[key]) {
|
||||
tr.mu.Unlock()
|
||||
return
|
||||
}
|
||||
run, stop := context.WithCancel(ctx)
|
||||
tr.seen[key] = true
|
||||
tr.inflight[key] = true
|
||||
tr.inflight[key] = stop
|
||||
tr.mu.Unlock()
|
||||
tr.wg.Add(1)
|
||||
go func() {
|
||||
defer tr.wg.Done()
|
||||
defer func() { tr.mu.Lock(); delete(tr.inflight, key); tr.mu.Unlock() }()
|
||||
tr.run(ctx, key)
|
||||
defer func() { tr.mu.Lock(); delete(tr.inflight, key); tr.mu.Unlock(); stop() }()
|
||||
tr.run(run, key)
|
||||
}()
|
||||
}
|
||||
|
||||
@@ -340,28 +401,34 @@ func (tr *Tryer) run(ctx context.Context, key string) {
|
||||
Queued: t.Asked,
|
||||
Progress: &Progress{},
|
||||
Try: &TryOptions{Level: t.Level, Run: t.Tests}}
|
||||
out, judged := judgeWithin(ctx, tr.Judge, cs, tr.Bound)
|
||||
judging := ctx
|
||||
if tr.NotTaken > 0 {
|
||||
judging = buildrun.NotTakenBound(ctx, tr.NotTaken)
|
||||
}
|
||||
out, judged := judgeWithin(judging, tr.Judge, cs, tr.Bound)
|
||||
if ctx.Err() != nil {
|
||||
return // left running: the checker takes it over when it starts
|
||||
return // left running: the checker takes it over when it starts; or ended by hand, which wrote its end
|
||||
}
|
||||
v := out.Verdict
|
||||
t.Ended = tr.now().UTC()
|
||||
t.Took = t.Ended.Sub(t.Started).Round(time.Second).String()
|
||||
t.Levels, t.Machines = v.Levels, v.Machines
|
||||
t.Levels, t.Machines, t.Asks = v.Levels, v.Machines, cs.Asks()
|
||||
t.Said = Report(v, cs, cs.Pulls[0], out.Builds, out.Slowest, out.Judged)
|
||||
for _, n := range cs.Notes() {
|
||||
t.Said += " | " + n
|
||||
}
|
||||
switch {
|
||||
case !judged || IsNodesFailure(v):
|
||||
case !judged || IsNodesFailure(v) || IsLostAsk(v) || IsNotTaken(v):
|
||||
// Not the change's: its ask was given up or never taken (issue 734), or a part of the mesh was not heard.
|
||||
t.State = TryError
|
||||
if IsLostAsk(v) || IsNotTaken(v) {
|
||||
t.Failed = []string{v.Failed.Words}
|
||||
}
|
||||
case v.State == delivery.VerdictPass:
|
||||
t.State = TryPass
|
||||
default:
|
||||
t.State = TryFail
|
||||
if f := firstFailed(v); f != "" {
|
||||
t.Failed = []string{f}
|
||||
}
|
||||
t.Failed = failedLines(v)
|
||||
}
|
||||
if t.Tests != "" {
|
||||
t.Said += " | only the tests " + t.Tests + " (Go); nothing kept"
|
||||
|
||||
@@ -97,6 +97,12 @@ func TestATryIsJudgedAsAPullRequestsHeadAndOwesTheForgeNothing(t *testing.T) { /
|
||||
if got.State != TryPass || got.Took == "" || got.Head != head1 || got.Trunk != trunkA || got.Said == "" || got.Levels[delivery.LevelUnit].Run != 2 {
|
||||
t.Fatalf("try = %+v", got)
|
||||
}
|
||||
// The try names the asks it put, so the delivery view counts their steps as a try's (build_usage).
|
||||
if len(got.Asks) != 1 {
|
||||
t.Errorf("asks = %v, want the one check ask the try put", got.Asks)
|
||||
} else if _, err := delivery.AskStepKey("shanks", got.Asks[0]); err != nil {
|
||||
t.Errorf("the try names %q, which is no ask id: %v", got.Asks[0], err)
|
||||
}
|
||||
fetch := b.steps.tool("fetch")
|
||||
if len(fetch) != 1 || fetch[0].args["commit"] != head1 || fetch[0].args["base"] != trunkA || fetch[0].args["ref"] != "" {
|
||||
t.Errorf("fetch = %+v, want the head merged onto the trunk, with no pull request's ref", fetch)
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Turns holds the gate's merge turns (novox/hq ADR 0548 §3), by repository, as a watch of the gate's turns bucket puts them
|
||||
// in: the checker claims no check of a repository a turn names, except the check of a waiting delivery's own heads, so an
|
||||
// approved merge goes before a check not yet started. A bucket not there is read as empty.
|
||||
type Turns struct {
|
||||
mu sync.Mutex
|
||||
byRepo map[string]delivery.Turn
|
||||
}
|
||||
|
||||
// Put takes a value of the turns bucket in, or its delete, and answers whether what the checker holds changed. A value that
|
||||
// is not a turn is dropped as a delete would be: a turn the checker cannot read holds nothing.
|
||||
func (t *Turns) Put(key string, value []byte, deleted bool) bool {
|
||||
repository, err := delivery.ParseTurnsKey(key)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if t.byRepo == nil {
|
||||
t.byRepo = map[string]delivery.Turn{}
|
||||
}
|
||||
cur, had := t.byRepo[repository]
|
||||
if !deleted {
|
||||
if turn, err := delivery.DecodeTurn(value); err == nil {
|
||||
t.byRepo[repository] = turn
|
||||
return !had || !delivery.SameTurn(cur, turn)
|
||||
}
|
||||
}
|
||||
delete(t.byRepo, repository)
|
||||
return had
|
||||
}
|
||||
|
||||
// Holding names the first of the repositories (by name) a turn holds off the check of the heads, and the delivery that
|
||||
// waits first in it; false when none does.
|
||||
func (t *Turns) Holding(repositories []string, heads string) (string, delivery.TurnWait, bool) {
|
||||
if t == nil {
|
||||
return "", delivery.TurnWait{}, false
|
||||
}
|
||||
sorted := make([]string, 0, len(repositories))
|
||||
for _, r := range repositories {
|
||||
sorted = append(sorted, strings.ToLower(r))
|
||||
}
|
||||
sort.Strings(sorted)
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
for _, r := range sorted {
|
||||
turn, ok := t.byRepo[r]
|
||||
if !ok || !delivery.TurnHolds(turn, heads) {
|
||||
continue
|
||||
}
|
||||
for _, w := range turn.Waiting {
|
||||
if w.Heads != heads {
|
||||
return r, w, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", delivery.TurnWait{}, false
|
||||
}
|
||||
|
||||
// All is every turn held now, by repository (a copy).
|
||||
func (t *Turns) All() map[string]delivery.Turn {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
out := make(map[string]delivery.Turn, len(t.byRepo))
|
||||
for r, turn := range t.byRepo {
|
||||
out[r] = turn
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -93,11 +93,11 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
|
||||
t.Errorf("module = %q", m.Module)
|
||||
}
|
||||
// delivery.CheckerKeptHistory, CheckerOwedHistory, PreviewsHistory, CheckerVerdictsHistory; the names follow the SDK's buckets.
|
||||
if want := []state{{Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "tries", History: 1}, {Name: "verdicts", History: 5}}; !reflect.DeepEqual(m.State, want) {
|
||||
if want := []state{{Name: "hearing", History: 1}, {Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "tries", History: 1}, {Name: "verdicts", History: 5}}; !reflect.DeepEqual(m.State, want) {
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
}
|
||||
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-controller.assignments",
|
||||
"mesh-delivery-assigner.compositions", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates",
|
||||
"mesh-delivery-assigner.compositions", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-gate.turns", "mesh-delivery-indexer.candidates",
|
||||
"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||
t.Errorf("reads = %v, want %v", m.Reads, want)
|
||||
}
|
||||
@@ -124,8 +124,8 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
|
||||
t.Errorf("streams = %+v, want %+v", m.Streams, want)
|
||||
}
|
||||
// The read tools (issue 1686) are served by the tools bundle, not by the checker's process.
|
||||
// The try's two tools (ADR 0501 §5) beside them: mesh-controller's try calls them.
|
||||
if want := []string{"checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_try", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
||||
// The try's tools (ADR 0501 §5; the end by hand, issue 734) beside them: mesh-controller's try calls them.
|
||||
if want := []string{"checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} {
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
{ "name": "checks", "events": ["preview.**", "owed.**"] }
|
||||
],
|
||||
"state": [
|
||||
{ "name": "hearing", "history": 1 },
|
||||
{ "name": "kept", "history": 1 },
|
||||
{ "name": "owed", "history": 5 },
|
||||
{ "name": "previews", "history": 5 },
|
||||
@@ -25,6 +26,7 @@
|
||||
"mesh-delivery-assigner.compositions",
|
||||
"mesh-delivery-builder.builds",
|
||||
"mesh-delivery-gate.source",
|
||||
"mesh-delivery-gate.turns",
|
||||
"mesh-delivery-indexer.candidates",
|
||||
"mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-indexer.modules",
|
||||
@@ -41,6 +43,7 @@
|
||||
"checker_pull",
|
||||
"checker_status",
|
||||
"checker_try",
|
||||
"checker_try_end",
|
||||
"checker_try_read",
|
||||
"checker_verdicts"
|
||||
],
|
||||
|
||||
@@ -91,6 +91,28 @@ func treesVia(ctx context.Context, call caller) bridge.Trees {
|
||||
}
|
||||
}
|
||||
|
||||
// listedVia asks the forge for the files directly under one directory of a commit through call: gitea_commit_trees with
|
||||
// listed (issue 1126), answer {listed: {directory: {name: blob id}}}.
|
||||
func listedVia(ctx context.Context, call caller) bridge.Listed {
|
||||
return func(owner, repo, sha, dir string) (map[string]string, error) {
|
||||
raw, err := call(ctx, commitTreesTool, map[string]any{"owner": owner, "repo": repo, "sha": sha, "listed": []string{dir}})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var a struct {
|
||||
Listed map[string]map[string]string `json:"listed"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("the answer of %s is not {listed}: %w", commitTreesTool, err)
|
||||
}
|
||||
if a.Listed == nil {
|
||||
// A forge module from before listed answers none: placement cannot be read, never read as empty.
|
||||
return nil, fmt.Errorf("the answer of %s lists no directory; the forge module does not list yet", commitTreesTool)
|
||||
}
|
||||
return a.Listed[dir], nil
|
||||
}
|
||||
}
|
||||
|
||||
// getFileTool is the forge's tool for one file at a ref (mesh-catalog modules/gitea tools/index.ts gitea_get_file):
|
||||
// arguments owner, repo, path and ref, answer {file: {path, ref, sha, size, content}}, the content decoded to text
|
||||
// (client.ts getFile). A path the forge does not have fails with `Gitea API <path>: 404 <body>` (client.ts request).
|
||||
|
||||
@@ -117,8 +117,8 @@ type publisher interface {
|
||||
// handle turns one event into its message on FORGE and acknowledges it. An event it cannot read is terminated, never
|
||||
// redelivered, and the log names its subject and why. A message it could not write stays unacknowledged: the server
|
||||
// delivers it again (the server's ack wait is the only clock).
|
||||
func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, logger *log.Logger) {
|
||||
msg, err := bridge.Convert(m.Subject(), m.Data(), trees, files)
|
||||
func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, list bridge.Listed, logger *log.Logger) {
|
||||
msg, err := bridge.ConvertWith(m.Subject(), m.Data(), trees, files, list)
|
||||
switch {
|
||||
case errors.Is(err, bridge.ErrRetry):
|
||||
logger.Printf("%v; heard again in %s", err, retryDelay)
|
||||
@@ -237,7 +237,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// Bounded: the turn is held, and no sync or other event goes on without it.
|
||||
hctx, cancel := context.WithTimeout(ctx, syncDeadline)
|
||||
defer cancel()
|
||||
handle(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), logger)
|
||||
handle(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), listedVia(hctx, call), logger)
|
||||
})
|
||||
},
|
||||
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
|
||||
|
||||
@@ -81,7 +81,7 @@ func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit
|
||||
})
|
||||
var logs bytes.Buffer
|
||||
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{}
|
||||
handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0))
|
||||
handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0))
|
||||
if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull" {
|
||||
t.Fatalf("published on %v", pub.subjects)
|
||||
}
|
||||
@@ -97,7 +97,7 @@ func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit
|
||||
func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5
|
||||
var logs bytes.Buffer
|
||||
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(map[string]any{"head_sha": "short"})}, &fakePub{}
|
||||
handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0))
|
||||
handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0))
|
||||
if len(pub.subjects) != 0 {
|
||||
t.Errorf("a message was written: %v", pub.subjects)
|
||||
}
|
||||
@@ -112,7 +112,7 @@ func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5
|
||||
func TestUnwrittenMessageIsNotAcked(t *testing.T) { // 1235 (a failed write is redelivered by the server, not retried here)
|
||||
var logs bytes.Buffer
|
||||
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{err: errors.New("no stream")}
|
||||
handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0))
|
||||
handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0))
|
||||
if m.acked != 0 || m.termd != 0 {
|
||||
t.Errorf("acked %d terminated %d, want neither", m.acked, m.termd)
|
||||
}
|
||||
@@ -134,7 +134,7 @@ func TestTreesRefusedLeavesTheEventForTenSecondsAndLogs(t *testing.T) { // 1236/
|
||||
call := func(context.Context, string, any) (json.RawMessage, error) {
|
||||
return nil, errors.New("the tree listing of novox/mesh-delivery was cut")
|
||||
}
|
||||
handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), log.New(&logs, "", 0))
|
||||
handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), nil, log.New(&logs, "", 0))
|
||||
if len(pub.subjects) != 0 {
|
||||
t.Errorf("a message was written: %v", pub.subjects)
|
||||
}
|
||||
@@ -163,7 +163,7 @@ func TestMergeCallsTheToolOnceAndWritesTheMergedMessage(t *testing.T) { // 1236/
|
||||
}
|
||||
return json.RawMessage(`{"root":"r0","trees":{"":"t0","modules/a":"t1"}}`), nil
|
||||
}
|
||||
handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), log.New(&logs, "", 0))
|
||||
handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), nil, log.New(&logs, "", 0))
|
||||
if len(keys) == 0 || keys[0] != "gitea.gitea_commit_trees" || slices.Index(keys[1:], "gitea.gitea_commit_trees") >= 0 {
|
||||
t.Fatalf("calls %v, want the trees once and first", keys)
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026193323-03dce1237803
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026193323-03dce1237803 h1:RInAboHBk5THYf6Agvy05iO9p2XV5zGb0ilYa+iSO7o=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026193323-03dce1237803/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"path"
|
||||
"regexp"
|
||||
"slices"
|
||||
@@ -88,6 +89,17 @@ func unreadable(subject, why string, args ...any) error {
|
||||
// the trunk asks trees once and becomes a merged message; a merge into any other branch is a closed pull message, as the
|
||||
// pull request left its group. A forge that cannot say is ErrRetry; a merge with no module.json is ErrNoModules.
|
||||
func Convert(subject string, data []byte, trees Trees, files Files) (Message, error) {
|
||||
return ConvertWith(subject, data, trees, files, nil)
|
||||
}
|
||||
|
||||
// Listed asks the forge for the files directly under one directory of a commit, by name to blob id (gitea_commit_trees
|
||||
// with listed); a directory the commit lacks lists none.
|
||||
type Listed func(owner, repo, sha, dir string) (map[string]string, error)
|
||||
|
||||
// ConvertWith is Convert; with list, a merge of the catalogue repository (delivery.CatalogueRepository) also carries every
|
||||
// placement file of its merge commit (novox/hq ADR 0549 §2, issue 1126), read whole or not at all: a forge that cannot
|
||||
// say is ErrRetry, as for the manifests, so the indexer never takes a part of placement for all of it.
|
||||
func ConvertWith(subject string, data []byte, trees Trees, files Files, list Listed) (Message, error) {
|
||||
var action string
|
||||
switch subject {
|
||||
case UpdatedSubject:
|
||||
@@ -114,7 +126,7 @@ func Convert(subject string, data []byte, trees Trees, files Files) (Message, er
|
||||
return Message{}, unreadable(subject, "no branch")
|
||||
}
|
||||
if subject == MergedSubject && p.Base == Trunk {
|
||||
return merged(subject, p, trees, files)
|
||||
return merged(subject, p, trees, files, list)
|
||||
}
|
||||
f := delivery.Forge{Kind: delivery.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number,
|
||||
Head: p.HeadSHA, Action: action}
|
||||
@@ -227,7 +239,7 @@ func inputsOf(owner, repo, ref string, dirs []string, files Files) (manifests ma
|
||||
}
|
||||
|
||||
// merged is the merged message of a merge into the trunk: the trees of the merge commit, asked once.
|
||||
func merged(subject string, p pull, trees Trees, files Files) (Message, error) {
|
||||
func merged(subject string, p pull, trees Trees, files Files, list Listed) (Message, error) {
|
||||
if !sha40.MatchString(p.MergeSHA) {
|
||||
return Message{}, unreadable(subject, "the merge commit %q is not 40 hex digits", p.MergeSHA)
|
||||
}
|
||||
@@ -252,8 +264,15 @@ func merged(subject string, p pull, trees Trees, files Files) (Message, error) {
|
||||
if err != nil {
|
||||
return Message{}, unreadable(subject, "%v", err)
|
||||
}
|
||||
body, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number,
|
||||
Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs, Manifests: manifests, Locks: locks})
|
||||
f := delivery.Forge{Kind: delivery.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number,
|
||||
Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs, Manifests: manifests, Locks: locks}
|
||||
if list != nil && delivery.SameRepository(repository, delivery.CatalogueRepository) {
|
||||
if f.Placements, err = placementsOf(p.Owner, p.Repo, p.MergeSHA, list, files); err != nil {
|
||||
return Message{}, fmt.Errorf("%w: %s: the placement files of %s at the merge commit %s: %v", ErrRetry, subject, repository, p.MergeSHA, err)
|
||||
}
|
||||
f.PlacementsRead = true
|
||||
}
|
||||
body, err := delivery.EncodeForge(f)
|
||||
if err != nil {
|
||||
return Message{}, unreadable(subject, "%v", err)
|
||||
}
|
||||
@@ -350,3 +369,30 @@ func OpenMessage(answer []byte) (Message, error) {
|
||||
}
|
||||
return Message{Event: delivery.OpenEvent(), Data: body}, nil
|
||||
}
|
||||
|
||||
// placementsOf is every placement file of the catalogue at a commit, by node, its bytes exact: each `<node>.json` directly
|
||||
// under delivery.PlacementDir. A file whose name is no node is left out and the rest read; the catalogue's check refuses
|
||||
// such a file before it merges (ADR 0549 §1). Nil when there is none.
|
||||
func placementsOf(owner, repo, sha string, list Listed, files Files) (map[string]string, error) {
|
||||
names, err := list(owner, repo, sha, delivery.PlacementDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out map[string]string
|
||||
for _, name := range slices.Sorted(maps.Keys(names)) {
|
||||
file := delivery.PlacementDir + "/" + name
|
||||
node, ok := delivery.PlacementFileNode(file)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
raw, err := files(owner, repo, file, sha)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", file, err)
|
||||
}
|
||||
if out == nil {
|
||||
out = map[string]string{}
|
||||
}
|
||||
out[node] = string(raw)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -79,6 +79,47 @@ func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1
|
||||
if want := map[string]map[string]string{"modules/a": {"go.sum": hexSum(sum)}}; !reflect.DeepEqual(f.Locks, want) {
|
||||
t.Errorf("locks = %v, want %v", f.Locks, want)
|
||||
}
|
||||
if f.Placements != nil || f.PlacementsRead {
|
||||
t.Errorf("a merge of another repository than the catalogue carries placement: %v", f.Placements)
|
||||
}
|
||||
|
||||
// Issue 1126 (ADR 0549 §2): a merge of the catalogue carries every placement file of its merge commit, by node; a file
|
||||
// whose name is no node is left out; a listing or a file the forge cannot give is a retry, never a part.
|
||||
catalogue := mergedBody(map[string]any{"repo": "mesh-catalog"})
|
||||
forge.have["placement/ace.json"] = []byte(`{"v":1,"node":"ace","modules":[]}`)
|
||||
forge.have["placement/g14.json"] = []byte(`{"v":1,"node":"g14","modules":[{"module":"zsh"}]}`)
|
||||
listing := map[string]string{"ace.json": "b1", "g14.json": "b2", "README.md": "b3", "a.b.json": "b4"}
|
||||
var listedAt []string
|
||||
list := func(owner, repo, sha, dir string) (map[string]string, error) {
|
||||
listedAt = append(listedAt, owner+"/"+repo+"@"+sha+":"+dir)
|
||||
return listing, nil
|
||||
}
|
||||
msg, err = ConvertWith(mergedSubject, catalogue, treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get, list)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f, err = delivery.DecodeForge(msg.Data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string]string{"ace": `{"v":1,"node":"ace","modules":[]}`, "g14": `{"v":1,"node":"g14","modules":[{"module":"zsh"}]}`}
|
||||
if !f.PlacementsRead || !reflect.DeepEqual(f.Placements, want) || !reflect.DeepEqual(listedAt, []string{"novox/mesh-catalog@" + merge + ":placement"}) {
|
||||
t.Errorf("placements = %v (read %v), listed at %v", f.Placements, f.PlacementsRead, listedAt)
|
||||
}
|
||||
listing = map[string]string{}
|
||||
if msg, err = ConvertWith(mergedSubject, catalogue, treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get, list); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f, _ = delivery.DecodeForge(msg.Data); !f.PlacementsRead || f.Placements != nil {
|
||||
t.Errorf("a catalogue with no placement file says it read none: %v, %v", f.Placements, f.PlacementsRead)
|
||||
}
|
||||
listing = map[string]string{"gone.json": "b5"}
|
||||
if _, err := ConvertWith(mergedSubject, catalogue, treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get, list); !errors.Is(err, ErrRetry) || !strings.Contains(err.Error(), "placement/gone.json") {
|
||||
t.Errorf("a placement file the forge cannot give: %v", err)
|
||||
}
|
||||
failing := func(_, _, _, _ string) (map[string]string, error) { return nil, errors.New("forge down") }
|
||||
if _, err := ConvertWith(mergedSubject, catalogue, treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get, failing); !errors.Is(err, ErrRetry) {
|
||||
t.Errorf("a listing the forge cannot give: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPullWithKnownDirsCarriesTheSameAtTheHead(t *testing.T) { // 1448/AC2
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
// Command mesh-delivery-indexer-tools serves the indexer's doing tool (novox/hq issue 1945): the runtime launches it for a
|
||||
// call. indexer_take_in writes one value of the indexer's takeins state and waits for the indexer's answer there. The
|
||||
// indexer is the other program: it serves no request subject (ADR 0437 §4). The read tools of the indexer's states are
|
||||
// mesh-delivery-view's (indexer_module, indexer_modules, indexer_candidate, indexer_candidates).
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
// asker writes a takeins value and waits for the indexer's answer to it.
|
||||
type asker func(key string, value []byte) (index.TakeInAsk, error)
|
||||
|
||||
// Tools are the indexer's tools.
|
||||
func Tools(ask asker, now func() time.Time) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
||||
return []stdio.Tool{
|
||||
{Name: index.TakeInTool,
|
||||
Description: "Take a registered module in now: the indexer reads its manifest from the repository and commit the controller's " +
|
||||
"build records register it from (at the head the gate's source names, when it names one), whatever it tried before, and " +
|
||||
"writes it to its modules state as a merge would. For a module registered from outside the indexed repositories whose " +
|
||||
"repository the forge could not read (a 404 before the forge's account could read it). Answers stored, from which " +
|
||||
"repository, commit and path, or why not (held already, no registration, the forge's error, no manifest there). " +
|
||||
"The indexer reads such a module again by itself on the next source revision and merged message; this asks now. " +
|
||||
"indexer_module (mesh-delivery-view) reads the result.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the module's name, as the catalogue registers it"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is taken in now, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
key, value, err := index.NewTakeIn(str(args, "module"), str(args, "why"), stdio.CallerOf(args), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a, err := ask(key, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return a.Result, nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
// askTakeIn writes the takeins value at key and waits, on a watch of the key, at most bound for the indexer's answer.
|
||||
func askTakeIn(s stdio.KeptState, bound time.Duration) asker {
|
||||
return func(key string, value []byte) (index.TakeInAsk, error) {
|
||||
answered := make(chan index.TakeInAsk, 1)
|
||||
var rev uint64
|
||||
var mu sync.Mutex
|
||||
take := func(revision uint64, v []byte) bool {
|
||||
mu.Lock()
|
||||
mine := rev
|
||||
mu.Unlock()
|
||||
if mine == 0 || revision <= mine {
|
||||
return false
|
||||
}
|
||||
a, err := index.DecodeTakeIn(v)
|
||||
if err != nil || a.Answered.IsZero() || a.Result == nil {
|
||||
return false
|
||||
}
|
||||
select {
|
||||
case answered <- a:
|
||||
default:
|
||||
}
|
||||
return true
|
||||
}
|
||||
if err := s.Watch(key, func(c stdio.StateChange) error {
|
||||
if c.Op == "put" {
|
||||
take(c.Revision, c.Value)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return index.TakeInAsk{}, fmt.Errorf("watching %s.%s: %w", index.TakeInsState, key, err)
|
||||
}
|
||||
r, err := s.Put(key, json.RawMessage(value))
|
||||
if err != nil {
|
||||
return index.TakeInAsk{}, fmt.Errorf("the ask could not be written to %s: %w", index.TakeInsBucket, err)
|
||||
}
|
||||
mu.Lock()
|
||||
rev = r
|
||||
mu.Unlock()
|
||||
// The watch may have heard the answer before the revision was known: read it once too.
|
||||
if e, err := s.Get(key); err == nil && e != nil {
|
||||
take(e.Revision, e.Value)
|
||||
}
|
||||
wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8)
|
||||
defer cancel()
|
||||
select {
|
||||
case a := <-answered:
|
||||
return a, nil
|
||||
case <-wait.Done():
|
||||
return index.TakeInAsk{}, fmt.Errorf("the indexer did not answer within %s: the take-in is still running (a forge read can take 30 s); its answer will be in %s.%s", bound, index.TakeInsBucket, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := stdio.Serve("", Tools(askTakeIn(stdio.State(index.TakeInsState), 25*time.Second), time.Now)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,8 @@
|
||||
// Command mesh-delivery-indexer is delivery's indexer (novox/hq ADR 0437 §4, §5, issue 1238): it hears the forge
|
||||
// bridge's messages on the stream FORGE and keeps the candidate groups (candidates.<group>): the pull requests that share
|
||||
// a branch, their heads and the modules each touches. It reads the controller's builds to know which module a changed
|
||||
// directory is. It serves no request subject and holds no timer: every reaction is a message or a watch (ADR 0437 §8).
|
||||
// directory is. It serves no request subject and holds no timer: every reaction is a message or a watch (ADR 0437 §8). Its
|
||||
// tool indexer_take_in (the program mesh-delivery-indexer-tools) asks through its state takeins, which it watches.
|
||||
//
|
||||
// Its build of 2026-10-24 00:37 named a program whose push never reached the artifact store, and a retry sends that
|
||||
// same build again; this paragraph was added only to give the indexer a new build (issue 1933).
|
||||
@@ -90,6 +91,58 @@ func pinned(want string) *tls.Config {
|
||||
}
|
||||
}
|
||||
|
||||
// awaitedStore is a state bound when it appears (process.AwaitBucket) as the indexer's store: until then every call is
|
||||
// process.ErrNotYet.
|
||||
type awaitedStore struct{ b *process.AwaitedBucket }
|
||||
|
||||
func (s awaitedStore) store() (kvStore, error) {
|
||||
kv := s.b.KV()
|
||||
if kv == nil {
|
||||
return kvStore{}, process.ErrNotYet
|
||||
}
|
||||
return kvStore{kv}, nil
|
||||
}
|
||||
|
||||
func (s awaitedStore) Get(ctx context.Context, key string) ([]byte, uint64, bool, error) {
|
||||
st, err := s.store()
|
||||
if err != nil {
|
||||
return nil, 0, false, err
|
||||
}
|
||||
return st.Get(ctx, key)
|
||||
}
|
||||
|
||||
func (s awaitedStore) Create(ctx context.Context, key string, value []byte) error {
|
||||
st, err := s.store()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return st.Create(ctx, key, value)
|
||||
}
|
||||
|
||||
func (s awaitedStore) Update(ctx context.Context, key string, value []byte, revision uint64) error {
|
||||
st, err := s.store()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return st.Update(ctx, key, value, revision)
|
||||
}
|
||||
|
||||
func (s awaitedStore) Delete(ctx context.Context, key string, revision uint64) error {
|
||||
st, err := s.store()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return st.Delete(ctx, key, revision)
|
||||
}
|
||||
|
||||
func (s awaitedStore) Keys(ctx context.Context) ([]string, error) {
|
||||
st, err := s.store()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return st.Keys(ctx)
|
||||
}
|
||||
|
||||
// kvStore is the candidates bucket as the indexer's store: compare-and-set through Create, Update and Delete at a revision.
|
||||
type kvStore struct{ kv jetstream.KeyValue }
|
||||
|
||||
@@ -252,6 +305,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Its placement state (ADR 0549 §2) is bound when it appears, as every state a delivery adds (issue 1790): until then a
|
||||
// catalogue merge's placement is said and not written, and the rest of the indexer runs.
|
||||
placement := process.AwaitBucket(ctx, js, conn, delivery.PlacementBucket, "no catalogue merge's placement is written", logger.Printf)
|
||||
// The controller's build records only name the directory of a module whose key names none (fillPaths): without them
|
||||
// such a module waits for its path, every other is indexed (issue 1945).
|
||||
builds := process.OptionalBucket(ctx, js, delivery.BuildsBucket, "a module whose key names no directory is not given one from the controller's build records until the indexer starts again", logger.Printf)
|
||||
@@ -266,11 +322,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
}
|
||||
errs := make(chan error, 1)
|
||||
var buildsWatch jetstream.KeyWatcher
|
||||
if builds != nil {
|
||||
if w, err := settle(ctx, builds, each); err != nil {
|
||||
process.NotWatched(logger.Printf, delivery.BuildsBucket, err, "a module whose key names no directory is not given one until the indexer starts again")
|
||||
} else {
|
||||
go func() { errs <- follow(ctx, w, each) }()
|
||||
buildsWatch = w
|
||||
}
|
||||
}
|
||||
|
||||
@@ -285,7 +342,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
defer calls.Close()
|
||||
ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Builds: known, Logf: logger.Printf,
|
||||
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call)}
|
||||
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call),
|
||||
Placement: awaitedStore{placement}, Say: func(event string, body any) error { return calls.Emit(event, body) }}
|
||||
// The gate's source bucket (mesh-delivery-gate_source) comes with the gate (release 3e, novox/hq ADR 0450 §1): its key
|
||||
// main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does
|
||||
// not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate.
|
||||
@@ -336,6 +394,40 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err := ix.RecomputeAtStart(ctx); err != nil {
|
||||
logger.Printf("computing the new-form fingerprints at start: %v", err)
|
||||
}
|
||||
// A module registered in the catalogue from its own repository gets its manifest indexed when the controller's records
|
||||
// name it and the bucket holds none (registered.go); the records that follow are heard below.
|
||||
if err := ix.IndexRegisteredAll(ctx); err != nil {
|
||||
logger.Printf("indexing the registered modules at start: %v", err)
|
||||
}
|
||||
if buildsWatch != nil {
|
||||
go func() {
|
||||
errs <- follow(ctx, buildsWatch, func(e jetstream.KeyValueEntry) {
|
||||
each(e)
|
||||
if !opOf(e) {
|
||||
return
|
||||
}
|
||||
var r delivery.BuildRecord
|
||||
if json.Unmarshal(e.Value(), &r) != nil || r.Module == "" {
|
||||
return
|
||||
}
|
||||
if err := ix.IndexRegisteredModule(ctx, r.Module); err != nil {
|
||||
logger.Printf("the registered module %s is not indexed: %v", r.Module, err)
|
||||
}
|
||||
})
|
||||
}()
|
||||
}
|
||||
// A person takes a registered module in now with indexer_take_in, which writes the takeins state (issue 1945): watched,
|
||||
// as every other input. A module the forge could not read is also read again by itself on the next source revision and
|
||||
// merged message (index.RetryUnread).
|
||||
// A state not on the bus yet at this start is watched once it appears (issue 1790).
|
||||
process.AwaitBucket(ctx, js, conn, index.TakeInsBucket, "indexer_take_in is not answered; a module the forge could not read is still read again by itself", logger.Printf).Ready(func(kv jetstream.KeyValue) {
|
||||
eachTakeIn := func(e jetstream.KeyValueEntry) { takenIn(ctx, ix, kv, e, logger.Printf) }
|
||||
if tw, err := settle(ctx, kv, eachTakeIn); err != nil {
|
||||
process.NotWatched(logger.Printf, index.TakeInsBucket, err, "indexer_take_in is not answered until the indexer starts again")
|
||||
} else {
|
||||
go func() { errs <- follow(ctx, tw, eachTakeIn) }()
|
||||
}
|
||||
})
|
||||
// The modules the trunk no longer holds go from the bucket (issue 1900); a merge does the same from its own message.
|
||||
if err := ix.Reconcile(ctx); err != nil {
|
||||
logger.Printf("dropping the modules the trunk does not hold at start: %v", err)
|
||||
|
||||
@@ -2,11 +2,16 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
// commitTreesTool is the forge's tool for a commit's trees (mesh-catalog modules/gitea): with `paths`, by module directory
|
||||
@@ -93,3 +98,75 @@ func containsVia(call toolCaller) func(ctx context.Context, repository, commit,
|
||||
return *a.Contains, nil
|
||||
}
|
||||
}
|
||||
|
||||
// getFileTool is the forge's tool for one file at a ref (mesh-catalog modules/gitea gitea_get_file): arguments owner,
|
||||
// repo, path and ref, answer {file: {content}}; a path it does not have fails with a 404 in its words.
|
||||
const getFileTool = "gitea.gitea_get_file"
|
||||
|
||||
// lockFiles are read beside a manifest, as the forge bridge reads them (bridge.LockFiles).
|
||||
var lockFiles = []string{"go.sum", "package-lock.json"}
|
||||
|
||||
// readVia reads a repository at a commit as the forge bridge does for a merged message: the trees by gitea_commit_trees,
|
||||
// then the module.json and lock files of the one directory dir by gitea_get_file (a repository can hold hundreds). A manifest the
|
||||
// forge does not have is an error; a directory with no module.json in the trees is answered with no manifest; a lock file it does not have is left out.
|
||||
func readVia(call toolCaller) func(ctx context.Context, repository, sha, dir string) (index.RepoRead, error) {
|
||||
return func(_ context.Context, repository, sha, dir string) (index.RepoRead, error) {
|
||||
owner, repo, ok := strings.Cut(delivery.RepositoryName(repository), "/")
|
||||
if !ok {
|
||||
return index.RepoRead{}, fmt.Errorf("%q is not <owner>/<name>", repository)
|
||||
}
|
||||
raw, err := call(commitTreesTool, map[string]any{"owner": owner, "repo": repo, "sha": sha})
|
||||
if err != nil {
|
||||
return index.RepoRead{}, err
|
||||
}
|
||||
var a struct {
|
||||
Root string `json:"root"`
|
||||
Trees map[string]string `json:"trees"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return index.RepoRead{}, fmt.Errorf("the answer of %s is not {root, trees}: %w", commitTreesTool, err)
|
||||
}
|
||||
out := index.RepoRead{Root: a.Root, Trees: a.Trees, Manifests: map[string]string{}, Locks: map[string]map[string]string{}}
|
||||
file := func(dir, name string) (string, error) {
|
||||
p := name
|
||||
if dir != "" {
|
||||
p = dir + "/" + name
|
||||
}
|
||||
raw, err := call(getFileTool, map[string]string{"owner": owner, "repo": repo, "path": p, "ref": sha})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var f struct {
|
||||
File *struct {
|
||||
Content *string `json:"content"`
|
||||
} `json:"file"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &f); err != nil || f.File == nil || f.File.Content == nil {
|
||||
return "", fmt.Errorf("the answer of %s is not {file: {content}}: %v", getFileTool, err)
|
||||
}
|
||||
return *f.File.Content, nil
|
||||
}
|
||||
if _, held := a.Trees[dir]; held {
|
||||
m, err := file(dir, "module.json")
|
||||
if err != nil {
|
||||
return index.RepoRead{}, fmt.Errorf("%s: %w", path.Join(dir, "module.json"), err)
|
||||
}
|
||||
out.Manifests[dir] = m
|
||||
for _, name := range lockFiles {
|
||||
c, err := file(dir, name)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), ": 404 ") {
|
||||
continue
|
||||
}
|
||||
return index.RepoRead{}, fmt.Errorf("%s: %w", path.Join(dir, name), err)
|
||||
}
|
||||
sum := sha256.Sum256([]byte(c))
|
||||
if out.Locks[dir] == nil {
|
||||
out.Locks[dir] = map[string]string{}
|
||||
}
|
||||
out.Locks[dir][name] = hex.EncodeToString(sum[:])
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -50,3 +51,33 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3
|
||||
t.Error("the trunk asked for paths")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadViaReadsManifestsAndLocks(t *testing.T) {
|
||||
var files []string
|
||||
call := func(key string, body any) (json.RawMessage, error) {
|
||||
switch key {
|
||||
case commitTreesTool:
|
||||
return json.RawMessage(`{"root":"r0","trees":{"":"t1","modules/other":"t2"}}`), nil
|
||||
case getFileTool:
|
||||
files = append(files, body.(map[string]string)["path"])
|
||||
switch body.(map[string]string)["path"] {
|
||||
case "module.json":
|
||||
return json.RawMessage(`{"file":{"content":"{\"module\":\"snake\"}"}}`), nil
|
||||
case "go.sum":
|
||||
return json.RawMessage(`{"file":{"content":"x"}}`), nil
|
||||
}
|
||||
return nil, errors.New("Gitea API /x: 404 not found")
|
||||
}
|
||||
return nil, errors.New("unexpected " + key)
|
||||
}
|
||||
got, err := readVia(call)(context.Background(), "http://novox.internal:20000/jschoubben/snake.git", "abc", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got.Manifests) != 1 || len(files) != 3 { // the one directory asked: its manifest and two lock files, never the others
|
||||
t.Fatalf("manifests = %v, files asked = %v", got.Manifests, files)
|
||||
}
|
||||
if got.Root != "r0" || got.Manifests[""] != `{"module":"snake"}` || len(got.Locks[""]) != 1 || got.Locks[""]["go.sum"] == "" {
|
||||
t.Fatalf("read = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
// takerIn is the part of the indexer a takeins value asks through.
|
||||
type takerIn interface {
|
||||
TakeIn(ctx context.Context, module string) index.TakenIn
|
||||
}
|
||||
|
||||
// takenIn answers one value of the takeins state (the tool indexer_take_in, issue 1945): a value not yet answered takes the
|
||||
// module in now and is written back, by compare-and-set at its revision, with what the indexer stored or why not. A value
|
||||
// answered already (this indexer's own write) or deleted is left.
|
||||
func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) {
|
||||
if e.Operation() != jetstream.KeyValuePut {
|
||||
return
|
||||
}
|
||||
a, err := index.DecodeTakeIn(e.Value())
|
||||
if err != nil {
|
||||
logf("%s.%s holds what is not a take-in ask, left: %v", index.TakeInsState, e.Key(), err)
|
||||
return
|
||||
}
|
||||
if !a.Answered.IsZero() {
|
||||
return
|
||||
}
|
||||
r := ix.TakeIn(ctx, a.Module)
|
||||
a.Result, a.Answered = &r, time.Now().UTC()
|
||||
logf("%s: a take-in asked by %s (%s): %s", a.Module, a.By, a.Why, r.Answer)
|
||||
data, err := json.Marshal(a)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
logf("%s.%s: the answer could not be written: %v", index.TakeInsState, e.Key(), err)
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026193323-03dce1237803
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026193323-03dce1237803 h1:RInAboHBk5THYf6Agvy05iO9p2XV5zGb0ilYa+iSO7o=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026193323-03dce1237803/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -2,6 +2,8 @@ package index
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
@@ -88,3 +90,102 @@ func (b *Builds) PathOf(module, repository string) (string, bool) {
|
||||
}
|
||||
return pathOf(strings.Trim(best.Path, "/")), true
|
||||
}
|
||||
|
||||
// Registration is a module registered in the catalogue from its own repository, as the controller's newest outcome record
|
||||
// of it names it: the repository, the directory of its manifest there ("" or "." for the root) and the commit it was
|
||||
// built at.
|
||||
type Registration struct {
|
||||
Module, Repository, Path, Commit, Ref string
|
||||
}
|
||||
|
||||
// registered is whether the record is an outcome that did not fail: what registers a module. Unlike IsTrunkBuild it
|
||||
// asks for no recorded source inputs, which no build records since ADR 0530 clause 1, and for no particular branch: a
|
||||
// repository's trunk is named main, master, or what its owner chose (the registered photos, de-spiegel and link2pay were
|
||||
// built from nox-mesh, and every record of them names that ref).
|
||||
func registered(r delivery.BuildRecord) bool {
|
||||
return r.State == "" && r.Failed == ""
|
||||
}
|
||||
|
||||
// RegistrationsOf are the places a module was registered from, by its outcome records: the newest record of each
|
||||
// repository and directory (a module's name was once another repository's, as photos was a directory of the catalogue's).
|
||||
// Records of a trunk's ref (main, master, trunk, none) come before those of another branch, then the newer first, so the
|
||||
// first that holds the module's manifest is the one indexed.
|
||||
func (b *Builds) RegistrationsOf(module string) []Registration {
|
||||
type place struct{ repository, path string }
|
||||
byPlace := map[place][]delivery.BuildRecord{}
|
||||
b.mu.Lock()
|
||||
for _, r := range b.records {
|
||||
if r.Module == module && registered(r) {
|
||||
k := place{delivery.RepositoryName(r.Repository), pathOf(strings.Trim(r.Path, "/"))}
|
||||
byPlace[k] = append(byPlace[k], r)
|
||||
}
|
||||
}
|
||||
b.mu.Unlock()
|
||||
// Within one place the record is that of the ref a registration comes from, never a newer pull request's or try's: a
|
||||
// trunk's ref (main, master, trunk, none), else the ref most of the place's records name (the registered photos, de-spiegel
|
||||
// and link2pay were built from nox-mesh), the newer of two equal; then its newest record.
|
||||
best := map[place]delivery.BuildRecord{}
|
||||
for k, recs := range byPlace {
|
||||
count, latest := map[string]int{}, map[string]delivery.BuildRecord{}
|
||||
for _, r := range recs {
|
||||
count[r.Ref]++
|
||||
if cur, ok := latest[r.Ref]; !ok || newer(r, cur) {
|
||||
latest[r.Ref] = r
|
||||
}
|
||||
}
|
||||
var pick string
|
||||
first := true
|
||||
for ref, rec := range latest {
|
||||
switch {
|
||||
case first:
|
||||
case trunkLike[ref] != trunkLike[pick]:
|
||||
if !trunkLike[ref] {
|
||||
continue
|
||||
}
|
||||
case count[ref] != count[pick]:
|
||||
if count[ref] < count[pick] {
|
||||
continue
|
||||
}
|
||||
case !newer(rec, latest[pick]):
|
||||
continue
|
||||
}
|
||||
pick, first = ref, false
|
||||
}
|
||||
best[k] = latest[pick]
|
||||
}
|
||||
recs := slices.Collect(maps.Values(best))
|
||||
slices.SortFunc(recs, func(x, y delivery.BuildRecord) int {
|
||||
switch {
|
||||
case trunkLike[x.Ref] != trunkLike[y.Ref]:
|
||||
if trunkLike[x.Ref] {
|
||||
return -1
|
||||
}
|
||||
return 1
|
||||
case newer(x, y):
|
||||
return -1
|
||||
case newer(y, x):
|
||||
return 1
|
||||
}
|
||||
return strings.Compare(x.Repository+x.Path, y.Repository+y.Path)
|
||||
})
|
||||
out := make([]Registration, 0, len(recs))
|
||||
for _, r := range recs {
|
||||
out = append(out, Registration{Module: r.Module, Repository: r.Repository, Path: r.Path, Commit: r.Commit, Ref: r.Ref})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Registered is the registrations of every module a record names, by module name.
|
||||
func (b *Builds) Registered() []Registration {
|
||||
b.mu.Lock()
|
||||
names := map[string]bool{}
|
||||
for _, r := range b.records {
|
||||
names[r.Module] = true
|
||||
}
|
||||
b.mu.Unlock()
|
||||
var out []Registration
|
||||
for _, name := range slices.Sorted(maps.Keys(names)) {
|
||||
out = append(out, b.RegistrationsOf(name)...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
@@ -53,14 +54,26 @@ type Indexer struct {
|
||||
// has none.
|
||||
Named func(ctx context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error)
|
||||
Trunk func(ctx context.Context, repository, sha string) ([]string, error)
|
||||
// Read asks the forge for a repository's trees at a commit and the manifest and lock-file hashes of the one module directory dir (the
|
||||
// root spelled ""), as the forge bridge reads them for a merged message (registered.go). Nil indexes no registered module.
|
||||
Read func(ctx context.Context, repository, sha, dir string) (RepoRead, error)
|
||||
// Contains says whether a commit is the second or an ancestor of it (gitea.gitea_contains).
|
||||
Contains func(ctx context.Context, repository, commit, in string) (bool, error)
|
||||
Logf func(format string, args ...any)
|
||||
// Placement is the placement bucket (delivery.PlacementBucket, ADR 0549 §2); nil writes no placement.
|
||||
Placement Store
|
||||
// Say says one of the indexer's events (placement-changed); nil says none.
|
||||
Say func(event string, body any) error
|
||||
|
||||
mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach
|
||||
source sourceNow
|
||||
previews map[string]delivery.Preview // by previews key (previews.go)
|
||||
toolchains map[string]map[string]delivery.Toolchain // language, then node (toolchains.go)
|
||||
tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go)
|
||||
unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread
|
||||
retrying atomic.Bool // a RetryUnread runs in the background (kickRetry)
|
||||
retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one
|
||||
retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on
|
||||
recompute sync.Mutex // one Recompute at a time
|
||||
}
|
||||
|
||||
@@ -85,6 +98,10 @@ func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error {
|
||||
if err := ix.applyMerged(ctx, f); err != nil {
|
||||
return err
|
||||
}
|
||||
ix.kickRetry(ctx) // a registered module the forge could not read is read again on a merge, in the background (issue 1945)
|
||||
}
|
||||
if err := ix.applyPlacement(ctx, f); err != nil {
|
||||
return err
|
||||
}
|
||||
var digests map[string]string
|
||||
var held map[string]delivery.Module
|
||||
|
||||
@@ -43,6 +43,8 @@ func withManifests(f delivery.Forge, manifests map[string]string, locks map[stri
|
||||
}
|
||||
|
||||
func TestAMergedManifestIsKeptByItsDigestAndNamedByTheModule(t *testing.T) { // 1449/AC1
|
||||
// A merge's placement files are written as its manifests are (issue 1126): folded here to hold the count.
|
||||
placementIsWrittenAtAMergeOnlyWhereItChanged(t)
|
||||
avahi := catalogueManifest(t, "avahi")
|
||||
ix, mods, manifests := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, nil)
|
||||
|
||||
@@ -25,6 +25,13 @@ import (
|
||||
// A value the module already holds but for its commit is not written again: its commit stays the one its fingerprint first
|
||||
// appeared at (writeOverlay), so a merge writes only the modules it changes.
|
||||
func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
|
||||
return ix.applyManifests(ctx, f, true)
|
||||
}
|
||||
|
||||
// applyManifests is applyMerged; drop says whether the modules of the repository the message holds no manifest for are
|
||||
// deleted. A message that names only some of a repository's modules (a registered module read on its own, registered.go)
|
||||
// drops none.
|
||||
func (ix *Indexer) applyManifests(ctx context.Context, f delivery.Forge, drop bool) error {
|
||||
digests, err := ix.keepManifests(ctx, f)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -84,7 +91,7 @@ func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
|
||||
}
|
||||
// A module of this repository the merge commit holds no module.json for is gone from the trunk, and so is its key
|
||||
// (issue 1900). A message that carries no manifests (an older writer) or one that cannot be read whole says nothing.
|
||||
if len(f.Manifests) > 0 && every {
|
||||
if drop && len(f.Manifests) > 0 && every {
|
||||
if err := ix.dropRemoved(ctx, f.Repository, held, func(key string, _ delivery.Module) bool { return named[key] }); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// Placement of modules on nodes (novox/hq ADR 0549 §2, issue 1126): at each merge of the catalogue the forge bridge
|
||||
// carries every placement file of the merge commit, and the indexer writes each node's key `placement.<node>` in its
|
||||
// state `placement` only when it differs (the revision alone is no change), deletes the key of a node whose file is
|
||||
// gone, and says `placement-changed` for each node it wrote or deleted, naming the modules added, removed or changed.
|
||||
// A merge that changes no placement writes nothing and says nothing.
|
||||
//
|
||||
// A file that does not read as a placement, or names another node than its file, is said in the log and its key is left
|
||||
// as it was: the catalogue's own check refuses such a file before it merges (ADR 0549 §1), so it is a fault of that
|
||||
// check, and a node keeps composing from the placement it had rather than from none.
|
||||
|
||||
// applyPlacement takes the placement files of a merged message in; a message that read none (another repository, an
|
||||
// older bridge) changes nothing.
|
||||
func (ix *Indexer) applyPlacement(ctx context.Context, f delivery.Forge) error {
|
||||
if ix.Placement == nil || f.Kind != delivery.ForgeMerged || !f.PlacementsRead {
|
||||
return nil
|
||||
}
|
||||
keys, err := ix.Placement.Keys(ctx)
|
||||
if errors.Is(err, process.ErrNotYet) {
|
||||
// The state is not on the bus yet: said, and the merge's candidates are still taken out (issue 1790's binding
|
||||
// writes the next catalogue merge's placement once it appears).
|
||||
ix.logf("%s at %s: its placement files are not written: %v", f.Repository, f.Merge, err)
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading the placement keys: %w", err)
|
||||
}
|
||||
gone := map[string]bool{}
|
||||
for _, key := range keys {
|
||||
if node, err := delivery.ParsePlacementKey(key); err == nil {
|
||||
gone[node] = true
|
||||
}
|
||||
}
|
||||
var refused []string
|
||||
for _, node := range slices.Sorted(maps.Keys(f.Placements)) {
|
||||
delete(gone, node)
|
||||
next, err := delivery.DecodePlacement([]byte(f.Placements[node]))
|
||||
if err == nil && next.Node != node {
|
||||
err = fmt.Errorf("the file of %s names the node %q", node, next.Node)
|
||||
}
|
||||
if err != nil {
|
||||
refused = append(refused, node)
|
||||
ix.logf("%s at %s: the placement file of %s is refused and placement.%s is left as it was (the catalogue's check should have refused it): %v",
|
||||
f.Repository, f.Merge, node, node, err)
|
||||
continue
|
||||
}
|
||||
next.Revision = f.Merge
|
||||
if err := ix.writePlacement(ctx, node, &next); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, node := range slices.Sorted(maps.Keys(gone)) {
|
||||
if err := ix.writePlacement(ctx, node, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(refused) > 0 {
|
||||
// Said, and the merge's candidates are still taken out: a refused file holds nobody's delivery.
|
||||
ix.logf("%s at %s: the placement files of %v do not read; their keys are left as they were", f.Repository, f.Merge, refused)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// writePlacement writes one node's key, or deletes it for next nil, only when it differs, and says what changed.
|
||||
func (ix *Indexer) writePlacement(ctx context.Context, node string, next *delivery.Placement) error {
|
||||
key, err := delivery.PlacementKey(node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var change *delivery.PlacementChange
|
||||
err = write(ctx, ix.Placement, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
change = nil
|
||||
var before delivery.Placement
|
||||
if found {
|
||||
cur, err := delivery.DecodePlacement(value)
|
||||
if err != nil {
|
||||
ix.logf("placement.%s holds what is not a placement, written again: %v", node, err)
|
||||
} else {
|
||||
before = cur
|
||||
}
|
||||
}
|
||||
if next == nil {
|
||||
if !found {
|
||||
return nil, false, nil
|
||||
}
|
||||
c := delivery.ChangeOf(before, delivery.Placement{Node: node})
|
||||
change = &c
|
||||
return nil, true, nil
|
||||
}
|
||||
if found && delivery.SamePlacement(before, *next) {
|
||||
return nil, false, nil
|
||||
}
|
||||
data, err := delivery.EncodePlacement(*next)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
c := delivery.ChangeOf(before, *next)
|
||||
change = &c
|
||||
return data, false, nil
|
||||
})
|
||||
if err != nil || change == nil {
|
||||
return err
|
||||
}
|
||||
if ix.Say != nil {
|
||||
if err := ix.Say(delivery.PlacementChangedEvent, *change); err != nil {
|
||||
// The key is written: a reader that watches it hears the change; the event is said loudly as not said.
|
||||
ix.logf("placement.%s is written, and %s could not be said: %v", node, delivery.PlacementChangedEvent, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// ADR 0549 §2 (issue 1126): a merge changing one node's file writes that key alone and says one event; a merge with no
|
||||
// placement change writes nothing and says nothing; a node whose file is gone loses its key and says its modules
|
||||
// removed; a file that does not read leaves its key; a message that read no placement changes none.
|
||||
func placementIsWrittenAtAMergeOnlyWhereItChanged(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ix, _, logs := newIndexer(t)
|
||||
place := newMem()
|
||||
ix.Placement = place
|
||||
var said []delivery.PlacementChange
|
||||
ix.Say = func(event string, body any) error {
|
||||
if event != delivery.PlacementChangedEvent {
|
||||
t.Errorf("said %s", event)
|
||||
}
|
||||
said = append(said, body.(delivery.PlacementChange))
|
||||
return nil
|
||||
}
|
||||
merged := func(merge string, files map[string]string) delivery.Forge {
|
||||
return delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: delivery.CatalogueRepository, Branch: "main",
|
||||
Number: 7, Head: "h", Merge: merge, Root: "r", Trees: map[string]string{"": "t"}, Placements: files, PlacementsRead: true}
|
||||
}
|
||||
ace := `{"v":1,"node":"ace","modules":[{"module":"plex"}]}`
|
||||
g14 := `{"v":1,"node":"g14","modules":[{"module":"zsh"}]}`
|
||||
written := func(node string) (delivery.Placement, uint64) {
|
||||
t.Helper()
|
||||
v, ok := place.vals["placement."+node]
|
||||
if !ok {
|
||||
return delivery.Placement{}, 0
|
||||
}
|
||||
p, err := delivery.DecodePlacement(v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p, place.revs["placement."+node]
|
||||
}
|
||||
|
||||
if err := ix.Apply(ctx, merged("m1", map[string]string{"ace": ace, "g14": g14})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p, _ := written("ace"); p.Revision != "m1" || !reflect.DeepEqual(p.Assigned(), []string{"plex"}) || len(said) != 2 {
|
||||
t.Fatalf("first merge: %+v, said %v", p, said)
|
||||
}
|
||||
_, aceRev := written("ace")
|
||||
|
||||
said = nil
|
||||
if err := ix.Apply(ctx, merged("m2", map[string]string{"ace": ace, "g14": g14})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p, rev := written("ace"); rev != aceRev || p.Revision != "m1" || len(said) != 0 {
|
||||
t.Errorf("a merge with no placement change wrote %+v (revision %d, was %d) and said %v", p, rev, aceRev, said)
|
||||
}
|
||||
|
||||
said = nil
|
||||
if err := ix.Apply(ctx, merged("m3", map[string]string{"ace": ace, "g14": `{"v":1,"node":"g14","modules":[{"module":"zsh"},{"module":"xterm"}]}`})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, rev := written("ace"); rev != aceRev || len(said) != 1 || said[0].Node != "g14" || said[0].Revision != "m3" ||
|
||||
!reflect.DeepEqual(said[0].Added, []string{"xterm"}) {
|
||||
t.Errorf("one file changed: ace's revision %d (was %d), said %+v", rev, aceRev, said)
|
||||
}
|
||||
|
||||
said = nil
|
||||
if err := ix.Apply(ctx, merged("m4", map[string]string{"ace": ace, "g14": `{"v":1,"node":"ace","modules":[]}`})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p, _ := written("g14"); len(p.Modules) != 2 || len(said) != 0 || !strings.Contains(logs.String(), "the placement file of %s is refused") {
|
||||
t.Errorf("a file naming another node replaced the key: %+v, said %v, logs %s", p, said, logs)
|
||||
}
|
||||
|
||||
said = nil
|
||||
if err := ix.Apply(ctx, merged("m5", map[string]string{"ace": ace})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, rev := written("g14"); rev != 0 || len(said) != 1 || !reflect.DeepEqual(said[0].Removed, []string{"xterm", "zsh"}) {
|
||||
t.Errorf("a file removed: g14's key at revision %d, said %+v", rev, said)
|
||||
}
|
||||
|
||||
said = nil
|
||||
unread := merged("m6", nil)
|
||||
unread.PlacementsRead = false
|
||||
if err := ix.Apply(ctx, unread); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p, _ := written("ace"); p.Revision != "m1" || len(said) != 0 {
|
||||
t.Errorf("a message that read no placement changed it: %+v, %v", p, said)
|
||||
}
|
||||
}
|
||||
@@ -164,6 +164,7 @@ func TestAPreviewHeardBeforeItsPullRequestGivesTheFingerprintWhenTheCandidateIsW
|
||||
}
|
||||
|
||||
func TestAPreviewNamingNoManifestOfTheModulesDirectoryGivesAReason(t *testing.T) {
|
||||
t.Run("the same preview again", testTheSamePreviewAppliedAgainWritesNothing)
|
||||
ix, _, cands := previewBed(t, true)
|
||||
if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -176,7 +177,7 @@ func TestAPreviewNamingNoManifestOfTheModulesDirectoryGivesAReason(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheSamePreviewAppliedAgainWritesNothing(t *testing.T) {
|
||||
func testTheSamePreviewAppliedAgainWritesNothing(t *testing.T) {
|
||||
ix, _, cands := previewBed(t, true)
|
||||
if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -0,0 +1,275 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// A module registered in the catalogue from its own repository (the operator registers it with the repository and the
|
||||
// path of its manifest) had no manifest here: the indexer learns a repository only from a merged message, and no merge
|
||||
// followed the registration. The controller's outcome records name each registered module's repository, directory and
|
||||
// commit (builds.go Registered), so the indexer reads that module's manifest at the repository's head (the gate's source names it) or, with none named, at the
|
||||
// registered commit, and writes the same keys a merge writes. A later merge to the repository's trunk reaches the module through the forge bridge's merged
|
||||
// message as for every other repository.
|
||||
|
||||
// RepoRead is a repository at a commit as the forge bridge reads it for a merged message: the root tree, the tree of each
|
||||
// directory holding a module.json (the root spelled ""), their manifests and the hex sha256 of the lock files beside them.
|
||||
type RepoRead struct {
|
||||
Root string
|
||||
Trees map[string]string
|
||||
Manifests map[string]string
|
||||
Locks map[string]map[string]string
|
||||
}
|
||||
|
||||
// IndexRegistered writes the module of a registration as a merge of its repository would, when the modules bucket holds
|
||||
// none of that name. Its commit is the head the gate's source names for the repository, else the registration's. The
|
||||
// repository's other modules are neither written nor dropped: they are the merges'. A manifest that is not at the path or
|
||||
// names another module is said and left. A forge that could not say (a timeout, or a 404 while the forge's account could
|
||||
// not read the repository) is an error, and the module is kept as unread: it is read again at the next source revision
|
||||
// and the next merged message (RetryUnread), not only when the indexer starts again (issue 1945).
|
||||
func (ix *Indexer) IndexRegistered(ctx context.Context, r Registration) error {
|
||||
_, err := ix.indexRegistered(ctx, r, false)
|
||||
return err
|
||||
}
|
||||
|
||||
// taken is what indexRegistered did with one registration: the module written, held already, or the reason it was not.
|
||||
type taken struct {
|
||||
stored, held bool
|
||||
repository, commit, dir string
|
||||
why string
|
||||
}
|
||||
|
||||
func (ix *Indexer) indexRegistered(ctx context.Context, r Registration, force bool) (taken, error) {
|
||||
if ix.Modules == nil || ix.Read == nil {
|
||||
return taken{why: "the indexer reads no forge or writes no modules"}, nil
|
||||
}
|
||||
key, err := delivery.ModulesKey(r.Module)
|
||||
if err != nil {
|
||||
ix.logf("%s: the registered module has no usable key, skipped: %v", r.Module, err)
|
||||
return taken{why: fmt.Sprintf("the module has no usable key: %v", err)}, nil
|
||||
}
|
||||
if _, _, found, err := ix.Modules.Get(ctx, key); err != nil {
|
||||
return taken{}, fmt.Errorf("modules.%s: %w", key, err)
|
||||
} else if found {
|
||||
ix.markRead(r.Module)
|
||||
return taken{held: true, why: "the modules state holds it already: a merge to its repository writes it from now on"}, nil
|
||||
}
|
||||
repository := delivery.RepositoryName(r.Repository)
|
||||
commit := r.Commit
|
||||
dir := strings.Trim(r.Path, "/")
|
||||
if dir == delivery.RootPath {
|
||||
dir = ""
|
||||
}
|
||||
// The head the gate's source names for the repository, when it names one, else the registration's commit. A module a
|
||||
// merge removed is absent at the head and is said and left.
|
||||
ix.mu.Lock()
|
||||
for repo, head := range ix.source.heads {
|
||||
if delivery.SameRepository(repo, r.Repository) && head != "" {
|
||||
commit = head
|
||||
}
|
||||
}
|
||||
ix.mu.Unlock()
|
||||
t := taken{repository: repository, commit: commit, dir: dir}
|
||||
if repository == "" || commit == "" {
|
||||
ix.logf("%s: registered with no repository or commit, so its manifest is not read", r.Module)
|
||||
t.why = "the controller's record names no repository or commit"
|
||||
return t, nil
|
||||
}
|
||||
// One try for each repository, commit and directory: a repository with no manifest where it was registered is said once,
|
||||
// not at every record of the controller's builds. A person's take-in (force) reads again.
|
||||
tried := repository + "@" + commit + ":" + dir
|
||||
ix.mu.Lock()
|
||||
if ix.tried[tried] && !force {
|
||||
ix.mu.Unlock()
|
||||
t.why = "tried already at this commit and directory"
|
||||
return t, nil
|
||||
}
|
||||
if ix.tried == nil {
|
||||
ix.tried = map[string]bool{}
|
||||
}
|
||||
ix.tried[tried] = true
|
||||
ix.mu.Unlock()
|
||||
read, err := ix.Read(ctx, repository, commit, dir)
|
||||
if err != nil {
|
||||
ix.mu.Lock()
|
||||
delete(ix.tried, tried) // a forge that could not say is asked again
|
||||
if ix.unread == nil {
|
||||
ix.unread = map[string]bool{}
|
||||
}
|
||||
ix.unread[r.Module] = true // and read again on the next source revision or merged message (RetryUnread)
|
||||
ix.mu.Unlock()
|
||||
return t, fmt.Errorf("%s: reading %s at %s: %w", r.Module, repository, commit, err)
|
||||
}
|
||||
ix.markRead(r.Module)
|
||||
raw, ok := read.Manifests[dir]
|
||||
if !ok || read.Trees[dir] == "" {
|
||||
ix.logf("%s: %s at %s holds no module.json at %q, so the registered module is not indexed", r.Module, repository, commit, dir)
|
||||
t.why = fmt.Sprintf("%s at %s holds no module.json at %q", repository, commit, dir)
|
||||
return t, nil
|
||||
}
|
||||
if name := manifestModule(raw); name != r.Module {
|
||||
ix.logf("%s: the manifest at %q of %s at %s names the module %q, so the registered module is not indexed", r.Module, dir, repository, commit, name)
|
||||
t.why = fmt.Sprintf("the manifest at %q of %s at %s names the module %q", dir, repository, commit, name)
|
||||
return t, nil
|
||||
}
|
||||
f := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: repository, Merge: commit, Root: read.Root,
|
||||
Trees: map[string]string{dir: read.Trees[dir]},
|
||||
Manifests: map[string]string{dir: raw}}
|
||||
if l := read.Locks[dir]; len(l) > 0 {
|
||||
f.Locks = map[string]map[string]string{dir: l}
|
||||
}
|
||||
if err := ix.applyManifests(ctx, f, false); err != nil {
|
||||
return t, err
|
||||
}
|
||||
if _, _, found, err := ix.Modules.Get(ctx, key); err != nil {
|
||||
return t, fmt.Errorf("modules.%s: %w", key, err)
|
||||
} else if !found {
|
||||
t.why = "its manifest was read but the modules state holds no value of it (the indexer's log says why)"
|
||||
return t, nil
|
||||
}
|
||||
t.stored = true
|
||||
ix.logf("%s: the registered module is indexed from %s at %s", r.Module, repository, commit)
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// markRead takes a module out of the unread ones: the forge answered for it.
|
||||
func (ix *Indexer) markRead(module string) {
|
||||
ix.mu.Lock()
|
||||
delete(ix.unread, module)
|
||||
ix.mu.Unlock()
|
||||
}
|
||||
|
||||
// RetryUnread reads again every registered module whose last read the forge could not answer (a timeout, or a 404 while
|
||||
// the forge's account could not read the repository; issue 1945). It is called on the events that already reach the
|
||||
// indexer, a new source revision and a merged message, so a repository that became readable is taken in without a hand
|
||||
// act and without a timer of its own (ADR 0437 §8). What still cannot be read is said and kept for the next.
|
||||
func (ix *Indexer) RetryUnread(ctx context.Context) {
|
||||
ix.mu.Lock()
|
||||
names := slices.Sorted(maps.Keys(ix.unread))
|
||||
ix.mu.Unlock()
|
||||
for _, module := range names {
|
||||
if err := ix.IndexRegisteredModule(ctx, module); err != nil {
|
||||
ix.logf("the registered module %s is still not indexed: %v", module, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// kickRetry runs RetryUnread in one background goroutine, so the message or revision that asked it is acknowledged without
|
||||
// waiting on the forge (each unread module costs up to two forge calls of 30 s each). A kick while one runs makes it run
|
||||
// once more when it ends; never two at once.
|
||||
func (ix *Indexer) kickRetry(ctx context.Context) {
|
||||
ix.mu.Lock()
|
||||
none := len(ix.unread) == 0
|
||||
ix.mu.Unlock()
|
||||
if none {
|
||||
return
|
||||
}
|
||||
ix.retryAgain.Store(true)
|
||||
if !ix.retrying.CompareAndSwap(false, true) {
|
||||
return
|
||||
}
|
||||
ix.retryWG.Add(1)
|
||||
go func() {
|
||||
defer ix.retryWG.Done()
|
||||
for ix.retryAgain.Swap(false) {
|
||||
ix.RetryUnread(ctx)
|
||||
}
|
||||
ix.retrying.Store(false)
|
||||
// A kick between the last Swap and the Store found it running and returned: run for it.
|
||||
if ix.retryAgain.Load() {
|
||||
ix.kickRetry(ctx)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// TakenIn is the answer of a take-in (the tool indexer_take_in): whether the module is now in the modules state, from
|
||||
// where, and otherwise why not.
|
||||
type TakenIn struct {
|
||||
Module string `json:"module"`
|
||||
Stored bool `json:"stored"`
|
||||
Held bool `json:"held,omitempty"`
|
||||
Repository string `json:"repository,omitempty"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Answer string `json:"answer"`
|
||||
}
|
||||
|
||||
// TakeIn reads a registered module's manifest now, whatever was tried before, and writes it as a merge would (the tool
|
||||
// indexer_take_in, issue 1945). A module the modules state holds is answered so and left; one with no registration in the
|
||||
// controller's records, or whose every registration could not be read or holds no manifest of it, is answered with why.
|
||||
func (ix *Indexer) TakeIn(ctx context.Context, module string) TakenIn {
|
||||
out := TakenIn{Module: module}
|
||||
if ix.Builds == nil {
|
||||
out.Answer = "the indexer reads no controller's build records, so it knows no registration"
|
||||
return out
|
||||
}
|
||||
regs := ix.Builds.RegistrationsOf(module)
|
||||
if len(regs) == 0 {
|
||||
if key, err := delivery.ModulesKey(module); err == nil && ix.Modules != nil {
|
||||
if _, _, found, err := ix.Modules.Get(ctx, key); err == nil && found {
|
||||
out.Held, out.Answer = true, "the modules state holds it already: a merge to its repository writes it from now on"
|
||||
return out
|
||||
}
|
||||
}
|
||||
out.Answer = "the controller's build records name no registration of " + module + ": a module from outside the indexed repositories is known only from them, and one inside them from a merge"
|
||||
return out
|
||||
}
|
||||
var whys []string
|
||||
for _, r := range regs {
|
||||
t, err := ix.indexRegistered(ctx, r, true)
|
||||
out.Repository, out.Commit, out.Path = t.repository, t.commit, pathOf(t.dir)
|
||||
switch {
|
||||
case err != nil:
|
||||
whys = append(whys, err.Error())
|
||||
continue
|
||||
case t.held:
|
||||
out.Held, out.Answer = true, t.why
|
||||
out.Repository, out.Commit, out.Path = "", "", ""
|
||||
return out
|
||||
case t.stored:
|
||||
out.Stored = true
|
||||
out.Answer = fmt.Sprintf("written to the modules state from %s at %s", t.repository, t.commit)
|
||||
return out
|
||||
}
|
||||
whys = append(whys, t.why)
|
||||
}
|
||||
out.Answer = "not taken in: " + strings.Join(whys, "; ")
|
||||
return out
|
||||
}
|
||||
|
||||
// IndexRegisteredModule indexes the registration the controller's records name for a module, when there is one.
|
||||
func (ix *Indexer) IndexRegisteredModule(ctx context.Context, module string) error {
|
||||
if ix.Builds == nil {
|
||||
return nil
|
||||
}
|
||||
var first error
|
||||
for _, r := range ix.Builds.RegistrationsOf(module) {
|
||||
if err := ix.IndexRegistered(ctx, r); err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
// IndexRegisteredAll indexes every registered module the modules bucket does not hold. A module that cannot be read is
|
||||
// said and the others go on; the first error is answered.
|
||||
func (ix *Indexer) IndexRegisteredAll(ctx context.Context) error {
|
||||
if ix.Builds == nil {
|
||||
return nil
|
||||
}
|
||||
var first error
|
||||
for _, r := range ix.Builds.Registered() {
|
||||
if err := ix.IndexRegistered(ctx, r); err != nil {
|
||||
ix.logf("the registered module %s is not indexed: %v", r.Module, err)
|
||||
if first == nil {
|
||||
first = err
|
||||
}
|
||||
}
|
||||
}
|
||||
return first
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func TestARegisteredModuleIsIndexedFromItsRepository(t *testing.T) { // the grants part's gaps: modules registered from their own repositories had no manifest here
|
||||
const forge = "http://novox.internal:20000/jschoubben/snake.git"
|
||||
snake := `{"module":"snake","version":"1"}`
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
path string
|
||||
read RepoRead
|
||||
err error
|
||||
ref string // the build record's ref
|
||||
heads map[string]string // the gate source's heads
|
||||
wantRead bool // the forge is read
|
||||
held bool // modules.snake exists already
|
||||
other bool // modules.other of the same repository exists
|
||||
want bool // modules.snake is written
|
||||
}{
|
||||
{wantRead: true, name: "a module at the root of its repository", path: "", want: true,
|
||||
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
|
||||
{wantRead: true, name: "a root spelled as a dot", path: ".", want: true,
|
||||
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
|
||||
{wantRead: true, name: "a module in a directory", path: "modules/snake", want: true,
|
||||
read: RepoRead{Root: "r0", Trees: map[string]string{"modules/snake": "t1"}, Manifests: map[string]string{"modules/snake": snake}}},
|
||||
{wantRead: true, name: "another module of the repository is neither written nor dropped", path: "", want: true, other: true,
|
||||
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
|
||||
{name: "a module the bucket holds is left", path: "", held: true,
|
||||
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
|
||||
{wantRead: true, name: "no manifest at the path", path: "modules/x",
|
||||
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
|
||||
{wantRead: true, name: "a manifest of another module", path: "",
|
||||
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": `{"module":"other","version":"1"}`}}},
|
||||
{wantRead: true, name: "a repository whose default branch is master", path: "", ref: "master", want: true,
|
||||
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
|
||||
{wantRead: true, name: "a repository built from a branch of its own (nox-mesh)", path: "", ref: "nox-mesh", want: true,
|
||||
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
|
||||
{wantRead: true, name: "a repository the source names, the module unheld and present at the head", path: "", want: true, heads: map[string]string{"jschoubben/snake": h2},
|
||||
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
|
||||
{wantRead: true, name: "a forge that cannot say", path: "", err: errors.New("down")},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "snake", forge, tc.path, tc.ref, "", "2026-10-19T09:00:00Z", ""))
|
||||
if tc.held {
|
||||
if err := mods.Create(context.Background(), "snake", mustModule(t, delivery.Module{Repository: "jschoubben/snake", Commit: h1, Path: ".", None2: "x"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if tc.other {
|
||||
if err := mods.Create(context.Background(), "other", mustModule(t, delivery.Module{Repository: "jschoubben/snake", Commit: h1, Path: "modules/other", None2: "x"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
ix.Manifests = newMem()
|
||||
var asked struct {
|
||||
repo, sha, dir string
|
||||
n int
|
||||
}
|
||||
ix.Read = func(_ context.Context, repo, sha, dir string) (RepoRead, error) {
|
||||
asked.repo, asked.sha, asked.dir = repo, sha, dir
|
||||
asked.n++
|
||||
return tc.read, tc.err
|
||||
}
|
||||
if tc.heads != nil {
|
||||
ix.source = sourceNow{rev: 1, heads: tc.heads}
|
||||
}
|
||||
err := ix.IndexRegisteredAll(context.Background())
|
||||
if (err != nil) != (tc.err != nil) {
|
||||
t.Fatalf("error = %v", err)
|
||||
}
|
||||
wantCommit := "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" // the record's
|
||||
if tc.heads != nil {
|
||||
wantCommit = h2 // the source's head
|
||||
}
|
||||
if tc.wantRead && asked.sha != wantCommit {
|
||||
t.Errorf("the forge was asked at %s, want %s", asked.sha, wantCommit)
|
||||
}
|
||||
wantReads := 1
|
||||
if !tc.wantRead {
|
||||
wantReads = 0
|
||||
}
|
||||
if tc.wantRead && asked.dir != dirOf(tc.path) {
|
||||
t.Errorf("the forge was asked for directory %q", asked.dir)
|
||||
}
|
||||
if asked.n != wantReads {
|
||||
t.Errorf("forge reads = %d, want %d", asked.n, wantReads)
|
||||
}
|
||||
m, ok := moduleOf(t, mods, "snake")
|
||||
if tc.want {
|
||||
if !ok || m.Manifest == "" || m.Repository != "jschoubben/snake" || m.Commit != wantCommit || m.Path != pathOf(dirOf(tc.path)) {
|
||||
t.Fatalf("modules.snake = %+v, found %v", m, ok)
|
||||
}
|
||||
if len(ix.Manifests.(*memStore).vals) != 1 {
|
||||
t.Errorf("manifests written = %d, want 1", len(ix.Manifests.(*memStore).vals))
|
||||
}
|
||||
if asked.repo != "jschoubben/snake" {
|
||||
t.Errorf("the forge was asked for %q", asked.repo)
|
||||
}
|
||||
} else if !tc.held && ok {
|
||||
t.Fatalf("modules.snake written: %+v", m)
|
||||
}
|
||||
if _, ok := moduleOf(t, mods, "other"); ok != tc.other {
|
||||
t.Errorf("modules.other held = %v, want %v", ok, tc.other)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func mustModule(t *testing.T, m delivery.Module) []byte {
|
||||
t.Helper()
|
||||
b, err := delivery.EncodeModule(withV(m))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func dirOf(p string) string {
|
||||
if p == "." {
|
||||
return ""
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func TestARegistrationOfAnotherRepositoryDoesNotHideTheRealOne(t *testing.T) { // photos: its newest trunk record was a directory of the catalogue's, then novox/photos at nox-mesh
|
||||
const catalogue, own = "http://novox.internal:20000/novox/mesh-catalog.git", "http://novox.internal:20000/novox/Link2Pay.git"
|
||||
ix, mods := modulesIndexer(t,
|
||||
rec(t, "build-1", "link2pay", own, "", "nox-mesh", "", "2026-10-06T23:02:00Z", ""),
|
||||
rec(t, "build-2", "link2pay", catalogue, "modules/link2pay", "main", "", "2026-10-04T16:02:00Z", ""),
|
||||
rec(t, "build-3", "link2pay", catalogue, "modules/link2pay", "main", "", "2026-10-03T16:02:00Z", ""))
|
||||
ix.Manifests = newMem()
|
||||
var asked []string
|
||||
ix.Read = func(_ context.Context, repo, _, dir string) (RepoRead, error) {
|
||||
asked = append(asked, repo+":"+dir)
|
||||
if repo == "novox/Link2Pay" {
|
||||
return RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": `{"module":"link2pay","version":"1"}`}}, nil
|
||||
}
|
||||
return RepoRead{Root: "r0"}, nil
|
||||
}
|
||||
if err := ix.IndexRegisteredAll(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, ok := moduleOf(t, mods, "link2pay")
|
||||
if !ok || m.Repository != "novox/Link2Pay" {
|
||||
t.Fatalf("modules.link2pay = %+v, found %v", m, ok)
|
||||
}
|
||||
if len(asked) != 2 || asked[0] != "novox/mesh-catalog:modules/link2pay" || asked[1] != "novox/Link2Pay:." && asked[1] != "novox/Link2Pay:" {
|
||||
t.Errorf("the forge was asked %v: want the catalogue's trunk record first, once, then the repository's own", asked)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAForgeThatCannotSayLeavesTheModuleUnwrittenAndAsksAgain(t *testing.T) { // a timeout or a 404 at the head: no read at an older commit
|
||||
// Issue 1945: car-hunter and snake answered 404 until the forge's account could read them, and were read again only when
|
||||
// the indexer started again. Each row's trigger must now take the module in once the forge reads it.
|
||||
const snake = `{"module":"snake","version":"1"}`
|
||||
ctx := context.Background()
|
||||
for _, tc := range []struct{ failure, trigger string }{
|
||||
{"timeout", "the indexer starts again"},
|
||||
{"Gitea API /repos/x/git/trees/y: 404 not found", "the indexer starts again"},
|
||||
{"Gitea API /repos/x/git/trees/y: 404 not found", "a new source revision"},
|
||||
{"Gitea API /repos/x/git/trees/y: 404 not found", "a merged message of another repository"},
|
||||
{"Gitea API /repos/x/git/trees/y: 404 not found", "a person's take-in"},
|
||||
{"timeout", "a person's take-in"},
|
||||
} {
|
||||
t.Run(tc.failure+", then "+tc.trigger, func(t *testing.T) {
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "snake", "http://novox.internal:20000/jschoubben/snake.git", "", "master", "", "2026-10-19T09:00:00Z", ""))
|
||||
ix.Manifests = newMem()
|
||||
ix.source = sourceNow{rev: 1, heads: map[string]string{"jschoubben/snake": h2}}
|
||||
var asked []string
|
||||
readable := false
|
||||
ix.Read = func(_ context.Context, _, sha, _ string) (RepoRead, error) {
|
||||
asked = append(asked, sha)
|
||||
if readable {
|
||||
return RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}, nil
|
||||
}
|
||||
return RepoRead{}, errors.New(tc.failure)
|
||||
}
|
||||
if err := ix.IndexRegisteredAll(ctx); err == nil {
|
||||
t.Errorf("no error")
|
||||
}
|
||||
if _, ok := moduleOf(t, mods, "snake"); ok || len(asked) != 1 || asked[0] != h2 {
|
||||
t.Errorf("written %v, asked %v", ok, asked)
|
||||
}
|
||||
if err := ix.IndexRegisteredAll(ctx); err == nil || len(asked) != 2 {
|
||||
t.Errorf("not asked again (asked %v)", asked)
|
||||
}
|
||||
readable = true // the forge's account reads the repository now
|
||||
var got TakenIn
|
||||
switch tc.trigger {
|
||||
case "the indexer starts again":
|
||||
if err := ix.IndexRegisteredAll(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "a new source revision":
|
||||
if err := ix.ApplySource(ctx, 2, delivery.Source{Heads: map[string]string{"jschoubben/snake": h2, "novox/other": h1}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ix.retryWG.Wait() // read again in the background
|
||||
case "a merged message of another repository":
|
||||
if err := ix.Apply(ctx, mergedMsg("novox/other", map[string]string{"": "t9"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ix.retryWG.Wait() // read again in the background
|
||||
case "a person's take-in":
|
||||
got = ix.TakeIn(ctx, "snake")
|
||||
if !got.Stored || got.Repository != "jschoubben/snake" || got.Commit != h2 || got.Path != "." || got.Answer == "" {
|
||||
t.Errorf("take-in answered %+v", got)
|
||||
}
|
||||
}
|
||||
if m, ok := moduleOf(t, mods, "snake"); !ok || m.Manifest == "" || m.Commit != h2 {
|
||||
t.Fatalf("after %s the readable module is not taken in: %+v, found %v (asked %v)", tc.trigger, m, ok, asked)
|
||||
}
|
||||
// Taken in, it is read no more: a further event asks the forge nothing.
|
||||
n := len(asked)
|
||||
if err := ix.ApplySource(ctx, 3, delivery.Source{Heads: map[string]string{"jschoubben/snake": h2}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ix.retryWG.Wait()
|
||||
if len(asked) != n {
|
||||
t.Errorf("the forge was asked again after the module was taken in: %v", asked)
|
||||
}
|
||||
if again := ix.TakeIn(ctx, "snake"); again.Stored || !again.Held || len(asked) != n {
|
||||
t.Errorf("a take-in of a held module answered %+v and asked %v", again, asked)
|
||||
}
|
||||
if none := ix.TakeIn(ctx, "nobody"); none.Stored || none.Held || !strings.Contains(none.Answer, "no registration") {
|
||||
t.Errorf("a take-in of an unregistered module answered %+v", none)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestARecordOfAPullRequestDoesNotReplaceTheRegisteredRef(t *testing.T) { // a newer feat/x record of the same place
|
||||
const repo = "http://novox.internal:20000/novox/photos.git"
|
||||
b := NewBuilds()
|
||||
for _, r := range [][2]any{
|
||||
rec(t, "b1", "photos", repo, "", "nox-mesh", "", "2026-10-01T09:00:00Z", ""),
|
||||
rec(t, "b2", "photos", repo, "", "nox-mesh", "", "2026-10-02T09:00:00Z", ""),
|
||||
rec(t, "b3", "photos", repo, "", "feat/x", "", "2026-10-09T09:00:00Z", ""),
|
||||
} {
|
||||
b.Put(r[0].(string), r[1].([]byte))
|
||||
}
|
||||
got := b.RegistrationsOf("photos")
|
||||
if len(got) != 1 || got[0].Ref != "nox-mesh" {
|
||||
t.Fatalf("registrations = %+v, want the ref most records name, not the newer feat/x", got)
|
||||
}
|
||||
k, v := record(t, "b4", "photos", repo, "", "main", "", "2026-10-03T09:00:00Z", "")
|
||||
b.Put(k, v) // a trunk's ref beats a branch of its own
|
||||
if got := b.RegistrationsOf("photos"); len(got) != 1 || got[0].Ref != "main" {
|
||||
t.Fatalf("registrations = %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -47,6 +47,10 @@ func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s delivery.Sourc
|
||||
if ix.Modules == nil {
|
||||
return ix.RecomputeCandidates(ctx)
|
||||
}
|
||||
// A registered module the forge could not read is read again at each new source revision (issue 1945), in the
|
||||
// background: a forge that does not answer must not hold this revision. One taken in later is stamped when it is written
|
||||
// (applyManifests reads srcOf).
|
||||
ix.kickRetry(ctx)
|
||||
held, err := ix.heldModules(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The indexer's doing tool (novox/hq issue 1945; the instruction "tools are how a module is spoken to"): indexer_take_in
|
||||
// reads a registered module's manifest now and writes it to the modules state. The tools bundle serves it through the tool
|
||||
// runner; the indexer's process serves no request subject (ADR 0437 §4), so the tool writes an ask into the indexer's
|
||||
// state takeins, which the indexer watches and answers there, as the builder answers builder_ask_again. The read tools
|
||||
// of the indexer's states stay in mesh-delivery-view.
|
||||
const (
|
||||
TakeInTool = "indexer_take_in"
|
||||
// TakeInsState is the indexer's state of take-in asks: the bucket mesh-delivery-indexer_takeins, keyed as the modules state.
|
||||
TakeInsState = "takeins"
|
||||
TakeInsBucket = delivery.IndexerModule + "_" + TakeInsState
|
||||
)
|
||||
|
||||
// TakeInAsk is a value of the takeins state: which module a person asked to be taken in, why, who and when, and, once the
|
||||
// indexer read it, its answer.
|
||||
type TakeInAsk struct {
|
||||
V int `json:"v"`
|
||||
Module string `json:"module"`
|
||||
Why string `json:"why,omitempty"`
|
||||
By string `json:"by,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
Result *TakenIn `json:"result,omitempty"`
|
||||
Answered time.Time `json:"answered,omitzero"`
|
||||
}
|
||||
|
||||
// DecodeTakeIn reads a takeins value.
|
||||
func DecodeTakeIn(data []byte) (TakeInAsk, error) {
|
||||
var a TakeInAsk
|
||||
if err := json.Unmarshal(data, &a); err != nil {
|
||||
return a, err
|
||||
}
|
||||
if a.V != 1 {
|
||||
return a, fmt.Errorf("a takeins value of version %d, not 1", a.V)
|
||||
}
|
||||
if a.Module == "" {
|
||||
return a, errors.New("a takeins value naming no module")
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// NewTakeIn is the value indexer_take_in writes for a module: its key and the value.
|
||||
func NewTakeIn(module, why, by string, at time.Time) (string, []byte, error) {
|
||||
module = strings.TrimSpace(module)
|
||||
if module == "" {
|
||||
return "", nil, errors.New("name the module to take in (module)")
|
||||
}
|
||||
key, err := delivery.ModulesKey(module)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
data, err := json.Marshal(TakeInAsk{V: 1, Module: module, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
|
||||
return key, data, err
|
||||
}
|
||||
@@ -24,18 +24,21 @@ type state struct {
|
||||
}
|
||||
|
||||
type artifact struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Language string `json:"language"`
|
||||
System string `json:"system"`
|
||||
From string `json:"from"`
|
||||
Binary string `json:"binary"`
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Language string `json:"language"`
|
||||
System string `json:"system"`
|
||||
From string `json:"from"`
|
||||
Binary string `json:"binary"`
|
||||
Loads []string `json:"loads"`
|
||||
}
|
||||
|
||||
type manifest struct {
|
||||
Module string `json:"module"`
|
||||
Version string `json:"version"`
|
||||
Slug string `json:"slug"`
|
||||
Tools []string `json:"tools"`
|
||||
Emits []string `json:"emits"`
|
||||
Consumes []string `json:"consumes"`
|
||||
State []state `json:"state"`
|
||||
Reads []string `json:"reads"`
|
||||
@@ -78,7 +81,11 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
if m.Module != "mesh-delivery-indexer" {
|
||||
t.Errorf("module = %q", m.Module)
|
||||
}
|
||||
if want := []state{{Name: "candidates", History: 10}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}}; !reflect.DeepEqual(m.State, want) {
|
||||
// 1126 (ADR 0549 §2): the indexer's placement state, and the event it says when a node's placement changed.
|
||||
if want := []string{"placement-changed"}; !reflect.DeepEqual(m.Emits, want) {
|
||||
t.Errorf("emits = %v, want %v", m.Emits, want)
|
||||
}
|
||||
if want := []state{{Name: "candidates", History: 10}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "placement", History: 10}, {Name: "takeins", History: 1}}; !reflect.DeepEqual(m.State, want) {
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
}
|
||||
if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-controller.assignments", "mesh-controller.builds", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||
@@ -95,14 +102,18 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
if m.SecretsOwner != "mesh-delivery-indexer" {
|
||||
t.Errorf("secrets-owner = %q", m.SecretsOwner)
|
||||
}
|
||||
// Two invokes: the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths:
|
||||
// Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths:
|
||||
// ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a
|
||||
// commit is in another (issue 1900).
|
||||
if want := []string{"gitea.gitea_commit_trees", "gitea.gitea_contains"}; !reflect.DeepEqual(m.Invokes, want) {
|
||||
if want := []string{"gitea.gitea_commit_trees", "gitea.gitea_contains", "gitea.gitea_get_file"}; !reflect.DeepEqual(m.Invokes, want) {
|
||||
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||
}
|
||||
// No emits, no streams, no bus, no seat, no tool (ADR 0437 §4).
|
||||
for _, k := range []string{"streams", "logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats", "bus"} {
|
||||
// 1945: one tool, indexer_take_in, served by its tools bundle through the takeins state; the process serves none.
|
||||
if want := []string{"indexer_take_in"}; !reflect.DeepEqual(m.Tools, want) {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
// No streams, no bus, no seat (ADR 0437 §4); its one event is placement-changed (ADR 0549 §2), checked above.
|
||||
for _, k := range []string{"streams", "logs", "claims", "provides", "requires", "uses", "defines-seats", "seat", "seats", "bus"} {
|
||||
if _, ok := top[k]; ok {
|
||||
t.Errorf("the manifest declares %q: the indexer has none", k)
|
||||
}
|
||||
@@ -134,8 +145,13 @@ func TestManifestProcess(t *testing.T) { // 1238/AC1
|
||||
if _, bus := p["bus"]; bus {
|
||||
t.Errorf("the process declares a bus: it runs on its own broker credential")
|
||||
}
|
||||
if len(m.Build.Artifacts) != 1 {
|
||||
t.Fatalf("artifacts = %+v", m.Build.Artifacts)
|
||||
if len(m.Build.Artifacts) != 2 {
|
||||
t.Fatalf("artifacts = %+v, want the program and the tools", m.Build.Artifacts)
|
||||
}
|
||||
if a := m.Build.Artifacts[1]; a.Name != "tools" || a.Kind != "bundle" || a.Binary != "mesh-delivery-indexer-tools" || !reflect.DeepEqual(a.Loads, []string{"mesh-delivery-indexer-tools"}) {
|
||||
t.Errorf("tools artifact = %+v", a) // 1945
|
||||
} else if st, err := os.Stat(a.From); err != nil || !st.IsDir() {
|
||||
t.Errorf("the tools' source %s: %v", a.From, err)
|
||||
}
|
||||
a := m.Build.Artifacts[0]
|
||||
if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-indexer" || !strings.HasSuffix(a.From, a.Binary) {
|
||||
|
||||
@@ -2,6 +2,12 @@
|
||||
"module": "mesh-delivery-indexer",
|
||||
"version": "1",
|
||||
"slug": "indexer",
|
||||
"tools": [
|
||||
"indexer_take_in"
|
||||
],
|
||||
"emits": [
|
||||
"placement-changed"
|
||||
],
|
||||
"consumes": [
|
||||
"mesh-build-go.toolchain.reported",
|
||||
"mesh-build-node.toolchain.reported",
|
||||
@@ -11,7 +17,9 @@
|
||||
"state": [
|
||||
{ "name": "candidates", "history": 10 },
|
||||
{ "name": "manifests", "history": 1 },
|
||||
{ "name": "modules", "history": 20 }
|
||||
{ "name": "modules", "history": 20 },
|
||||
{ "name": "placement", "history": 10 },
|
||||
{ "name": "takeins", "history": 1 }
|
||||
],
|
||||
"reads": [
|
||||
"mesh-build-go.toolchains",
|
||||
@@ -23,7 +31,8 @@
|
||||
],
|
||||
"invokes": [
|
||||
"gitea.gitea_commit_trees",
|
||||
"gitea.gitea_contains"
|
||||
"gitea.gitea_contains",
|
||||
"gitea.gitea_get_file"
|
||||
],
|
||||
"own-secrets": {
|
||||
"broker": "${dir:state}/broker"
|
||||
@@ -71,6 +80,17 @@
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-delivery-indexer",
|
||||
"binary": "mesh-delivery-indexer"
|
||||
},
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-delivery-indexer-tools",
|
||||
"binary": "mesh-delivery-indexer-tools",
|
||||
"loads": [
|
||||
"mesh-delivery-indexer-tools"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -11,12 +11,25 @@ import (
|
||||
|
||||
"mesh-delivery-view/internal/view"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// Tools are the read tools over a modules state.
|
||||
func Tools(r view.Reader) []stdio.Tool {
|
||||
// Tools are the read tools over a modules state, and build_usage over the build modules' states.
|
||||
func Tools(r view.Reader, u view.UsageReader) []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{Name: view.BuildUsageTool,
|
||||
Description: "How the build nodes are used, and whether a node is eligible to build, per node (every node holding a build module or named by a capacity): each build seat it holds (node-build-go, -node, -image) with its capacity " +
|
||||
"(mesh-config's resolved value, or the modules' default) and pause, and whether it is eligible to build and why not; the asks " +
|
||||
"running now by kind (delivery, check, recheck, try), the keys left running (past the step bound, or their module not running) and the ones " +
|
||||
"their module ended for that (a restart, or past the step bound), counted apart and in no median or occupancy; " +
|
||||
"the steps started and finished in the last hour and the last day, failed ones, total and median time, by kind, and how long and " +
|
||||
"what share of the window the node ran its capacity of asks. Mesh-wide: the backlog of asks waiting and held on each build seat's " +
|
||||
"stream, and the tries running. Read from the bus's states (have, mesh-config, the build modules' steps and pauses, the checker's tries); " +
|
||||
"the backlog from the view program's reading of the streams; CPU is not on the bus and the answer names the per-node tool. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) { return u.Usage(), nil }},
|
||||
{Name: view.ModuleReadTool,
|
||||
Description: "What the indexer stored for one module under modules.<module>, as stored: its fingerprint (fp, none), the " +
|
||||
"new-form fingerprint (fp2) or why it has none (none2), the inputs it was computed from (inputs2), the module's " +
|
||||
@@ -37,6 +50,15 @@ func Tools(r view.Reader) []stdio.Tool {
|
||||
mods, err := r.Modules(only)
|
||||
return map[string]any{"modules": mods}, err
|
||||
}},
|
||||
{Name: view.PlacementReadTool,
|
||||
Description: "Where modules run, as the indexer read it from the catalogue's placement files at the last merge (placement.<node>, novox/hq ADR 0549): " +
|
||||
"each node's modules in order with their host ports, bindings and settings placement, its pins and the seats its modules hold, and the catalogue revision. " +
|
||||
"With node, that node's; with module, every node placing it; with neither, every node's. Keys that do not read are listed as unreadable. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": nodeModule},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
node, module := nodeModuleOf(args)
|
||||
return r.Placement(node, module)
|
||||
}},
|
||||
{Name: view.CandidateReadTool,
|
||||
Description: "What the indexer stored for one candidate (the open pull requests of a branch) under candidates.<group>, as stored: " +
|
||||
"the pull requests and heads, the base, and per pull request the touched modules with their new-form fingerprint (fp2) or why " +
|
||||
@@ -131,6 +153,24 @@ func nodeModuleOf(args map[string]any) (string, string) {
|
||||
return strings.TrimSpace(node), strings.TrimSpace(module)
|
||||
}
|
||||
|
||||
// socket is the view program's socket, as the program reads it (MESH_DELIVERY_SOCKET, else its default).
|
||||
func socket() string {
|
||||
if s := strings.TrimSpace(os.Getenv("MESH_DELIVERY_SOCKET")); s != "" {
|
||||
return s
|
||||
}
|
||||
return "/var/lib/mesh-delivery-view-cli/cli.sock"
|
||||
}
|
||||
|
||||
// backlogOf asks the view program on its socket for the backlog it reads from the seats' streams' workers: the tool
|
||||
// runtime grants a bundle states, and the stream and worker information is granted to the view program's own account
|
||||
// (mesh-sdk buscompose.BuildBacklogReads).
|
||||
func backlogOf(path string) func() ([]view.BacklogRow, error) {
|
||||
return func() ([]view.BacklogRow, error) {
|
||||
s, err := view.Ask(path)
|
||||
return s.Backlog, err
|
||||
}
|
||||
}
|
||||
|
||||
func read(state stdio.KeptState) view.StateRead {
|
||||
return view.StateRead{Keys: state.Keys, GetEntry: func(key string) (json.RawMessage, uint64, error) {
|
||||
e, err := state.Get(key)
|
||||
@@ -149,8 +189,16 @@ func read(state stdio.KeptState) view.StateRead {
|
||||
|
||||
func main() {
|
||||
r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds")), CandidatesState: read(stdio.State("mesh-delivery-indexer.candidates")),
|
||||
WantsState: read(stdio.State("mesh-delivery-assigner.wants")), DifferencesState: read(stdio.State("mesh-delivery-assigner.differences")), HavesState: read(stdio.State("node-engine.have"))}
|
||||
if err := stdio.Serve("", Tools(r)); err != nil {
|
||||
WantsState: read(stdio.State("mesh-delivery-assigner.wants")), DifferencesState: read(stdio.State("mesh-delivery-assigner.differences")), HavesState: read(stdio.State("node-engine.have")),
|
||||
PlacementState: read(stdio.State("mesh-delivery-indexer.placement"))}
|
||||
u := view.UsageReader{Haves: r.HavesState, Resolved: read(stdio.State("mesh-config.resolved")), Tries: read(stdio.State("mesh-delivery-checker.tries")),
|
||||
Steps: map[string]view.StateRead{}, Pauses: map[string]view.StateRead{}, Backlog: backlogOf(socket())}
|
||||
for _, lang := range delivery.AskLanguages {
|
||||
module, _ := delivery.BuildModuleOf(lang)
|
||||
u.Steps[lang] = read(stdio.State(module + "." + delivery.StepsState))
|
||||
u.Pauses[lang] = read(stdio.State(module + "." + delivery.PausesState))
|
||||
}
|
||||
if err := stdio.Serve("", Tools(r, u)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-view-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -33,7 +33,7 @@ func (f fake) reader() view.Reader {
|
||||
|
||||
func run(t *testing.T, r view.Reader, name string, args map[string]any) (any, error) {
|
||||
t.Helper()
|
||||
for _, tl := range Tools(r) {
|
||||
for _, tl := range Tools(r, view.UsageReader{}) {
|
||||
if tl.Name == name {
|
||||
return tl.Run(args)
|
||||
}
|
||||
@@ -64,11 +64,7 @@ func TestModuleToolAnswersTheStoredValue(t *testing.T) {
|
||||
if err := json.Unmarshal(out.(json.RawMessage), &m); err != nil || m["fp"] != "a1" || m["fp2"] != "b2" || m["path"] != "." || m["inputs2"] == nil || m["toolchains"] == nil {
|
||||
t.Errorf("stored fields are not all there: %v %v", m, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A module with no key, and a name that is no key, are said so.
|
||||
func TestModuleToolRefusesWhatIsNotThere(t *testing.T) {
|
||||
r := fake{"a": withFP2}.reader()
|
||||
// A module with no key, and a name that is no key, are said so.
|
||||
if _, err := run(t, r, view.ModuleReadTool, map[string]any{"module": "gone"}); err == nil || !strings.Contains(err.Error(), "no module gone") {
|
||||
t.Errorf("a missing module: %v", err)
|
||||
}
|
||||
@@ -96,6 +92,32 @@ func TestModulesToolAnswersEveryKeyAsStored(t *testing.T) {
|
||||
if len(only) != 1 || string(only["b"]) != withNone {
|
||||
t.Errorf("none2 modules = %v", only)
|
||||
}
|
||||
|
||||
// 1126 (ADR 0549 §2): indexer_placement answers a node's key as stored, every node placing a module, or every node's,
|
||||
// and lists a key that does not read.
|
||||
const ace, g14 = `{"v":1,"node":"ace","revision":"m1","modules":[{"module":"plex"}]}`, `{"v":1,"node":"g14","revision":"m1","modules":[{"module":"zsh"},{"module":"plex"}]}`
|
||||
p := view.Reader{PlacementState: fake{"placement.ace": ace, "placement.g14": g14, "placement.bad": `{"v":1}`, "other": "{}"}.reader().State}
|
||||
got := func(args map[string]any) map[string]json.RawMessage {
|
||||
t.Helper()
|
||||
out, err := run(t, p, view.PlacementReadTool, args)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out.(map[string]any)["placement"].(map[string]json.RawMessage)
|
||||
}
|
||||
if one := got(map[string]any{"node": "ace"}); len(one) != 1 || string(one["ace"]) != ace {
|
||||
t.Errorf("node ace = %v", one)
|
||||
}
|
||||
if zsh := got(map[string]any{"module": "zsh"}); len(zsh) != 1 || string(zsh["g14"]) != g14 {
|
||||
t.Errorf("module zsh = %v", zsh)
|
||||
}
|
||||
out, err = run(t, p, view.PlacementReadTool, map[string]any{})
|
||||
if all := out.(map[string]any); err != nil || len(all["placement"].(map[string]json.RawMessage)) != 2 || len(all["unreadable"].([]string)) != 2 {
|
||||
t.Errorf("every node = %v, %v", all, err)
|
||||
}
|
||||
if _, err := run(t, p, view.PlacementReadTool, map[string]any{"node": "novox"}); err == nil || !strings.Contains(err.Error(), "no placement of novox") {
|
||||
t.Errorf("a node with no key: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-view
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025224357-095d41fa3e81
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026193323-03dce1237803
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025224357-095d41fa3e81 h1:LhljEtt7FbNpVn67eOQbdsFDS8Qj6WPUF6l9IXr9mo4=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025224357-095d41fa3e81/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026193323-03dce1237803 h1:RInAboHBk5THYf6Agvy05iO9p2XV5zGb0ilYa+iSO7o=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026193323-03dce1237803/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -20,6 +20,8 @@ import (
|
||||
const (
|
||||
ModuleReadTool = "indexer_module"
|
||||
ModulesReadTool = "indexer_modules"
|
||||
// PlacementReadTool reads the indexer's placement (novox/hq ADR 0549 §2, issue 1126).
|
||||
PlacementReadTool = "indexer_placement"
|
||||
)
|
||||
|
||||
// StateRead reads the indexer's modules state, as the view's reads grant it: its keys and a key's stored value (nil for a key with none).
|
||||
@@ -39,6 +41,7 @@ type Reader struct {
|
||||
WantsState StateRead // the assigner's wants and holds
|
||||
DifferencesState StateRead // the assigner's differences
|
||||
HavesState StateRead // the node-engine's haves
|
||||
PlacementState StateRead // the indexer's placement
|
||||
}
|
||||
|
||||
// Module is the stored value of modules.<module>, exactly as the indexer wrote it.
|
||||
@@ -613,3 +616,55 @@ func (r Reader) Differences(node, module string) (KeptAnswer, error) {
|
||||
func (r Reader) Haves(node, module string) (KeptAnswer, error) {
|
||||
return r.kept(r.HavesState, "the haves", node, module)
|
||||
}
|
||||
|
||||
// Placement is the indexer's placement as stored (ADR 0549 §2): one node's key with node, every node placing a module
|
||||
// with module (each node's key whole, so its pins, ports and bindings are beside the module), and every node's key with
|
||||
// neither. A key that does not read as a placement is listed as unreadable, never dropped.
|
||||
func (r Reader) Placement(node, module string) (map[string]any, error) {
|
||||
node, module = strings.TrimSpace(node), strings.TrimSpace(module)
|
||||
if r.PlacementState.Keys == nil || r.PlacementState.Get == nil {
|
||||
return nil, fmt.Errorf("the placement state cannot be read here")
|
||||
}
|
||||
if node != "" {
|
||||
key, err := delivery.PlacementKey(node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := r.PlacementState.Get(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return nil, fmt.Errorf("no placement of %s: the catalogue holds no placement file for it, or the indexer has read none yet", node)
|
||||
}
|
||||
return map[string]any{"placement": map[string]json.RawMessage{node: raw}}, nil
|
||||
}
|
||||
keys, err := r.PlacementState.Keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(keys)
|
||||
values, err := r.PlacementState.values(keys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]json.RawMessage{}
|
||||
var unreadable []string
|
||||
for i, key := range keys {
|
||||
n, err := delivery.ParsePlacementKey(key)
|
||||
if err != nil {
|
||||
unreadable = append(unreadable, key)
|
||||
continue
|
||||
}
|
||||
p, err := delivery.DecodePlacement(values[i])
|
||||
if err != nil {
|
||||
unreadable = append(unreadable, key)
|
||||
continue
|
||||
}
|
||||
if module != "" && !slices.Contains(p.Assigned(), module) {
|
||||
continue
|
||||
}
|
||||
out[n] = values[i]
|
||||
}
|
||||
return map[string]any{"placement": out, "unreadable": unreadable}, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,692 @@
|
||||
package view
|
||||
|
||||
// The build nodes' use, in one answer (the operator, 2026-10-26: "how much are our build agents being used, and is ace
|
||||
// eligible for building?" took six calls over four tools). Read from the bus's state alone: which build seats each node
|
||||
// holds (the node-engine's have of mesh-build-<language>), its capacity for each (mesh-config's resolved value of the
|
||||
// module's setting `capacity` on that node, novox/hq ADR 0525 §5) and pause (the module's pauses state), and from each
|
||||
// build module's steps state (ADR 0525 §6: an ask ends on <node>.<ask id>) the asks running now by kind, the ones left
|
||||
// running, and the steps started and finished in the last hour and day with their total and median time and how long the
|
||||
// node was at its capacity. A try's asks are told from a candidate's by the checker's tries state (each try names the
|
||||
// asks it put). The backlog of the seats' streams is the view program's reading of the streams' workers (BacklogOver),
|
||||
// asked over the view's own socket: the tool runtime grants a bundle states, not stream information. CPU is not on the
|
||||
// bus: the disk-load module keeps no state, so the answer names the per-node tool that reads it.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// BuildUsageTool is the view's tool of the build nodes' use.
|
||||
const BuildUsageTool = "build_usage"
|
||||
|
||||
// endedStale says why a step's end was written by its build module for a key left running (mesh-sdk buildstep's ending
|
||||
// of stale steps): a restart's, or the step bound's; "" for a step that ran to its own end. Such an end is the moment the
|
||||
// module noticed, not the step's, so it is counted apart and in neither the medians nor the time at capacity.
|
||||
func endedStale(st delivery.Step) string {
|
||||
if st.State != delivery.StepFailed {
|
||||
return ""
|
||||
}
|
||||
if st.Reason == buildstep.RestartReason || strings.HasPrefix(st.Reason, "past the step bound") {
|
||||
return st.Reason
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// The kinds a step is counted under: an ask's kind (delivery, check, recheck), a try (a check ask a try of the checker
|
||||
// put), or hand (a build tool called directly, outside the seat's asks, which takes no slot of the capacity).
|
||||
const (
|
||||
KindTry = "try"
|
||||
KindHand = "hand"
|
||||
)
|
||||
|
||||
// UsageReader reads what build_usage answers from.
|
||||
type UsageReader struct {
|
||||
// Steps and Pauses are each build module's states by language (go, node, image).
|
||||
Steps map[string]StateRead
|
||||
Pauses map[string]StateRead
|
||||
// Resolved is mesh-config's resolved values (<node>.<module>.<key>); Haves the node-engine's have.
|
||||
Resolved StateRead
|
||||
Haves StateRead
|
||||
// Tries is the checker's tries (try-<unix ns>), each naming the asks it put.
|
||||
Tries StateRead
|
||||
// Backlog is the seats' streams' waiting and held asks, as the view program reads them.
|
||||
Backlog func() ([]BacklogRow, error)
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// UsageAnswer is build_usage's answer.
|
||||
type UsageAnswer struct {
|
||||
At time.Time `json:"at"`
|
||||
Nodes []NodeUsage `json:"nodes"`
|
||||
Backlog []BacklogRow `json:"backlog"`
|
||||
// TriesRunning are the tries the checker runs now; their asks are counted under check until the try ends and names them.
|
||||
TriesRunning []TryRunning `json:"tries_running"`
|
||||
Notes []string `json:"notes,omitempty"`
|
||||
}
|
||||
|
||||
// TryRunning is one try the checker runs now.
|
||||
type TryRunning struct {
|
||||
ID string `json:"id"`
|
||||
Repository string `json:"repository"`
|
||||
Started time.Time `json:"started,omitzero"`
|
||||
}
|
||||
|
||||
// NodeUsage is one node: each build seat it holds or is set to build for, whether it is eligible to build, and its CPU.
|
||||
type NodeUsage struct {
|
||||
Node string `json:"node"`
|
||||
// Eligible is the node's verdict: "eligible" when a seat of it is, "not eligible" when every seat is known not to be,
|
||||
// and "not known" when none is known eligible and this tool cannot tell for some seat.
|
||||
Eligible string `json:"eligible"`
|
||||
// Said is one line per seat, the answer in words.
|
||||
Said []string `json:"said"`
|
||||
Seats []SeatUsage `json:"seats"`
|
||||
CPU string `json:"cpu"`
|
||||
}
|
||||
|
||||
// SeatUsage is one build seat on one node.
|
||||
type SeatUsage struct {
|
||||
Seat string `json:"seat"`
|
||||
Module string `json:"module"`
|
||||
// Holds says the node runs the seat's module (its have says running); Phase is what its have says, or absent.
|
||||
Holds bool `json:"holds"`
|
||||
Phase string `json:"phase,omitempty"`
|
||||
// Capacity is the asks it runs at once; CapacityFrom where that came from.
|
||||
Capacity int `json:"capacity"`
|
||||
CapacityFrom string `json:"capacity_from"`
|
||||
Paused bool `json:"paused"`
|
||||
// Eligible says it takes asks: it holds the seat, has a capacity above 0 and is not paused; WhyNot says why not.
|
||||
// EligibleKnown is false when this tool cannot tell (it cannot read mesh-config's capacities): Eligible is then
|
||||
// absent, never false, and WhyNot starts "not known: ".
|
||||
Eligible *bool `json:"eligible,omitempty"`
|
||||
EligibleKnown bool `json:"eligible_known"`
|
||||
WhyNot string `json:"why_not,omitempty"`
|
||||
Running RunningUsage `json:"running"`
|
||||
LeftRunning []StepUsage `json:"left_running"`
|
||||
// EndedStale are the steps of the last day whose end their build module wrote for a key left running (a restart, or
|
||||
// past the step bound): counted here alone, never as finished.
|
||||
EndedStale []StepUsage `json:"ended_stale"`
|
||||
LastHour WindowUsage `json:"last_hour"`
|
||||
LastDay WindowUsage `json:"last_day"`
|
||||
Steps []StepUsage `json:"running_steps"`
|
||||
}
|
||||
|
||||
// usageStep is a step read, with the kind it is counted under.
|
||||
type usageStep struct {
|
||||
step delivery.Step
|
||||
kind string
|
||||
}
|
||||
|
||||
// RunningUsage is the asks running now: how many in all and by kind, against the capacity.
|
||||
type RunningUsage struct {
|
||||
Total int `json:"total"`
|
||||
ByKind map[string]int `json:"by_kind"`
|
||||
}
|
||||
|
||||
// StepUsage is one step running now, or one left running and why.
|
||||
type StepUsage struct {
|
||||
ID string `json:"id"`
|
||||
Kind string `json:"kind"`
|
||||
Tool string `json:"tool"`
|
||||
Started time.Time `json:"started"`
|
||||
Secs int64 `json:"secs"`
|
||||
Why string `json:"why,omitempty"`
|
||||
}
|
||||
|
||||
// WindowUsage is the steps of one window: started and finished in it, the finished ones' total and median time, by kind,
|
||||
// and how long the node ran its capacity of asks (a hand step takes no slot).
|
||||
type WindowUsage struct {
|
||||
Started int `json:"started"`
|
||||
Finished int `json:"finished"`
|
||||
Failed int `json:"failed"`
|
||||
TotalSecs int64 `json:"total_secs"`
|
||||
MedianSecs int64 `json:"median_secs"`
|
||||
ByKind map[string]KindWindow `json:"by_kind"`
|
||||
AtCapacitySecs int64 `json:"at_capacity_secs"`
|
||||
AtCapacityShare float64 `json:"at_capacity_share"`
|
||||
}
|
||||
|
||||
// KindWindow is one kind's finished steps in a window.
|
||||
type KindWindow struct {
|
||||
Finished int `json:"finished"`
|
||||
TotalSecs int64 `json:"total_secs"`
|
||||
MedianSecs int64 `json:"median_secs"`
|
||||
}
|
||||
|
||||
// stepIDTime is when a step id was made: its base-36 milliseconds before the six hex digits (buildstep.NewID).
|
||||
func stepIDTime(id string) (time.Time, bool) {
|
||||
if len(id) <= 6 {
|
||||
return time.Time{}, false
|
||||
}
|
||||
ms, err := strconv.ParseInt(id[:len(id)-6], 36, 64)
|
||||
if err != nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return time.UnixMilli(ms).UTC(), true
|
||||
}
|
||||
|
||||
// stepKind is the kind a step is counted under.
|
||||
func stepKind(s delivery.Step, tries map[string]bool) string {
|
||||
kind, ask := strings.CutSuffix(s.Tool, " ask")
|
||||
if !ask {
|
||||
return KindHand
|
||||
}
|
||||
if kind == delivery.AskCheck && tries[s.ID] {
|
||||
return KindTry
|
||||
}
|
||||
return kind
|
||||
}
|
||||
|
||||
// Usage reads every state and answers. A state it cannot read is said in a note, never left out silently.
|
||||
func (u UsageReader) Usage() UsageAnswer {
|
||||
now := time.Now().UTC()
|
||||
if u.Now != nil {
|
||||
now = u.Now().UTC()
|
||||
}
|
||||
ans := UsageAnswer{At: now, Nodes: []NodeUsage{}, Backlog: []BacklogRow{}, TriesRunning: []TryRunning{}}
|
||||
note := func(f string, a ...any) { ans.Notes = append(ans.Notes, fmt.Sprintf(f, a...)) }
|
||||
|
||||
tries := u.tries(now, &ans, note)
|
||||
seats := map[string]map[string]*SeatUsage{} // node -> language -> seat
|
||||
seat := func(node, lang string) *SeatUsage {
|
||||
if seats[node] == nil {
|
||||
seats[node] = map[string]*SeatUsage{}
|
||||
}
|
||||
if s := seats[node][lang]; s != nil {
|
||||
return s
|
||||
}
|
||||
name, _ := delivery.BuildSeat(lang)
|
||||
module, _ := delivery.BuildModuleOf(lang)
|
||||
s := &SeatUsage{Seat: name, Module: module, Running: RunningUsage{ByKind: map[string]int{}}, LeftRunning: []StepUsage{}, EndedStale: []StepUsage{}, Steps: []StepUsage{}}
|
||||
seats[node][lang] = s
|
||||
return s
|
||||
}
|
||||
|
||||
// Which node runs which build module: the have, <node>.mesh-build-<language>.
|
||||
haves := map[string]delivery.Have{}
|
||||
if u.Haves.Keys == nil || u.Haves.Get == nil {
|
||||
note("the node-engine's have is not readable here, so no seat is said held")
|
||||
} else if keys, err := u.Haves.Keys(); err != nil {
|
||||
note("the node-engine's have cannot be listed, so no seat is said held: %v", err)
|
||||
} else {
|
||||
var want []string
|
||||
for _, k := range keys {
|
||||
for _, lang := range delivery.AskLanguages {
|
||||
if m, _ := delivery.BuildModuleOf(lang); strings.HasSuffix(k, "."+m) {
|
||||
want = append(want, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
raws, err := u.Haves.values(want)
|
||||
if err != nil {
|
||||
note("the node-engine's have cannot be read: %v", err)
|
||||
}
|
||||
for i, k := range want {
|
||||
if i < len(raws) && len(raws[i]) > 0 {
|
||||
h, err := delivery.DecodeHave(raws[i])
|
||||
if err != nil {
|
||||
note("the have %s cannot be read, so its seat is not said held: %v", k, err)
|
||||
continue
|
||||
}
|
||||
haves[k] = h
|
||||
}
|
||||
}
|
||||
}
|
||||
for k, h := range haves {
|
||||
node, module, _ := strings.Cut(k, ".")
|
||||
lang := strings.TrimPrefix(module, "mesh-build-")
|
||||
s := seat(node, lang)
|
||||
s.Phase, s.Holds = string(h.Phase), h.Phase == delivery.PhaseRunning
|
||||
}
|
||||
|
||||
// Capacity: mesh-config's resolved value of the module's `capacity` on each node.
|
||||
resolved := map[string]string{} // <node>.<module> -> setting
|
||||
unknown := "" // why this tool cannot say a capacity: mesh-config's values cannot be read here
|
||||
unreadable := map[string]string{} // <node>.<module> -> why its resolved key holds no readable value: the puller uses 0
|
||||
if u.Resolved.Keys == nil || u.Resolved.Get == nil {
|
||||
unknown = "mesh-config's resolved values are not readable here"
|
||||
} else if keys, err := u.Resolved.Keys(); err != nil {
|
||||
unknown = "mesh-config's resolved values cannot be listed: " + err.Error()
|
||||
} else {
|
||||
var want []string
|
||||
for _, k := range keys {
|
||||
if strings.HasSuffix(k, "."+buildstep.CapacitySetting) && strings.Contains(k, ".mesh-build-") {
|
||||
want = append(want, k)
|
||||
}
|
||||
}
|
||||
raws, err := u.Resolved.values(want)
|
||||
if err != nil {
|
||||
unknown = "mesh-config's resolved capacities cannot be read: " + err.Error()
|
||||
}
|
||||
for i, k := range want {
|
||||
if err != nil {
|
||||
break // the read failed: the values are unknown to this tool, not unreadable in mesh-config
|
||||
}
|
||||
var r struct {
|
||||
Value json.RawMessage `json:"value"`
|
||||
Source string `json:"source"`
|
||||
}
|
||||
key := strings.TrimSuffix(k, "."+buildstep.CapacitySetting)
|
||||
setting := ""
|
||||
switch {
|
||||
case i >= len(raws) || len(raws[i]) == 0:
|
||||
unreadable[key] = "its resolved key holds no value"
|
||||
case json.Unmarshal(raws[i], &r) != nil:
|
||||
unreadable[key] = "its resolved key is not a resolved value"
|
||||
case r.Source == "unreadable":
|
||||
unreadable[key] = "mesh-config resolves it as unreadable"
|
||||
case len(r.Value) == 0 || string(r.Value) == "null" || json.Unmarshal(r.Value, &setting) != nil || strings.TrimSpace(setting) == "":
|
||||
unreadable[key] = fmt.Sprintf("its resolved value %s is empty or not a capacity", strings.TrimSpace(string(r.Value)))
|
||||
default:
|
||||
resolved[key] = setting
|
||||
}
|
||||
}
|
||||
}
|
||||
if unknown != "" {
|
||||
note("capacity unknown to this tool: %s; eligibility is not answered for a seat that holds its module", unknown)
|
||||
}
|
||||
for key := range unreadable {
|
||||
node, module, _ := strings.Cut(key, ".")
|
||||
seat(node, strings.TrimPrefix(module, "mesh-build-"))
|
||||
}
|
||||
// Every node a capacity names is listed, so "is this node eligible" is answered for any node: the nodes each module's
|
||||
// resolved value names (as resolved on the first node in name order), or the modules' default when mesh-config
|
||||
// resolves none for the module.
|
||||
keys := make([]string, 0, len(resolved))
|
||||
for key := range resolved {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
named := map[string]string{} // language -> the module's setting, as resolved on some node
|
||||
namedOn := map[string]string{} // language -> that node
|
||||
for _, key := range keys {
|
||||
node, module, _ := strings.Cut(key, ".")
|
||||
lang := strings.TrimPrefix(module, "mesh-build-")
|
||||
seat(node, lang)
|
||||
if _, ok := named[lang]; !ok {
|
||||
named[lang], namedOn[lang] = resolved[key], node
|
||||
}
|
||||
}
|
||||
for _, lang := range delivery.AskLanguages {
|
||||
setting, ok := named[lang]
|
||||
if !ok {
|
||||
setting = buildstep.DefaultCapacities
|
||||
}
|
||||
if caps, err := buildrun.ParseCapacity(setting); err == nil {
|
||||
for node := range caps {
|
||||
seat(node, lang)
|
||||
}
|
||||
}
|
||||
}
|
||||
// The steps of each build module.
|
||||
steps := map[string]map[string][]usageStep{} // node -> language -> steps
|
||||
for _, lang := range delivery.AskLanguages {
|
||||
module, _ := delivery.BuildModuleOf(lang)
|
||||
sr, ok := u.Steps[lang]
|
||||
if !ok || sr.Keys == nil || sr.Get == nil {
|
||||
note("the steps of %s are not readable here", module)
|
||||
continue
|
||||
}
|
||||
keys, err := sr.Keys()
|
||||
if err != nil {
|
||||
note("the steps of %s cannot be listed: %v", module, err)
|
||||
continue
|
||||
}
|
||||
// A step id is its start in base-36 milliseconds, so the newest keys are the ones of the last day; at most
|
||||
// MaxStateReads of them are read, newest first.
|
||||
type keyAt struct {
|
||||
key string
|
||||
at time.Time
|
||||
}
|
||||
var recent []keyAt
|
||||
for _, k := range keys {
|
||||
_, id, err := delivery.ParseStepKey(k)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if at, ok := stepIDTime(id); ok && now.Sub(at) <= 24*time.Hour+delivery.StepBound {
|
||||
recent = append(recent, keyAt{k, at})
|
||||
}
|
||||
}
|
||||
sort.Slice(recent, func(i, j int) bool { return recent[i].at.After(recent[j].at) })
|
||||
if len(recent) > MaxStateReads {
|
||||
note("the steps of %s: read the newest %d of %d keys of the last day; the older ones are not counted", module, MaxStateReads, len(recent))
|
||||
recent = recent[:MaxStateReads]
|
||||
}
|
||||
want := make([]string, len(recent))
|
||||
for i, r := range recent {
|
||||
want[i] = r.key
|
||||
}
|
||||
raws, err := sr.values(want)
|
||||
if err != nil {
|
||||
note("the steps of %s cannot all be read: %v", module, err)
|
||||
}
|
||||
for i := range want {
|
||||
if i >= len(raws) || len(raws[i]) == 0 {
|
||||
continue
|
||||
}
|
||||
s, err := delivery.DecodeStep(raws[i])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if steps[s.Node] == nil {
|
||||
steps[s.Node] = map[string][]usageStep{}
|
||||
}
|
||||
steps[s.Node][lang] = append(steps[s.Node][lang], usageStep{s, stepKind(s, tries)})
|
||||
seat(s.Node, lang)
|
||||
}
|
||||
}
|
||||
|
||||
// Pauses: paused.<node> of each module.
|
||||
paused := map[string]bool{} // <node>.<language>
|
||||
for _, lang := range delivery.AskLanguages {
|
||||
module, _ := delivery.BuildModuleOf(lang)
|
||||
pr, ok := u.Pauses[lang]
|
||||
if !ok || pr.Get == nil {
|
||||
note("the pauses of %s are not readable here, so none is said paused", module)
|
||||
continue
|
||||
}
|
||||
for node := range seats {
|
||||
if seats[node][lang] == nil {
|
||||
continue
|
||||
}
|
||||
key, err := delivery.PauseKey(node)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
raw, err := pr.Get(key)
|
||||
if err != nil {
|
||||
note("the pause of %s on %s cannot be read: %v", module, node, err)
|
||||
continue
|
||||
}
|
||||
if len(raw) > 0 {
|
||||
if p, err := delivery.DecodePause(raw); err == nil {
|
||||
paused[node+"."+lang] = p.Paused
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
nodes := make([]string, 0, len(seats))
|
||||
for n := range seats {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
for _, node := range nodes {
|
||||
nu := NodeUsage{Node: node, Seats: []SeatUsage{}, Said: []string{},
|
||||
CPU: "not on the bus: the disk-load module keeps no state; " + node + "/disk-load.cpu_top reads it (the build modules' share among its units)"}
|
||||
for _, lang := range delivery.AskLanguages {
|
||||
s := seats[node][lang]
|
||||
if s == nil {
|
||||
continue
|
||||
}
|
||||
setting, ok := resolved[node+"."+s.Module]
|
||||
from := "mesh-config, resolved for " + s.Module + " on " + node
|
||||
if _, has := named[lang]; !ok && has {
|
||||
from = "mesh-config, resolved for " + s.Module + " on " + namedOn[lang] + " (" + s.Module + " is not assigned to " + node + ")"
|
||||
// mesh-config resolves the module's value only where it is assigned; elsewhere its capacity is what that
|
||||
// value names for the node.
|
||||
setting, ok = named[lang], true
|
||||
}
|
||||
bad, isBad := unreadable[node+"."+s.Module]
|
||||
if unknown != "" {
|
||||
s.CapacityFrom = "capacity unknown to this tool: " + unknown
|
||||
} else if isBad {
|
||||
s.CapacityFrom = "mesh-config's value for " + s.Module + " on " + node + " cannot be read (" + bad + "), so its puller uses 0"
|
||||
} else if ok {
|
||||
s.CapacityFrom = from
|
||||
if caps, err := buildrun.ParseCapacity(setting); err != nil {
|
||||
s.CapacityFrom = fmt.Sprintf("mesh-config's value %q, which cannot be read, so the module pulls nothing: %v", setting, err)
|
||||
} else {
|
||||
s.Capacity = (&buildrun.Places{Capacity: caps}).CapacityOf(node)
|
||||
}
|
||||
} else {
|
||||
caps, _ := buildrun.ParseCapacity(buildstep.DefaultCapacities)
|
||||
s.Capacity = (&buildrun.Places{Capacity: caps}).CapacityOf(node)
|
||||
s.CapacityFrom = fmt.Sprintf("the modules' default %q: mesh-config holds no resolved value for %s on %s", buildstep.DefaultCapacities, s.Module, node)
|
||||
}
|
||||
s.Paused = paused[node+"."+lang]
|
||||
s.EligibleKnown = true
|
||||
switch {
|
||||
case !s.Holds && s.Phase == "":
|
||||
s.WhyNot = s.Module + " is not on " + node
|
||||
case !s.Holds:
|
||||
s.WhyNot = s.Module + " on " + node + " is " + s.Phase
|
||||
case s.Paused:
|
||||
s.WhyNot = "paused (" + s.Seat + ".resume takes asks again)"
|
||||
case unknown != "":
|
||||
s.EligibleKnown = false
|
||||
s.WhyNot = "not known: this tool cannot read its capacity (" + unknown + ")"
|
||||
case isBad:
|
||||
s.WhyNot = "its capacity on " + node + " cannot be read, so its puller uses 0"
|
||||
case s.Capacity == 0:
|
||||
s.WhyNot = "its capacity on " + node + " is 0"
|
||||
}
|
||||
if s.EligibleKnown {
|
||||
e := s.WhyNot == ""
|
||||
s.Eligible = &e
|
||||
}
|
||||
switch {
|
||||
case s.Eligible != nil && *s.Eligible:
|
||||
nu.Eligible = "eligible"
|
||||
case !s.EligibleKnown && nu.Eligible != "eligible":
|
||||
nu.Eligible = "not known"
|
||||
case nu.Eligible == "":
|
||||
nu.Eligible = "not eligible"
|
||||
}
|
||||
s.fill(steps[node][lang], now)
|
||||
nu.Said = append(nu.Said, s.said())
|
||||
nu.Seats = append(nu.Seats, *s)
|
||||
}
|
||||
ans.Nodes = append(ans.Nodes, nu)
|
||||
}
|
||||
|
||||
if u.Backlog == nil {
|
||||
note("the backlog of the seats' streams is not readable here")
|
||||
} else if rows, err := u.Backlog(); err != nil {
|
||||
note("the backlog of the seats' streams is read by the view program, which did not answer: %v", err)
|
||||
} else if rows != nil {
|
||||
ans.Backlog = rows
|
||||
}
|
||||
return ans
|
||||
}
|
||||
|
||||
// tries reads the checker's tries of the last day: the running ones, and the asks every one put.
|
||||
func (u UsageReader) tries(now time.Time, ans *UsageAnswer, note func(string, ...any)) map[string]bool {
|
||||
asks := map[string]bool{}
|
||||
if u.Tries.Keys == nil || u.Tries.Get == nil {
|
||||
note("the checker's tries are not readable here, so a try's asks are counted under check")
|
||||
return asks
|
||||
}
|
||||
keys, err := u.Tries.Keys()
|
||||
if err != nil {
|
||||
note("the checker's tries cannot be listed, so a try's asks are counted under check: %v", err)
|
||||
return asks
|
||||
}
|
||||
var want []string
|
||||
for _, k := range keys {
|
||||
ns, err := strconv.ParseInt(strings.TrimPrefix(k, "try-"), 10, 64)
|
||||
if err == nil && strings.HasPrefix(k, "try-") && now.Sub(time.Unix(0, ns)) <= 48*time.Hour {
|
||||
want = append(want, k)
|
||||
}
|
||||
}
|
||||
raws, err := u.Tries.values(want)
|
||||
if err != nil {
|
||||
note("the checker's tries cannot all be read: %v", err)
|
||||
}
|
||||
for i := range want {
|
||||
if i >= len(raws) || len(raws[i]) == 0 {
|
||||
continue
|
||||
}
|
||||
var t struct {
|
||||
ID string `json:"id"`
|
||||
Repository string `json:"repository"`
|
||||
State string `json:"state"`
|
||||
Started time.Time `json:"started"`
|
||||
Asks []string `json:"asks"`
|
||||
}
|
||||
if json.Unmarshal(raws[i], &t) != nil {
|
||||
continue
|
||||
}
|
||||
for _, a := range t.Asks {
|
||||
asks[a] = true
|
||||
}
|
||||
if t.State == "running" {
|
||||
ans.TriesRunning = append(ans.TriesRunning, TryRunning{ID: t.ID, Repository: t.Repository, Started: t.Started})
|
||||
}
|
||||
}
|
||||
if len(ans.TriesRunning) > 0 {
|
||||
note("a running try's asks are counted under check until the try ends and names them")
|
||||
}
|
||||
return asks
|
||||
}
|
||||
|
||||
// fill counts a seat's steps: running now, left running (started past delivery.StepBound, or its module no longer running on
|
||||
// the node: no step behind the key runs any more), and the two windows.
|
||||
func (s *SeatUsage) fill(steps []usageStep, now time.Time) {
|
||||
sort.Slice(steps, func(i, j int) bool { return steps[i].step.Started.Before(steps[j].step.Started) })
|
||||
var live []usageStep // ended, or running and not left: what the windows count
|
||||
for _, us := range steps {
|
||||
st := us.step
|
||||
if why := endedStale(st); why != "" {
|
||||
s.EndedStale = append(s.EndedStale, StepUsage{ID: st.ID, Kind: us.kind, Tool: st.Tool, Started: st.Started, Why: why})
|
||||
continue
|
||||
}
|
||||
if st.State != delivery.StepRunning {
|
||||
live = append(live, us)
|
||||
continue
|
||||
}
|
||||
line := StepUsage{ID: st.ID, Kind: us.kind, Tool: st.Tool, Started: st.Started, Secs: int64(now.Sub(st.Started).Seconds())}
|
||||
switch {
|
||||
case !s.Holds:
|
||||
line.Why = "its key says running, and " + s.Module + " is not running on its node, so no step behind it runs"
|
||||
case now.Sub(st.Started) > delivery.StepBound:
|
||||
line.Why = fmt.Sprintf("its key says running since %s, past the step bound of %s: its module stopped or restarted before it wrote the end, and no asker waits for it", st.Started.Format(time.RFC3339), delivery.StepBound)
|
||||
}
|
||||
if line.Why != "" {
|
||||
s.LeftRunning = append(s.LeftRunning, line)
|
||||
continue
|
||||
}
|
||||
s.Steps = append(s.Steps, line)
|
||||
live = append(live, us)
|
||||
if us.kind != KindHand {
|
||||
s.Running.Total++
|
||||
s.Running.ByKind[us.kind]++
|
||||
}
|
||||
}
|
||||
s.LastHour = window(live, now, time.Hour, s.Capacity)
|
||||
s.LastDay = window(live, now, 24*time.Hour, s.Capacity)
|
||||
}
|
||||
|
||||
// said is the seat's use in one line.
|
||||
func (s *SeatUsage) said() string {
|
||||
head := s.Seat + ": eligible"
|
||||
switch {
|
||||
case !s.EligibleKnown:
|
||||
head = s.Seat + ": eligibility " + s.WhyNot
|
||||
case s.Eligible == nil || !*s.Eligible:
|
||||
head = s.Seat + ": not eligible, " + s.WhyNot
|
||||
}
|
||||
kinds := make([]string, 0, len(s.Running.ByKind))
|
||||
for k, n := range s.Running.ByKind {
|
||||
kinds = append(kinds, fmt.Sprintf("%s %d", k, n))
|
||||
}
|
||||
sort.Strings(kinds)
|
||||
line := fmt.Sprintf("%s; %d of capacity %d running", head, s.Running.Total, s.Capacity)
|
||||
if len(kinds) > 0 {
|
||||
line += " (" + strings.Join(kinds, ", ") + ")"
|
||||
}
|
||||
if len(s.LeftRunning) > 0 {
|
||||
line += fmt.Sprintf(", %d left running", len(s.LeftRunning))
|
||||
}
|
||||
return line + fmt.Sprintf("; last hour %d finished, median %d s, at capacity %.0f%% of it; last day %d finished, median %d s, at capacity %.1f%% of it",
|
||||
s.LastHour.Finished, s.LastHour.MedianSecs, s.LastHour.AtCapacityShare*100, s.LastDay.Finished, s.LastDay.MedianSecs, s.LastDay.AtCapacityShare*100)
|
||||
}
|
||||
|
||||
// window counts the steps of the last span before now: started in it, finished in it with their total and median time,
|
||||
// and the time in it during which capacity asks (not hand steps) ran at once.
|
||||
func window(steps []usageStep, now time.Time, span time.Duration, capacity int) WindowUsage {
|
||||
from := now.Add(-span)
|
||||
w := WindowUsage{ByKind: map[string]KindWindow{}}
|
||||
var all []int64
|
||||
byKind := map[string][]int64{}
|
||||
type edge struct {
|
||||
at time.Time
|
||||
d int
|
||||
}
|
||||
var edges []edge
|
||||
for _, us := range steps {
|
||||
st := us.step
|
||||
if !st.Started.Before(from) {
|
||||
w.Started++
|
||||
}
|
||||
if !st.Ended.IsZero() && !st.Ended.Before(from) {
|
||||
w.Finished++
|
||||
if st.State == delivery.StepFailed || st.State == delivery.StepRefused {
|
||||
w.Failed++
|
||||
}
|
||||
secs := int64(st.Ended.Sub(st.Started).Seconds())
|
||||
all = append(all, secs)
|
||||
byKind[us.kind] = append(byKind[us.kind], secs)
|
||||
}
|
||||
if us.kind == KindHand {
|
||||
continue
|
||||
}
|
||||
start, end := st.Started, st.Ended
|
||||
if end.IsZero() {
|
||||
end = now
|
||||
}
|
||||
if start.Before(from) {
|
||||
start = from
|
||||
}
|
||||
if end.After(start) {
|
||||
edges = append(edges, edge{start, 1}, edge{end, -1})
|
||||
}
|
||||
}
|
||||
w.TotalSecs, w.MedianSecs = sumMedian(all)
|
||||
for k, d := range byKind {
|
||||
kw := KindWindow{Finished: len(d)}
|
||||
kw.TotalSecs, kw.MedianSecs = sumMedian(d)
|
||||
w.ByKind[k] = kw
|
||||
}
|
||||
if capacity > 0 {
|
||||
// An end before a start at the same instant: a slot handed from one ask to the next is not a moment at capacity+1.
|
||||
sort.Slice(edges, func(i, j int) bool {
|
||||
if edges[i].at.Equal(edges[j].at) {
|
||||
return edges[i].d < edges[j].d
|
||||
}
|
||||
return edges[i].at.Before(edges[j].at)
|
||||
})
|
||||
var at time.Duration
|
||||
n := 0
|
||||
for i, e := range edges {
|
||||
n += e.d
|
||||
if n >= capacity && i+1 < len(edges) {
|
||||
at += edges[i+1].at.Sub(e.at)
|
||||
}
|
||||
}
|
||||
w.AtCapacitySecs = int64(at.Seconds())
|
||||
w.AtCapacityShare = float64(int64(at*1000/span)) / 1000
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// sumMedian is the sum and the median of d (the lower middle of an even count); zero for none.
|
||||
func sumMedian(d []int64) (sum, median int64) {
|
||||
if len(d) == 0 {
|
||||
return 0, 0
|
||||
}
|
||||
s := append([]int64(nil), d...)
|
||||
sort.Slice(s, func(i, j int) bool { return s[i] < s[j] })
|
||||
for _, v := range s {
|
||||
sum += v
|
||||
}
|
||||
return sum, s[(len(s)-1)/2]
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
package view
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// usageNow is the fixture's clock.
|
||||
var usageNow = time.Date(2026, 10, 26, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
// at is the fixture's time hh:mm on its day.
|
||||
func at(hm string) time.Time {
|
||||
t, _ := time.Parse("2006-01-02 15:04", "2026-10-26 "+hm)
|
||||
return t.UTC()
|
||||
}
|
||||
|
||||
// stepID is a step id made at t, as buildstep.NewID makes it, with suffix as its six hex digits.
|
||||
func stepID(t time.Time, suffix string) string { return strconv.FormatInt(t.UnixMilli(), 36) + suffix }
|
||||
|
||||
func states(m map[string]string) StateRead {
|
||||
return StateRead{
|
||||
Keys: func() ([]string, error) {
|
||||
var k []string
|
||||
for key := range m {
|
||||
k = append(k, key)
|
||||
}
|
||||
return k, nil
|
||||
},
|
||||
Get: func(key string) (json.RawMessage, error) {
|
||||
if v, ok := m[key]; ok {
|
||||
return json.RawMessage(v), nil
|
||||
}
|
||||
return nil, nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func step(node, id, tool, state string, started, ended time.Time) string {
|
||||
e := ""
|
||||
if !ended.IsZero() {
|
||||
e = `,"ended":"` + ended.Format(time.RFC3339) + `"`
|
||||
}
|
||||
return fmt.Sprintf(`{"v":1,"id":%q,"tool":%q,"node":%q,"state":%q,"started":%q%s}`, id, tool, node, state, started.Format(time.RFC3339), e)
|
||||
}
|
||||
|
||||
// usageFixture is four nodes: shanks builds Go at capacity 2 (two asks running, one left running, a try's ask, a check,
|
||||
// a recheck of the morning and a hand step ended); ace holds the Go seat at capacity 2 and is paused; g14 holds it at
|
||||
// capacity 0; the Node seat on shanks has no resolved capacity and takes the modules' default.
|
||||
func usageFixture() UsageReader {
|
||||
running := `{"v":1,"phase":"running","health":"ok"}`
|
||||
tryAsk := stepID(at("11:30"), "a00003")
|
||||
goSteps := map[string]string{}
|
||||
add := func(node, id, tool, state string, started, ended time.Time) {
|
||||
goSteps[node+"."+id] = step(node, id, tool, state, started, ended)
|
||||
}
|
||||
add("shanks", stepID(at("11:50"), "a00001"), "delivery ask", "running", at("11:50"), time.Time{})
|
||||
add("shanks", stepID(at("11:55"), "a00002"), "check ask", "running", at("11:55"), time.Time{})
|
||||
add("shanks", tryAsk, "check ask", "passed", at("11:30"), at("11:40"))
|
||||
add("shanks", stepID(at("11:20"), "a00004"), "check ask", "failed", at("11:20"), at("11:26"))
|
||||
add("shanks", stepID(at("06:00"), "a00005"), "recheck ask", "passed", at("06:00"), at("06:10"))
|
||||
add("shanks", stepID(at("10:00"), "a00006"), "check ask", "running", at("10:00"), time.Time{})
|
||||
add("shanks", stepID(at("11:45"), "a00007"), "build_go_test", "passed", at("11:45"), at("11:46"))
|
||||
// Two keys a build module ended for being left running: their end is when it noticed, so neither is finished work.
|
||||
goSteps["shanks."+stepID(at("08:00"), "a00008")] = `{"v":1,"id":"` + stepID(at("08:00"), "a00008") + `","tool":"check ask","node":"shanks","state":"failed","started":"2026-10-26T08:00:00Z","ended":"2026-10-26T11:59:00Z","reason":"the build module restarted while it ran"}`
|
||||
goSteps["shanks."+stepID(at("09:00"), "a00009")] = `{"v":1,"id":"` + stepID(at("09:00"), "a00009") + `","tool":"delivery ask","node":"shanks","state":"failed","started":"2026-10-26T09:00:00Z","ended":"2026-10-26T11:58:00Z","reason":"past the step bound: its asker stopped waiting at 2026-10-26T09:30:00Z"}`
|
||||
goSteps["shanks.notastepkey"] = `{}` // a key that is no step id is passed over
|
||||
return UsageReader{
|
||||
Now: func() time.Time { return usageNow },
|
||||
Haves: states(map[string]string{"shanks.mesh-build-go": running, "shanks.mesh-build-node": running, "ace.mesh-build-go": running,
|
||||
"g14.mesh-build-go": running, "g14.mesh-build-node": running, "shanks.node": `{}`, "shanks.mesh-delivery-view": running}),
|
||||
Resolved: states(map[string]string{
|
||||
"shanks.mesh-build-go.capacity": `{"node":"shanks","module":"mesh-build-go","key":"capacity","value":"shanks=2,ace=2,g14=0,novox=0","source":"mesh"}`,
|
||||
"ace.mesh-build-go.capacity": `{"node":"ace","module":"mesh-build-go","key":"capacity","value":"shanks=2,ace=2,g14=0,novox=0","source":"mesh"}`,
|
||||
"g14.mesh-build-go.capacity": `{"node":"g14","module":"mesh-build-go","key":"capacity","value":"shanks=2,ace=2,g14=0,novox=0","source":"mesh"}`,
|
||||
"shanks.mesh-config.other": `{"value":"x"}`,
|
||||
"g14.mesh-build-node.capacity": `{"node":"g14","module":"mesh-build-node","key":"capacity","source":"unreadable"}`,
|
||||
}),
|
||||
Steps: map[string]StateRead{"go": states(goSteps), "node": states(map[string]string{}), "image": states(map[string]string{})},
|
||||
Pauses: map[string]StateRead{"go": states(map[string]string{"paused.ace": `{"v":1,"paused":true,"by":"lead@shanks","at":"2026-10-26T09:00:00Z"}`}), "node": states(nil), "image": states(nil)},
|
||||
Tries: states(map[string]string{
|
||||
"try-" + strconv.FormatInt(at("11:29").UnixNano(), 10): `{"id":"try-1","repository":"novox/mesh-sdk","head":"h","trunk":"t","state":"pass","asks":["` + tryAsk + `"]}`,
|
||||
"try-" + strconv.FormatInt(at("11:58").UnixNano(), 10): `{"id":"try-2","repository":"novox/mesh-delivery","head":"h","trunk":"t","state":"running","started":"2026-10-26T11:58:00Z"}`,
|
||||
}),
|
||||
Backlog: func() ([]BacklogRow, error) {
|
||||
return []BacklogRow{{Language: "go", Kind: "check", Waiting: 3, Held: 2, OldestSecs: 240}}, nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func seatOf(t *testing.T, a UsageAnswer, node, seat string) SeatUsage {
|
||||
t.Helper()
|
||||
for _, n := range a.Nodes {
|
||||
for _, s := range n.Seats {
|
||||
if n.Node == node && s.Seat == seat {
|
||||
return s
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Fatalf("no seat %s on %s in %+v", seat, node, a.Nodes)
|
||||
return SeatUsage{}
|
||||
}
|
||||
|
||||
// build_usage answers, from the states alone, what the operator asked in six calls: per node its seats, capacity,
|
||||
// eligibility, the asks running by kind and the ones left running, the last hour and day, and mesh-wide the backlog.
|
||||
func TestBuildUsageAnswersEveryNodesUseFromTheStates(t *testing.T) {
|
||||
a := usageFixture().Usage()
|
||||
if testing.Verbose() {
|
||||
out, _ := json.MarshalIndent(a, "", " ")
|
||||
t.Logf("build_usage answers:\n%s", out)
|
||||
}
|
||||
sg := seatOf(t, a, "shanks", "node-build-go")
|
||||
eligible := map[string]string{}
|
||||
for _, n := range a.Nodes {
|
||||
eligible[n.Node] = n.Eligible
|
||||
}
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
got, want any
|
||||
}{
|
||||
{"nodes, novox named by the default capacity", len(a.Nodes), 4},
|
||||
{"eligible", eligible, map[string]string{"shanks": "eligible", "ace": "not eligible", "g14": "not eligible", "novox": "not eligible"}},
|
||||
{"a resolved key with no readable value: the puller uses 0", seatOf(t, a, "g14", "node-build-node").WhyNot, "its capacity on g14 cannot be read, so its puller uses 0"},
|
||||
{"and is said so", seatOf(t, a, "g14", "node-build-node").CapacityFrom, "mesh-config's value for mesh-build-node on g14 cannot be read (mesh-config resolves it as unreadable), so its puller uses 0"},
|
||||
{"novox why not", seatOf(t, a, "novox", "node-build-go").WhyNot, "mesh-build-go is not on novox"},
|
||||
{"novox capacity, from the module's value resolved elsewhere", seatOf(t, a, "novox", "node-build-go").CapacityFrom, "mesh-config, resolved for mesh-build-go on ace (mesh-build-go is not assigned to novox)"},
|
||||
{"ended by restart or bound, apart", len(sg.EndedStale), 2},
|
||||
{"ace why not", seatOf(t, a, "ace", "node-build-go").WhyNot, "paused (node-build-go.resume takes asks again)"},
|
||||
{"g14 why not", seatOf(t, a, "g14", "node-build-go").WhyNot, "its capacity on g14 is 0"},
|
||||
{"shanks capacity", sg.Capacity, 2},
|
||||
{"shanks node seat takes the default", seatOf(t, a, "shanks", "node-build-node").Capacity, 3},
|
||||
{"running", sg.Running.Total, 2},
|
||||
{"running by kind", sg.Running.ByKind, map[string]int{"delivery": 1, "check": 1}},
|
||||
{"running steps, the hand step's ended", len(sg.Steps), 2},
|
||||
{"left running", len(sg.LeftRunning), 1},
|
||||
{"hour started", sg.LastHour.Started, 5}, // delivery, check, try, check, hand
|
||||
{"hour finished", sg.LastHour.Finished, 3}, // try 600 s, check 360 s, hand 60 s
|
||||
{"hour failed", sg.LastHour.Failed, 1},
|
||||
{"hour total", sg.LastHour.TotalSecs, int64(1020)},
|
||||
{"hour median", sg.LastHour.MedianSecs, int64(360)},
|
||||
{"hour try", sg.LastHour.ByKind["try"], KindWindow{Finished: 1, TotalSecs: 600, MedianSecs: 600}},
|
||||
{"hour hand", sg.LastHour.ByKind["hand"], KindWindow{Finished: 1, TotalSecs: 60, MedianSecs: 60}},
|
||||
{"hour at capacity", sg.LastHour.AtCapacitySecs, int64(300)}, // 11:55-12:00, two asks of two
|
||||
{"hour at capacity share", sg.LastHour.AtCapacityShare, 0.083},
|
||||
{"day finished", sg.LastDay.Finished, 4},
|
||||
{"day recheck", sg.LastDay.ByKind["recheck"].Finished, 1},
|
||||
{"day at capacity", sg.LastDay.AtCapacitySecs, int64(300)},
|
||||
{"backlog", a.Backlog, []BacklogRow{{Language: "go", Kind: "check", Waiting: 3, Held: 2, OldestSecs: 240}}},
|
||||
{"tries running", len(a.TriesRunning), 1},
|
||||
{"said", a.Nodes[0].Said[:1], []string{"node-build-go: not eligible, paused (node-build-go.resume takes asks again); 0 of capacity 2 running; last hour 0 finished, median 0 s, at capacity 0% of it; last day 0 finished, median 0 s, at capacity 0.0% of it"}},
|
||||
{"said shanks", a.Nodes[3].Said[0], "node-build-go: eligible; 2 of capacity 2 running (check 1, delivery 1), 1 left running; last hour 3 finished, median 360 s, at capacity 8% of it; last day 4 finished, median 360 s, at capacity 0.3% of it"},
|
||||
{"cpu says where it is", strings.Contains(a.Nodes[0].CPU, "/disk-load.cpu_top"), true},
|
||||
} {
|
||||
if fmt.Sprint(c.got) != fmt.Sprint(c.want) {
|
||||
t.Errorf("%s = %v, want %v", c.what, c.got, c.want)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(sg.LeftRunning[0].Why, "past the step bound") {
|
||||
t.Errorf("left running says %q", sg.LeftRunning[0].Why)
|
||||
}
|
||||
|
||||
// Nothing readable: each state is named in a note, the backlog's refusal too, and nothing is said held.
|
||||
none := UsageReader{Now: func() time.Time { return usageNow }, Backlog: func() ([]BacklogRow, error) { return nil, errors.New("no socket") }}.Usage()
|
||||
notes := strings.Join(none.Notes, " | ")
|
||||
for _, want := range []string{"have is not readable", "capacity unknown to this tool: mesh-config's resolved values are not readable", "steps of mesh-build-go are not readable", "tries are not readable", "did not answer: no socket"} {
|
||||
if !strings.Contains(notes, want) {
|
||||
t.Errorf("the notes %q do not say %q", notes, want)
|
||||
}
|
||||
}
|
||||
if none.Backlog == nil {
|
||||
t.Errorf("nothing readable answered no backlog list")
|
||||
}
|
||||
|
||||
// mesh-config unreadable to this tool: a seat that holds its module is not answered (never said not eligible), and its
|
||||
// node's verdict is not known; a seat known not eligible on other grounds (paused) stays so.
|
||||
u := usageFixture()
|
||||
u.Resolved = StateRead{}
|
||||
blind := u.Usage()
|
||||
sb := seatOf(t, blind, "shanks", "node-build-go")
|
||||
verdict := map[string]string{}
|
||||
for _, n := range blind.Nodes {
|
||||
verdict[n.Node] = n.Eligible
|
||||
}
|
||||
if sb.EligibleKnown || sb.Eligible != nil || !strings.HasPrefix(sb.WhyNot, "not known: ") || !strings.HasPrefix(sb.CapacityFrom, "capacity unknown to this tool: ") {
|
||||
t.Errorf("unknown capacity: known %v eligible %v why %q from %q", sb.EligibleKnown, sb.Eligible, sb.WhyNot, sb.CapacityFrom)
|
||||
}
|
||||
if verdict["shanks"] != "not known" || verdict["ace"] != "not eligible" {
|
||||
t.Errorf("verdicts with mesh-config unreadable = %v, want shanks not known, ace not eligible (paused)", verdict)
|
||||
}
|
||||
}
|
||||
@@ -84,7 +84,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7
|
||||
if m.Module != "mesh-delivery-view" {
|
||||
t.Errorf("module = %q", m.Module)
|
||||
}
|
||||
want := []string{"mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-assigner.wants", "mesh-delivery-builder.builds", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "node-engine.have"} // 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9
|
||||
want := []string{"mesh-build-go.pauses", "mesh-build-go.steps", "mesh-build-image.pauses", "mesh-build-image.steps", "mesh-build-node.pauses", "mesh-build-node.steps", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-assigner.wants", "mesh-delivery-builder.builds", "mesh-delivery-checker.tries", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.modules", "mesh-delivery-indexer.placement", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "node-engine.have"} // build_usage reads the build modules' steps and pauses, mesh-config's capacities and the checker's tries; 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9
|
||||
got := slices.Clone(m.Reads)
|
||||
slices.Sort(got)
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
@@ -115,7 +115,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7
|
||||
}
|
||||
// The two read tools of the indexer's modules (issue 1562), carried by a bundle the runtime launches; the indexer
|
||||
// itself serves no tool (ADR 0437 §4). Carrying them gives the tool runner read access to every bucket the view reads.
|
||||
if want := []string{"indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "builder_build", "builder_builds", "assigner_wants", "assigner_want", "assigner_differences", "node_haves"}; !reflect.DeepEqual(m.Tools, want) {
|
||||
if want := []string{"indexer_module", "indexer_modules", "indexer_placement", "indexer_candidate", "indexer_candidates", "builder_build", "builder_builds", "assigner_wants", "assigner_want", "assigner_differences", "node_haves", "build_usage"}; !reflect.DeepEqual(m.Tools, want) {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
"tools": [
|
||||
"indexer_module",
|
||||
"indexer_modules",
|
||||
"indexer_placement",
|
||||
"indexer_candidate",
|
||||
"indexer_candidates",
|
||||
"builder_build",
|
||||
@@ -13,7 +14,8 @@
|
||||
"assigner_wants",
|
||||
"assigner_want",
|
||||
"assigner_differences",
|
||||
"node_haves"
|
||||
"node_haves",
|
||||
"build_usage"
|
||||
],
|
||||
"emits": [
|
||||
"delivered",
|
||||
@@ -27,15 +29,24 @@
|
||||
"reads": [
|
||||
"mesh-controller.assignments",
|
||||
"mesh-controller.builds",
|
||||
"mesh-build-go.pauses",
|
||||
"mesh-build-go.steps",
|
||||
"mesh-build-image.pauses",
|
||||
"mesh-build-image.steps",
|
||||
"mesh-build-node.pauses",
|
||||
"mesh-build-node.steps",
|
||||
"mesh-config.resolved",
|
||||
"mesh-controller.composed",
|
||||
"mesh-delivery-assigner.asks",
|
||||
"mesh-delivery-assigner.differences",
|
||||
"mesh-delivery-assigner.wants",
|
||||
"mesh-delivery-builder.builds",
|
||||
"mesh-delivery-checker.tries",
|
||||
"mesh-delivery-gate.asks",
|
||||
"mesh-delivery-gate.source",
|
||||
"mesh-delivery-indexer.candidates",
|
||||
"mesh-delivery-indexer.modules",
|
||||
"mesh-delivery-indexer.placement",
|
||||
"mesh-delivery-teller.answers",
|
||||
"mesh-delivery-watchman.presence",
|
||||
"mesh-delivery-watchman.stuck",
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
)
|
||||
|
||||
// The three bus adapters for mesh-sdk go/asker on the part's own credential, as mesh-delivery-teller has them (copied: the
|
||||
// SDK's asker reaches the bus through a bundle's runtime; moving the adapters into the SDK is a change of its own). What
|
||||
// the account may do is the controller's composed grant for a module that uses the seat operator-channel: publish its own
|
||||
// ask and condition, hear its own decided event, and read the router's record of its own asks.
|
||||
|
||||
const recordWithin = 30 * time.Second
|
||||
|
||||
func publishVia(ctx context.Context, js jetstream.JetStream) func(subject string, body any, id string) error {
|
||||
return func(subject string, body any, id string) error {
|
||||
data, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var opts []jetstream.PublishOpt
|
||||
if id != "" {
|
||||
opts = append(opts, jetstream.WithMsgID(id))
|
||||
}
|
||||
_, err = js.Publish(ctx, subject, data, opts...)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
func recordVia(ctx context.Context, conn *nats.Conn, module string) func(id string) (*stdio.StateEntry, error) {
|
||||
return func(id string) (*stdio.StateEntry, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, recordWithin)
|
||||
defer cancel()
|
||||
msg, err := conn.RequestWithContext(ctx, "$JS.API.DIRECT.GET.KV_asks.$KV.asks."+module+"."+id, nil)
|
||||
// No responders is the bucket out of reach for a moment, never "no record": read as none, a live ask would be
|
||||
// closed by Reconcile a minute later and the operator's yes lost.
|
||||
if errors.Is(err, nats.ErrNoResponders) {
|
||||
return nil, fmt.Errorf("the router's record of %s cannot be reached: %w", id, err)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if status := msg.Header.Get("Status"); status == "404" || len(msg.Data) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return &stdio.StateEntry{Key: id, Value: msg.Data}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func decidedVia(conn *nats.Conn, module string, logf func(string, ...any)) func(pattern string, handle func(stdio.Envelope) error) error {
|
||||
return func(pattern string, handle func(stdio.Envelope) error) error {
|
||||
_, err := conn.Subscribe(asks.DecidedSubject(module), func(m *nats.Msg) {
|
||||
if err := handle(stdio.Envelope{Key: pattern, Body: m.Data}); err != nil {
|
||||
logf("a warrant could not be handled and is read again from the router's record at the next start: %v", err)
|
||||
}
|
||||
})
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
func sayVia(ctx context.Context, js jetstream.JetStream, module string) func(c asks.Condition) error {
|
||||
return func(c asks.Condition) error {
|
||||
subject, err := asks.ConditionSubject(module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := asks.EncodeCondition(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = js.Publish(ctx, subject, data)
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -7,9 +7,10 @@
|
||||
// module placed since cannot serve its seat or reach a new state. Each is published on the assignments stream only when
|
||||
// it differs from the one last published there. **A bucket, log or stream a manifest declares that the bus lacks is
|
||||
// created** (the operator, 2026-10-26 08:54; issue 1790), once per manifest, with the manifest's limits, on the event
|
||||
// that brought the manifest; one the bus holds is never changed. **The user list, the rest of the assertions and the
|
||||
// grants stay in shadow** until
|
||||
// the operator's take-over, which this milestone always refuses: they are composed, compared with what the bus loaded
|
||||
// that brought the manifest; one the bus holds is never changed. **The user list is written sealed to the bus node**
|
||||
// (ADR 0547 clause 7) as an object of the part's own bucket named in `user-list.sealed`, which the bus node follows only
|
||||
// from take-over. **The rest of the assertions and the grants stay in shadow** until
|
||||
// the operator's take-over, asked on their channel and run on their yes after its refusals: they are composed, compared with what the bus loaded
|
||||
// and holds (through the bus module's own read tools) and kept in the part's own state, and take no effect.
|
||||
//
|
||||
// It holds no timer, and never exits for a refused read, write or publish: a refusal is said and tried again at the next
|
||||
@@ -39,6 +40,7 @@ import (
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
"git.novox.be/novox/mesh-sdk/go/asker"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-grants/internal/grants"
|
||||
@@ -115,8 +117,18 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
bus := newBusReader(ctx, js)
|
||||
f.Part = &grants.Part{Read: bus, Call: conn, Store: ownStore{conn}, Emit: emitting{conn},
|
||||
Publish: memberships{ctx: ctx, nc: nc, js: js}, Create: creating{ctx: ctx, js: js}}
|
||||
Publish: memberships{ctx: ctx, nc: nc, js: js}, Create: creating{ctx: ctx, js: js},
|
||||
Objects: sealedObjects{ctx: ctx, js: js}, Logf: logger.Printf}
|
||||
f.Watch = bus
|
||||
// The operator's word on take-over and hand-back is their yes on their channel: the part asks it itself, and acts on
|
||||
// the warrant once (mesh-sdk asker), after its refusals.
|
||||
client := &asker.Client{Module: account.Module, State: conn.State(grants.StateTakeover), Publish: publishVia(ctx, js),
|
||||
Record: recordVia(ctx, nc, account.Module), Subscribe: decidedVia(nc, account.Module, logger.Printf)}
|
||||
f.Part.Asker, f.Part.Say = client, sayVia(ctx, js, account.Module)
|
||||
f.Part.Forget = func(id string) error { return conn.State(grants.StateTakeover).Delete("ask." + id) }
|
||||
if err := client.Listen(f.Part.Warranted, f.Part.Said); err != nil {
|
||||
logger.Printf("the operator's answers cannot be heard yet, so take-over and hand-back wait: %v", err)
|
||||
}
|
||||
f.Asked = func(wake func()) error {
|
||||
return conn.State(grants.StateUserList).Watch(grants.KeyAsked, func(c stdio.StateChange) error {
|
||||
if !c.Current {
|
||||
@@ -125,7 +137,21 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return nil
|
||||
})
|
||||
}
|
||||
logger.Printf("started on the account %s: publishing memberships and creating the declared buckets, logs and streams the bus lacks; the user list, grants and any change to a held stream in shadow", account.User)
|
||||
// The controller makes this module's durable consumer on EVENTS from the manifest's consumes (filtered on the two
|
||||
// condition subjects); it is pulled and every message acknowledged, so its pending never grows. Where it is not there
|
||||
// yet the pass says so and hears again at the next change.
|
||||
f.Hear = func(handle func(string, []byte)) error {
|
||||
cons, err := js.Consumer(ctx, "EVENTS", account.Node+"_"+account.Module)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the consumer %s_%s on EVENTS: %w", account.Node, account.Module, err)
|
||||
}
|
||||
_, err = cons.Consume(func(m jetstream.Msg) {
|
||||
handle(m.Subject(), m.Data())
|
||||
_ = m.Ack()
|
||||
}, jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
|
||||
return err
|
||||
}
|
||||
logger.Printf("started on the account %s: publishing memberships and creating the declared buckets, logs and streams the bus lacks; and writing the user list sealed to the bus node, which the bus node follows from take-over; grants and any change to a held stream in shadow", account.User)
|
||||
f.Run(ctx)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// sealedObjects is the part's object bucket of user lists sealed to the bus node (novox/hq ADR 0547 clause 7), created on
|
||||
// its first put: the composer grants the part its create, its puts and deletes, and nothing of another bucket.
|
||||
type sealedObjects struct {
|
||||
ctx context.Context
|
||||
js jetstream.JetStream
|
||||
}
|
||||
|
||||
func (o sealedObjects) store(ctx context.Context) (jetstream.ObjectStore, error) {
|
||||
s, err := o.js.ObjectStore(ctx, delivery.GrantsUserListObjects)
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
s, err = o.js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: delivery.GrantsUserListObjects,
|
||||
Description: "the bus's user list composed by the grants part, each sealed to the bus node's key under the hex of its digest (novox/hq ADR 0547 clause 7): written by mesh-grants alone, read by name by the node-engine of the node holding mesh-broker",
|
||||
Storage: jetstream.FileStorage, MaxBytes: 64 << 20})
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the object bucket %s: %w", delivery.GrantsUserListObjects, err)
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (o sealedObjects) PutObject(name string, data []byte) error {
|
||||
ctx, cancel := context.WithTimeout(o.ctx, requestWithin)
|
||||
defer cancel()
|
||||
s, err := o.store(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.PutBytes(ctx, name, data)
|
||||
return err
|
||||
}
|
||||
|
||||
func (o sealedObjects) DeleteObject(name string) error {
|
||||
ctx, cancel := context.WithTimeout(o.ctx, requestWithin)
|
||||
defer cancel()
|
||||
s, err := o.store(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.Delete(ctx, name); err != nil && !errors.Is(err, jetstream.ErrObjectNotFound) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -97,20 +97,25 @@ func toolsOf(p *grants.Part) []stdio.Tool {
|
||||
"changed, a difference is only said. Answers the comparison and the ask; call again to see what the bus holds.",
|
||||
Input: schema(map[string]string{"module": "the module"}, "module"),
|
||||
Run: func(a map[string]any) (any, error) { return p.AssertOne(grants.Answer(a, "module")) }},
|
||||
{Name: grants.ToolTakeOver, Description: "The operator's switch to the grants part's list (ADR 0547 clause 8), refused without the " +
|
||||
"operator's credential (a person's account), refused while any user of the shadow list differs from the loaded list (but " +
|
||||
"those placement no longer names), naming them; in this milestone always refused, since the enrolment users and the " +
|
||||
"persons cannot be composed. Every ask is recorded in the part's state takeover.",
|
||||
{Name: grants.ToolTakeOver, Description: "Ask the operator to switch the bus node to the grants part's list (ADR 0547 clause 8). " +
|
||||
"The operator's word is their yes on their channel (a warrant), never a caller's name: this tool only asks, once at a time. " +
|
||||
"After the yes, the switch is recorded and takes effect when the bus node loads the part's list; it is refused while the loaded list could not be read, a user differs between the lists " +
|
||||
"(an enrolment user or a person the part cannot compose among them), or the vault holds no hash for a user, and the " +
|
||||
"refusal is said to the operator naming the users. Every act is recorded in the part's state takeover.",
|
||||
Input: schema(map[string]string{"why": "why, kept with the act"}, "why"),
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
return nil, p.TakeOver(stdio.CallerOf(a), grants.Answer(a, "why"))
|
||||
return p.TakeOver(stdio.CallerOf(a), grants.Answer(a, "why"))
|
||||
}},
|
||||
{Name: grants.ToolHandBack, Description: "The operator's switch back to the controller's list (ADR 0547 clause 8), refused without " +
|
||||
"the operator's credential; in shadow there is nothing to hand back, and it says so. Every ask is recorded.",
|
||||
{Name: grants.ToolHandBack, Description: "Ask the operator to switch the bus node back to the controller's list (ADR 0547 clause 8), " +
|
||||
"by their yes on their channel; in shadow there is nothing to hand back, and it says so. Every act is recorded.",
|
||||
Input: schema(map[string]string{"why": "why, kept with the act"}, "why"),
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
return nil, p.HandBack(stdio.CallerOf(a), grants.Answer(a, "why"))
|
||||
return p.HandBack(stdio.CallerOf(a), grants.Answer(a, "why"))
|
||||
}},
|
||||
{Name: grants.ToolTakeOverState, Description: "Where the switch stands: the list the bus node follows, the ask open now, the request, and " +
|
||||
"every act with what became of it (asked, switched, declined, expired, refused and why).",
|
||||
Input: schema(nil),
|
||||
Run: func(map[string]any) (any, error) { return p.TakeOverState() }},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user